From 7979c8b0cae18f0402ab535841dedce2972ae2dd Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Thu, 29 Aug 2024 14:24:47 +0300 Subject: [PATCH 01/65] fix(aggregation, RAM): MCOL-5715 Changes the second phase aggregation. (#3171) This patch changes the second phase aggregation pipeline - takes into account current memory consumption. Co-authored-by: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com> Co-authored-by: drrtuy --- dbcon/joblist/tupleaggregatestep.cpp | 228 +++++++++++++++------------ 1 file changed, 131 insertions(+), 97 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 0d6723083..ae31f0fa6 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -360,45 +360,86 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID) if (threadID >= fNumOfBuckets) return; - scoped_array rowBucketVecs(new RowBucketVec[fNumOfBuckets]); - scoped_array bucketDone(new bool[fNumOfBuckets]); - uint32_t hashlen = fAggregator->aggMapKeyLength(); + bool finishedSecondPhase = false; + bool diskAggAllowed = fRm->getAllowDiskAggregation(); + const uint32_t maxRowsSize = 8192; - try + while (!finishedSecondPhase && !fEndOfResult) { - RowAggregationDistinct* aggDist = dynamic_cast(fAggregators[threadID].get()); - RowAggregationMultiDistinct* multiDist = - dynamic_cast(fAggregators[threadID].get()); - Row rowIn; - RowGroup* rowGroupIn = nullptr; - rowGroupIn = (aggDist->aggregator()->getOutputRowGroup()); - uint32_t bucketID; - std::vector> rgDataVec; + scoped_array rowBucketVecs(new RowBucketVec[fNumOfBuckets]); + scoped_array bucketDone(new bool[fNumOfBuckets]); + uint32_t hashlen = fAggregator->aggMapKeyLength(); + bool outOfMemory = false; + size_t totalMemSizeConsumed = 0; - if (multiDist) + try { - for (uint32_t i = 0; i < fNumOfBuckets; i++) - rowBucketVecs[i].resize(multiDist->subAggregators().size()); - } - else - { - for (uint32_t i = 0; i < fNumOfBuckets; i++) - rowBucketVecs[i].resize(1); - } + RowAggregationDistinct* aggDist = dynamic_cast(fAggregators[threadID].get()); + RowAggregationMultiDistinct* multiDist = + dynamic_cast(fAggregators[threadID].get()); + Row rowIn; + RowGroup* rowGroupIn = nullptr; + rowGroupIn = (aggDist->aggregator()->getOutputRowGroup()); + uint32_t bucketID; + std::vector> rgDataVec; - // dispatch rows to bucket - if (multiDist) - { - for (uint32_t j = 0; j < multiDist->subAggregators().size(); j++) + if (multiDist) { - rowGroupIn = (multiDist->subAggregators()[j]->getOutputRowGroup()); - rowGroupIn->initRow(&rowIn); - auto* subDistAgg = dynamic_cast(multiDist->subAggregators()[j].get()); + for (uint32_t i = 0; i < fNumOfBuckets; i++) + rowBucketVecs[i].resize(multiDist->subAggregators().size()); + } + else + { + for (uint32_t i = 0; i < fNumOfBuckets; i++) + rowBucketVecs[i].resize(1); + } - while (subDistAgg->nextOutputRowGroup()) + // dispatch rows to bucket + if (multiDist) + { + for (uint32_t j = 0; j < multiDist->subAggregators().size(); j++) { rowGroupIn = (multiDist->subAggregators()[j]->getOutputRowGroup()); - rgDataVec.emplace_back(subDistAgg->moveCurrentRGData()); + rowGroupIn->initRow(&rowIn); + auto* subDistAgg = dynamic_cast(multiDist->subAggregators()[j].get()); + + while (subDistAgg->nextOutputRowGroup()) + { + rowGroupIn = (multiDist->subAggregators()[j]->getOutputRowGroup()); + rgDataVec.emplace_back(subDistAgg->moveCurrentRGData()); + rowGroupIn->getRow(0, &rowIn); + + for (uint64_t i = 0; i < rowGroupIn->getRowCount(); ++i) + { + // The key is the groupby columns, which are the leading columns. + // uint8_t* hashMapKey = rowIn.getData() + 2; + // bucketID = hash.operator()(hashMapKey) & fBucketMask; + uint64_t hash = rowgroup::hashRow(rowIn, hashlen - 1); + bucketID = hash % fNumOfBuckets; + rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash); + rowIn.nextRow(); + } + + const auto rgSize = diskAggAllowed ? rowGroupIn->getSizeWithStrings(maxRowsSize) + : rowGroupIn->getSizeWithStrings(); + totalMemSizeConsumed += rgSize; + if (!fRm->getMemory(rgSize, fSessionMemLimit, !diskAggAllowed)) + { + outOfMemory = true; + break; + } + } + } + } + else + { + rowGroupIn->initRow(&rowIn); + auto* subAgg = dynamic_cast(aggDist->aggregator().get()); + + while (subAgg->nextOutputRowGroup()) + { + rowGroupIn->setData(aggDist->aggregator()->getOutputRowGroup()->getRGData()); + rgDataVec.emplace_back(subAgg->moveCurrentRGData()); rowGroupIn->getRow(0, &rowIn); for (uint64_t i = 0; i < rowGroupIn->getRowCount(); ++i) @@ -408,87 +449,80 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID) // bucketID = hash.operator()(hashMapKey) & fBucketMask; uint64_t hash = rowgroup::hashRow(rowIn, hashlen - 1); bucketID = hash % fNumOfBuckets; - rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash); + rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash); rowIn.nextRow(); } - } - } - } - else - { - rowGroupIn->initRow(&rowIn); - auto* subAgg = dynamic_cast(aggDist->aggregator().get()); - while (subAgg->nextOutputRowGroup()) - { - rowGroupIn->setData(aggDist->aggregator()->getOutputRowGroup()->getRGData()); - rgDataVec.emplace_back(subAgg->moveCurrentRGData()); - rowGroupIn->getRow(0, &rowIn); - - for (uint64_t i = 0; i < rowGroupIn->getRowCount(); ++i) - { - // The key is the groupby columns, which are the leading columns. - // uint8_t* hashMapKey = rowIn.getData() + 2; - // bucketID = hash.operator()(hashMapKey) & fBucketMask; - uint64_t hash = rowgroup::hashRow(rowIn, hashlen - 1); - bucketID = hash % fNumOfBuckets; - rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash); - rowIn.nextRow(); - } - } - } - - bool done = false; - - // reset bucketDone[] to be false - // memset(bucketDone, 0, sizeof(bucketDone)); - fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false); - - while (!done && !cancelled()) - { - done = true; - - for (uint32_t c = 0; c < fNumOfBuckets && !cancelled(); c++) - { - if (!bucketDone[c] && fAgg_mutex[c]->try_lock()) - { - try + const auto rgSize = + diskAggAllowed ? rowGroupIn->getSizeWithStrings(maxRowsSize) : rowGroupIn->getSizeWithStrings(); + totalMemSizeConsumed += rgSize; + if (!fRm->getMemory(rgSize, fSessionMemLimit, !diskAggAllowed)) { - if (multiDist) - dynamic_cast(fAggregators[c].get()) - ->doDistinctAggregation_rowVec(rowBucketVecs[c]); - else - dynamic_cast(fAggregators[c].get()) - ->doDistinctAggregation_rowVec(rowBucketVecs[c][0]); + outOfMemory = true; + break; } - catch (...) + } + } + + if (!outOfMemory) + finishedSecondPhase = true; + + bool done = false; + // reset bucketDone[] to be false + // memset(bucketDone, 0, sizeof(bucketDone)); + fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false); + + while (!done && !cancelled()) + { + done = true; + + for (uint32_t c = 0; c < fNumOfBuckets && !cancelled(); c++) + { + if (!bucketDone[c] && fAgg_mutex[c]->try_lock()) { + try + { + if (multiDist) + dynamic_cast(fAggregators[c].get()) + ->doDistinctAggregation_rowVec(rowBucketVecs[c]); + else + dynamic_cast(fAggregators[c].get()) + ->doDistinctAggregation_rowVec(rowBucketVecs[c][0]); + } + catch (...) + { + fAgg_mutex[c]->unlock(); + throw; + } + fAgg_mutex[c]->unlock(); - throw; + bucketDone[c] = true; + rowBucketVecs[c][0].clear(); + } + else if (!bucketDone[c]) + { + done = false; } - - fAgg_mutex[c]->unlock(); - bucketDone[c] = true; - rowBucketVecs[c][0].clear(); - } - else if (!bucketDone[c]) - { - done = false; } } - } - if (cancelled()) + fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit); + if (cancelled()) + { + fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit); + finishedSecondPhase = true; + fEndOfResult = true; + } + } // try + catch (...) { + fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit); + handleException(std::current_exception(), logging::tupleAggregateStepErr, + logging::ERR_AGGREGATION_TOO_BIG, + "TupleAggregateStep::doThreadedSecondPhaseAggregate()"); fEndOfResult = true; + finishedSecondPhase = true; } - - } // try - catch (...) - { - handleException(std::current_exception(), logging::tupleAggregateStepErr, - logging::ERR_AGGREGATION_TOO_BIG, "TupleAggregateStep::doThreadedSecondPhaseAggregate()"); - fEndOfResult = true; } fDoneAggregate = true; From 148063e07a80264b00318e362f2be25bc6a26d28 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Fri, 23 Aug 2024 18:24:48 +0000 Subject: [PATCH 02/65] feat(ddl,partitions): better partition-related commands logging --- dbcon/ddlpackageproc/ddlpackageprocessor.h | 26 +++++++++++++++++++ .../ddlpackageproc/droppartitionprocessor.cpp | 3 ++- dbcon/ddlpackageproc/droppartitionprocessor.h | 2 +- .../ddlpackageproc/markpartitionprocessor.cpp | 3 ++- dbcon/ddlpackageproc/markpartitionprocessor.h | 2 +- .../restorepartitionprocessor.cpp | 3 ++- .../restorepartitionprocessor.h | 2 +- 7 files changed, 35 insertions(+), 6 deletions(-) diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.h b/dbcon/ddlpackageproc/ddlpackageprocessor.h index 6b3ccdf07..6d5171019 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.h +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.h @@ -863,6 +863,32 @@ bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&) return !(iss >> f >> t).fail(); } +class FormatStatementString +{ + public: + FormatStatementString() = default; + ~FormatStatementString() = default; + + std::string formatStatementString(const std::string& cmd, const std::string& schema, + const std::string& table, + const std::set& partitions) + { + std::ostringstream stmt; + stmt << cmd << "(" << schema << "," << table << ","; + + // There must be at least one partition to drop. + for (auto& p : partitions) + { + stmt << p << ","; + } + + auto res = stmt.str(); + res.back() = ')'; + + return res; + } +}; + } // namespace ddlpackageprocessor #undef EXPORT diff --git a/dbcon/ddlpackageproc/droppartitionprocessor.cpp b/dbcon/ddlpackageproc/droppartitionprocessor.cpp index f5fd55747..f01fa1ade 100644 --- a/dbcon/ddlpackageproc/droppartitionprocessor.cpp +++ b/dbcon/ddlpackageproc/droppartitionprocessor.cpp @@ -119,7 +119,8 @@ DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackageInternal return result; } - string stmt = dropPartitionStmt->fSql + "|" + dropPartitionStmt->fTableName->fSchema + "|"; + auto stmt = formatStatementString(dropPartitionStmt->fSql, dropPartitionStmt->fTableName->fSchema, + dropPartitionStmt->fTableName->fName, dropPartitionStmt->fPartitions); SQLLogger logger(stmt, fDDLLoggingId, sessionID, txnID.id); try diff --git a/dbcon/ddlpackageproc/droppartitionprocessor.h b/dbcon/ddlpackageproc/droppartitionprocessor.h index 6c0b66d39..914df8f56 100644 --- a/dbcon/ddlpackageproc/droppartitionprocessor.h +++ b/dbcon/ddlpackageproc/droppartitionprocessor.h @@ -33,7 +33,7 @@ namespace ddlpackageprocessor * for interacting with the Write Engine to process * drop table ddl statements. */ -class DropPartitionProcessor : public DDLPackageProcessor +class DropPartitionProcessor : public DDLPackageProcessor, FormatStatementString { public: DropPartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) diff --git a/dbcon/ddlpackageproc/markpartitionprocessor.cpp b/dbcon/ddlpackageproc/markpartitionprocessor.cpp index 6c87d5e0e..a5c166022 100644 --- a/dbcon/ddlpackageproc/markpartitionprocessor.cpp +++ b/dbcon/ddlpackageproc/markpartitionprocessor.cpp @@ -81,7 +81,8 @@ MarkPartitionProcessor::DDLResult MarkPartitionProcessor::processPackageInternal CalpontSystemCatalog::DictOIDList dictOIDList; std::string processName("DDLProc"); - string stmt = markPartitionStmt->fSql + "|" + markPartitionStmt->fTableName->fSchema + "|"; + auto stmt = formatStatementString(markPartitionStmt->fSql, markPartitionStmt->fTableName->fSchema, + markPartitionStmt->fTableName->fName, markPartitionStmt->fPartitions); SQLLogger logger(stmt, fDDLLoggingId, markPartitionStmt->fSessionID, txnID.id); uint32_t processID = 0; diff --git a/dbcon/ddlpackageproc/markpartitionprocessor.h b/dbcon/ddlpackageproc/markpartitionprocessor.h index c3661a81d..ed3454f94 100644 --- a/dbcon/ddlpackageproc/markpartitionprocessor.h +++ b/dbcon/ddlpackageproc/markpartitionprocessor.h @@ -33,7 +33,7 @@ namespace ddlpackageprocessor * for interacting with the Write Engine * to process create table ddl statements. */ -class MarkPartitionProcessor : public DDLPackageProcessor +class MarkPartitionProcessor : public DDLPackageProcessor, FormatStatementString { public: MarkPartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) diff --git a/dbcon/ddlpackageproc/restorepartitionprocessor.cpp b/dbcon/ddlpackageproc/restorepartitionprocessor.cpp index d2883222b..2ff559626 100644 --- a/dbcon/ddlpackageproc/restorepartitionprocessor.cpp +++ b/dbcon/ddlpackageproc/restorepartitionprocessor.cpp @@ -82,7 +82,8 @@ RestorePartitionProcessor::DDLResult RestorePartitionProcessor::processPackageIn CalpontSystemCatalog::DictOIDList dictOIDList; std::string processName("DDLProc"); - string stmt = restorePartitionStmt->fSql + "|" + restorePartitionStmt->fTableName->fSchema + "|"; + auto stmt = formatStatementString(restorePartitionStmt->fSql, restorePartitionStmt->fTableName->fSchema, + restorePartitionStmt->fTableName->fName, restorePartitionStmt->fPartitions); SQLLogger logger(stmt, fDDLLoggingId, restorePartitionStmt->fSessionID, txnID.id); uint32_t processID = 0; diff --git a/dbcon/ddlpackageproc/restorepartitionprocessor.h b/dbcon/ddlpackageproc/restorepartitionprocessor.h index 8e77c7eb9..d2b6e74f9 100644 --- a/dbcon/ddlpackageproc/restorepartitionprocessor.h +++ b/dbcon/ddlpackageproc/restorepartitionprocessor.h @@ -33,7 +33,7 @@ namespace ddlpackageprocessor * for interacting with the Write Engine to process * drop table ddl statements. */ -class RestorePartitionProcessor : public DDLPackageProcessor +class RestorePartitionProcessor : public DDLPackageProcessor, FormatStatementString { public: RestorePartitionProcessor(BRM::DBRM* aDbrm) : DDLPackageProcessor(aDbrm) From f6c9d3b89d2d6f3c4951bdba14f4af7e166d7a38 Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Wed, 3 Jul 2024 12:54:11 +0300 Subject: [PATCH 03/65] MCOL-5695: Add FoundationDB package as a dependency for CMAPI. --- .drone.jsonnet | 6 ++++-- cmapi/CMakeLists.txt | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 9e0ad2d65..1fb531914 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -590,10 +590,12 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') image: 'plugins/docker', environment: { VERSION: container_version, - MCS_REPO: 'columnstore', DEV: 'true', + MCS_REPO: 'columnstore', // branchp has slash if not empty MCS_BASEURL: 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/', + FDB_REPO: 'foundationdb', + FDB_BASEURL: 'https://cspkg.s3.amazonaws.com/FoundationDB/' + arch + '/' + result + '/', CMAPI_REPO: 'cmapi', CMAPI_BASEURL: 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/', }, @@ -601,7 +603,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') repo: 'mariadb/enterprise-columnstore-dev', context: 'docker', dockerfile: 'docker/Dockerfile', - build_args_from_env: ['VERSION', 'MCS_REPO', 'MCS_BASEURL', 'CMAPI_REPO', 'CMAPI_BASEURL', 'DEV'], + build_args_from_env: ['VERSION', 'MCS_REPO', 'MCS_BASEURL', 'CMAPI_REPO', 'CMAPI_BASEURL', 'DEV', 'FDB_REPO', 'FDB_BASEURL'], tags: container_tags, username: { from_secret: 'dockerhub_user', diff --git a/cmapi/CMakeLists.txt b/cmapi/CMakeLists.txt index b546560ed..af72e94c1 100644 --- a/cmapi/CMakeLists.txt +++ b/cmapi/CMakeLists.txt @@ -132,7 +132,7 @@ IF(RPM) SET(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAPI_CONF_FILEPATH}") SET(CPACK_RPM_PACKAGE_OBSOLETES "mariadb-columnstore-cmapi") - SET(CPACK_RPM_PACKAGE_REQUIRES "curl") + SET(CPACK_RPM_PACKAGE_REQUIRES "curl, foundationdb-server") STRING(REPLACE "-" "_" SERVER_VERSION ${SERVER_VERSION}) get_linux_lsb_release_information() @@ -174,7 +174,7 @@ IF(DEB) SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/prerm;${CMAKE_CURRENT_SOURCE_DIR}/postinst;${CMAKE_CURRENT_SOURCE_DIR}/conffiles") SET(CPACK_DEBIAN_PACKAGE_REPLACES "mariadb-columnstore-cmapi") - SET(CPACK_DEBIAN_PACKAGE_DEPENDS "curl") + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "curl, foundationdb-server") STRING(REPLACE "-" "." SERVER_VERSION ${SERVER_VERSION}) SET(PATCHLEVEL "+maria") get_linux_lsb_release_information() From 1d40b4bb4518d13e9495dbe4778d3faa32a57164 Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Tue, 3 Sep 2024 15:52:42 +0300 Subject: [PATCH 04/65] MCOL-5695: Fix CI. Use specific branches from docker repo. --- .drone.jsonnet | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 1fb531914..9c88e151c 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -144,7 +144,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') local socket_path = if (pkg_format == 'rpm') then '/var/lib/mysql/mysql.sock' else '/run/mysqld/mysqld.sock', local config_path_prefix = if (pkg_format == 'rpm') then '/etc/my.cnf.d/' else '/etc/mysql/mariadb.conf.d/50-', local img = if (platform == 'rockylinux:8') then platform else 'detravi/' + std.strReplace(platform, '/', '-'), - local regression_ref = if (branch == any_branch) then 'develop' else branch, + local branch_ref = if (branch == any_branch) then 'develop' else branch, // local regression_tests = if (std.startsWith(platform, 'debian') || std.startsWith(platform, 'ubuntu:20')) then 'test000.sh' else 'test000.sh,test001.sh', local branchp = if (branch == '**') then '' else branch + '/', @@ -461,7 +461,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], environment: { REGRESSION_BRANCH_REF: '${DRONE_SOURCE_BRANCH}', - REGRESSION_REF_AUX: regression_ref, + REGRESSION_REF_AUX: branch_ref, }, commands: [ // compute branch. @@ -578,8 +578,22 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') depends_on: ['publish pkg', 'publish cmapi build'], //failure: 'ignore', image: 'alpine/git', + environment: { + DOCKER_BRANCH_REF: '${DRONE_SOURCE_BRANCH}', + DOCKER_REF_AUX: branch_ref, + }, commands: [ - 'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker', + // compute branch. + 'echo "$$DOCKER_REF"', + 'echo "$$DOCKER_BRANCH_REF"', + // if DOCKER_REF is empty, try to see whether docker repository has a branch named as one we PR. + 'export DOCKER_REF=$${DOCKER_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-docker --h --sort origin "refs/heads/$$DOCKER_BRANCH_REF" | grep -E -o "[^/]+$$")}', + 'echo "$$DOCKER_REF"', + // DOCKER_REF can be empty if there is no appropriate branch in docker repository. + // assign what is appropriate by default. + 'export DOCKER_REF=$${DOCKER_REF:-$$DOCKER_REF_AUX}', + 'echo "$$DOCKER_REF"', + 'git clone --branch $$DOCKER_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker', 'touch docker/.secrets', ], }, From 47d01b2d2fb291c6feb99fc14eef728758555bc5 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Wed, 28 Aug 2024 18:18:57 +0100 Subject: [PATCH 05/65] fix(join, UM, perf): UM join is multi-threaded now (#3286) * chore: UM join is multi-threaded now * fix(UMjoin): replace TR1 maps with stdlib versions --- dbcon/joblist/tuplehashjoin.cpp | 58 +++++++++++++++------------------ dbcon/joblist/tuplehashjoin.h | 49 ++++++++++++++-------------- utils/joiner/tuplejoiner.cpp | 27 ++++++++------- utils/joiner/tuplejoiner.h | 15 ++++----- 4 files changed, 75 insertions(+), 74 deletions(-) diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 3428756ed..e6c6b9796 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -29,7 +29,7 @@ #include #include #include -//#define NDEBUG +// #define NDEBUG #include #include using namespace std; @@ -119,12 +119,9 @@ TupleHashJoinStep::TupleHashJoinStep(const JobInfo& jobInfo) allowDJS = false; numCores = resourceManager->numCores(); + if (numCores <= 0) numCores = 8; - /* Debugging, rand() is used to simulate failures - time_t t = time(NULL); - srand(t); - */ } TupleHashJoinStep::~TupleHashJoinStep() @@ -139,8 +136,8 @@ TupleHashJoinStep::~TupleHashJoinStep() for (uint i = 0; i < smallDLs.size(); i++) { if (memUsedByEachJoin[i]) - resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); - } + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); + } } returnMemory(); // cout << "deallocated THJS, UM memory available: " << resourceManager.availableMemory() << endl; @@ -207,7 +204,6 @@ void TupleHashJoinStep::join() jobstepThreadPool.join(djsReader); jobstepThreadPool.join(djsRelay); - // cout << "THJS: joined all DJS threads, shared usage = " << *djsSmallUsage << endl; } } @@ -228,7 +224,7 @@ void TupleHashJoinStep::trackMem(uint index) { gotMem = resourceManager->getMemory(memAfter - memBefore, sessionMemLimit, true); if (gotMem) - atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); + atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); else return; @@ -245,7 +241,7 @@ void TupleHashJoinStep::trackMem(uint index) gotMem = resourceManager->getMemory(memAfter - memBefore, sessionMemLimit, true); if (gotMem) { - atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); + atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); } else { @@ -305,28 +301,32 @@ void TupleHashJoinStep::startSmallRunners(uint index) handle abort, out of memory, etc */ - /* To measure wall-time spent constructing the small-side tables... - boost::posix_time::ptime end_time, start_time = - boost::posix_time::microsec_clock::universal_time(); - */ - stopMemTracking = false; utils::VLArray jobs(numCores); uint64_t memMonitor = jobstepThreadPool.invoke([this, index] { this->trackMem(index); }); // starting 1 thread when in PM mode, since it's only inserting into a // vector of rows. The rest will be started when converted to UM mode. if (joiner->inUM()) + { for (int i = 0; i < numCores; i++) + { jobs[i] = jobstepThreadPool.invoke([this, i, index, &jobs] { this->smallRunnerFcn(index, i, jobs); }); + } + } else + { jobs[0] = jobstepThreadPool.invoke([this, index, &jobs] { this->smallRunnerFcn(index, 0, jobs); }); + } // wait for the first thread to join, then decide whether the others exist and need joining jobstepThreadPool.join(jobs[0]); if (joiner->inUM()) + { for (int i = 1; i < numCores; i++) + { jobstepThreadPool.join(jobs[i]); - + } + } // stop the monitor thread memTrackMutex.lock(); stopMemTracking = true; @@ -435,7 +435,7 @@ void TupleHashJoinStep::smallRunnerFcn(uint32_t index, uint threadID, uint64_t* gotMem = resourceManager->getMemory(rgSize, sessionMemLimit, true); if (gotMem) { - atomicops::atomicAdd(&memUsedByEachJoin[index], rgSize); + atomicops::atomicAdd(&memUsedByEachJoin[index], rgSize); } else { @@ -468,9 +468,12 @@ void TupleHashJoinStep::smallRunnerFcn(uint32_t index, uint threadID, uint64_t* if (!joiner->inUM() && (memUsedByEachJoin[index] > pmMemLimit)) { joiner->setInUM(rgData[index]); + for (int i = 1; i < numCores; i++) + { jobs[i] = jobstepThreadPool.invoke([this, i, index, jobs] { this->smallRunnerFcn(index, i, jobs); }); + } } next: dlMutex.lock(); @@ -1727,8 +1730,8 @@ void TupleHashJoinStep::joinOneRG( std::shared_ptr& smallRowTemplates, RowGroupDL* outputDL, // disk-join support vars. This param list is insane; refactor attempt would be nice at some point. vector >* tjoiners, - std::shared_ptr[] >* rgMappings, - std::shared_ptr[] >* feMappings, + std::shared_ptr[]>* rgMappings, + std::shared_ptr[]>* feMappings, boost::scoped_array >* smallNullMem) { /* Disk-join support. @@ -1765,15 +1768,7 @@ void TupleHashJoinStep::joinOneRG( for (j = 0; j < smallSideCount; j++) { (*tjoiners)[j]->match(largeSideRow, k, threadID, &joinMatches[j]); - /* Debugging code to print the matches - Row r; - smallRGs[j].initRow(&r); - cout << joinMatches[j].size() << " matches: \n"; - for (uint32_t z = 0; z < joinMatches[j].size(); z++) { - r.setData(joinMatches[j][z]); - cout << " " << r.toString() << endl; - } - */ + matchCount = joinMatches[j].size(); if ((*tjoiners)[j]->hasFEFilter() && matchCount > 0) @@ -1859,10 +1854,11 @@ void TupleHashJoinStep::joinOneRG( } void TupleHashJoinStep::generateJoinResultSet(const vector >& joinerOutput, Row& baseRow, - const std::shared_ptr[] >& mappings, + const std::shared_ptr[]>& mappings, const uint32_t depth, RowGroup& l_outputRG, RGData& rgData, - vector& outputData, const std::shared_ptr& smallRows, - Row& joinedRow, RowGroupDL* dlp) + vector& outputData, + const std::shared_ptr& smallRows, Row& joinedRow, + RowGroupDL* dlp) { uint32_t i; Row& smallRow = smallRows[depth]; diff --git a/dbcon/joblist/tuplehashjoin.h b/dbcon/joblist/tuplehashjoin.h index 519beba6f..1636ccd18 100644 --- a/dbcon/joblist/tuplehashjoin.h +++ b/dbcon/joblist/tuplehashjoin.h @@ -74,8 +74,10 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep void tableOid1(execplan::CalpontSystemCatalog::OID tableOid1) { fTableOID1 = tableOid1; - if (fTableOID1 < 3000) + if (fTableOID1 >= 1000 && fTableOID1 < 3000) + { numCores = 1; // syscat query, no need for more than 1 thread + } } void tableOid2(execplan::CalpontSystemCatalog::OID tableOid2) { @@ -199,16 +201,16 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep void addSmallSideRG(const std::vector& rgs, const std::vector& tableNames); void addJoinKeyIndex(const std::vector& jt, const std::vector& typeless, - const std::vector >& smallkeys, - const std::vector >& largekeys); + const std::vector>& smallkeys, + const std::vector>& largekeys); void configSmallSideRG(const std::vector& rgs, const std::vector& tableNames); void configLargeSideRG(const rowgroup::RowGroup& rg); void configJoinKeyIndex(const std::vector& jt, const std::vector& typeless, - const std::vector >& smallkeys, - const std::vector >& largekeys); + const std::vector>& smallkeys, + const std::vector>& largekeys); void setOutputRowGroup(const rowgroup::RowGroup& rg); @@ -234,11 +236,11 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep { return smallSideKeys[0][0]; } - const std::vector >& getSmallKeys() const + const std::vector>& getSmallKeys() const { return smallSideKeys; } - const std::vector >& getLargeKeys() const + const std::vector>& getLargeKeys() const { return largeSideKeys; } @@ -434,8 +436,8 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep int fCorrelatedSide; std::vector typelessJoin; // the size of the vector is # of small side - std::vector > largeSideKeys; - std::vector > smallSideKeys; + std::vector> largeSideKeys; + std::vector> smallSideKeys; ResourceManager* resourceManager; uint64_t fMemSizeForOutputRG; @@ -448,8 +450,8 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep return *j1 < *j2; } }; - std::vector > joiners; - boost::scoped_array > rgData; + std::vector> joiners; + boost::scoped_array> rgData; TupleBPS* largeBPS; rowgroup::RowGroup largeRG, outputRG; std::vector smallRGs; @@ -511,7 +513,7 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep /* Semi-join support */ std::vector feIndexes; - std::vector > fe; + std::vector> fe; rowgroup::RowGroup joinFilterRG; /* Casual Partitioning forwarding */ @@ -534,10 +536,10 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep }; void joinRunnerFcn(uint32_t index); void startJoinThreads(); - void generateJoinResultSet(const std::vector >& joinerOutput, + void generateJoinResultSet(const std::vector>& joinerOutput, rowgroup::Row& baseRow, - const std::shared_ptr[] >& mappings, - const uint32_t depth, rowgroup::RowGroup& outputRG, rowgroup::RGData& rgData, + const std::shared_ptr[]>& mappings, const uint32_t depth, + rowgroup::RowGroup& outputRG, rowgroup::RGData& rgData, std::vector& outputData, const std::shared_ptr& smallRows, rowgroup::Row& joinedRow, RowGroupDL* outputDL); @@ -549,11 +551,11 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep void joinOneRG(uint32_t threadID, std::vector& out, rowgroup::RowGroup& inputRG, rowgroup::RowGroup& joinOutput, rowgroup::Row& largeSideRow, rowgroup::Row& joinFERow, rowgroup::Row& joinedRow, rowgroup::Row& baseRow, - std::vector >& joinMatches, + std::vector>& joinMatches, std::shared_ptr& smallRowTemplates, RowGroupDL* outputDL, - std::vector >* joiners = NULL, - std::shared_ptr[] >* rgMappings = NULL, - std::shared_ptr[] >* feMappings = NULL, + std::vector>* joiners = NULL, + std::shared_ptr[]>* rgMappings = NULL, + std::shared_ptr[]>* feMappings = NULL, boost::scoped_array>* smallNullMem = NULL); void finishSmallOuterJoin(); void makeDupList(const rowgroup::RowGroup& rg); @@ -564,10 +566,10 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep std::shared_ptr[]> columnMappings, fergMappings; std::shared_ptr fe2Mapping; uint32_t joinThreadCount; - boost::scoped_array > smallNullMemory; + boost::scoped_array> smallNullMemory; uint64_t outputIt; bool moreInput; - std::vector > dupList; + std::vector> dupList; boost::scoped_array dupRows; std::vector smallTableNames; bool isExeMgr; @@ -633,8 +635,8 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep bool ownsOutputDL; void segregateJoiners(); - std::vector > tbpsJoiners; - std::vector > djsJoiners; + std::vector> tbpsJoiners; + std::vector> djsJoiners; std::vector djsJoinerMap; boost::scoped_array memUsedByEachJoin; boost::mutex djsLock; @@ -653,4 +655,3 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep }; } // namespace joblist - diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 3dea44b56..0db8f88db 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -20,12 +20,13 @@ #include #include #include -#include +#include #include "hasher.h" #include "lbidlist.h" #include "spinlock.h" #include "vlarray.h" +#include "threadnaming.h" using namespace std; using namespace rowgroup; @@ -282,8 +283,7 @@ void TupleJoiner::bucketsToTables(buckets_t* buckets, hash_table_t* tables) done = false; continue; } - for (auto& element : buckets[i]) - tables[i]->insert(element); + tables[i]->insert(buckets[i].begin(), buckets[i].end()); m_bucketLocks[i].unlock(); wasProductive = true; buckets[i].clear(); @@ -306,7 +306,7 @@ void TupleJoiner::um_insertTypeless(uint threadID, uint rowCount, Row& r) if (td[i].len == 0) continue; uint bucket = bucketPicker((char*)td[i].data, td[i].len, bpSeed) & bucketMask; - v[bucket].push_back(pair(td[i], r.getPointer())); + v[bucket].emplace_back(pair(td[i], r.getPointer())); } bucketsToTables(&v[0], ht.get()); } @@ -323,9 +323,9 @@ void TupleJoiner::um_insertLongDouble(uint rowCount, Row& r) uint bucket = bucketPicker((char*)&smallKey, 10, bpSeed) & bucketMask; // change if we decide to support windows again if (UNLIKELY(smallKey == joblist::LONGDOUBLENULL)) - v[bucket].push_back(pair(joblist::LONGDOUBLENULL, r.getPointer())); + v[bucket].emplace_back(pair(joblist::LONGDOUBLENULL, r.getPointer())); else - v[bucket].push_back(pair(smallKey, r.getPointer())); + v[bucket].emplace_back(pair(smallKey, r.getPointer())); } bucketsToTables(&v[0], ld.get()); } @@ -345,9 +345,9 @@ void TupleJoiner::um_insertInlineRows(uint rowCount, Row& r) smallKey = (int64_t)r.getUintField(smallKeyColumn); uint bucket = bucketPicker((char*)&smallKey, sizeof(smallKey), bpSeed) & bucketMask; if (UNLIKELY(smallKey == nullValueForJoinColumn)) - v[bucket].push_back(pair(getJoinNullValue(), r.getData())); + v[bucket].emplace_back(pair(getJoinNullValue(), r.getData())); else - v[bucket].push_back(pair(smallKey, r.getData())); + v[bucket].emplace_back(pair(smallKey, r.getData())); } bucketsToTables(&v[0], h.get()); } @@ -367,9 +367,9 @@ void TupleJoiner::um_insertStringTable(uint rowCount, Row& r) smallKey = (int64_t)r.getUintField(smallKeyColumn); uint bucket = bucketPicker((char*)&smallKey, sizeof(smallKey), bpSeed) & bucketMask; if (UNLIKELY(smallKey == nullValueForJoinColumn)) - v[bucket].push_back(pair(getJoinNullValue(), r.getPointer())); + v[bucket].emplace_back(pair(getJoinNullValue(), r.getPointer())); else - v[bucket].push_back(pair(smallKey, r.getPointer())); + v[bucket].emplace_back(pair(smallKey, r.getPointer())); } bucketsToTables(&v[0], sth.get()); } @@ -670,6 +670,8 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin } } +using unordered_set_int128 = std::unordered_set; + void TupleJoiner::doneInserting() { // a minor textual cleanup @@ -694,7 +696,6 @@ void TupleJoiner::doneInserting() for (col = 0; col < smallKeyColumns.size(); col++) { - typedef std::tr1::unordered_set unordered_set_int128; unordered_set_int128 uniquer; unordered_set_int128::iterator uit; sthash_t::iterator sthit; @@ -811,6 +812,8 @@ void TupleJoiner::setInPM() void TupleJoiner::umJoinConvert(size_t begin, size_t end) { + utils::setThreadName("TJUMJoinConvert1"); + Row smallRow; smallRG.initRow(&smallRow); @@ -862,6 +865,8 @@ void TupleJoiner::setInUM() void TupleJoiner::umJoinConvert(uint threadID, vector& rgs, size_t begin, size_t end) { + utils::setThreadName("TJUMJoinConvert2"); + RowGroup l_smallRG(smallRG); while (begin < end) diff --git a/utils/joiner/tuplejoiner.h b/utils/joiner/tuplejoiner.h index f97045351..aa4aaf647 100644 --- a/utils/joiner/tuplejoiner.h +++ b/utils/joiner/tuplejoiner.h @@ -24,7 +24,7 @@ #include #include -#include +#include #include "rowgroup.h" #include "joiner.h" @@ -464,19 +464,18 @@ class TupleJoiner void setConvertToDiskJoin(); private: - typedef std::tr1::unordered_multimap, - utils::STLPoolAllocator > > + typedef std::unordered_multimap, + utils::STLPoolAllocator > > hash_t; - typedef std::tr1::unordered_multimap< - int64_t, rowgroup::Row::Pointer, hasher, std::equal_to, - utils::STLPoolAllocator > > + typedef std::unordered_multimap, + utils::STLPoolAllocator > > sthash_t; - typedef std::tr1::unordered_multimap< + typedef std::unordered_multimap< TypelessData, rowgroup::Row::Pointer, hasher, std::equal_to, utils::STLPoolAllocator > > typelesshash_t; // MCOL-1822 Add support for Long Double AVG/SUM small side - typedef std::tr1::unordered_multimap< + typedef std::unordered_multimap< long double, rowgroup::Row::Pointer, hasher, LongDoubleEq, utils::STLPoolAllocator > > ldhash_t; From 029a5736398a2cbe1d5617a3dbcbcd46d2231c5c Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Thu, 5 Sep 2024 14:49:08 +0000 Subject: [PATCH 06/65] chore(tests): canonize tests after server MDEV-19052 chore(tests): canonize hex(-1) after some server fixes chore(tools) update fullmtr manual runner chore(tests): canonize hex values for negative --- .../r/mcs4463_function_CNX_HEX_NS.result | 12 +++++----- .../r/mcs4471_function_CNXasPP_HEX_NS.result | 12 +++++----- .../mcs127_window_function_last_value.result | 22 +++++++++++++++++-- .../r/mcs129_window_function_nth_value.result | 22 +++++++++++++++++-- .../basic/r/mcs152_win_frame_avg.result | 22 +++++++++++++++++-- .../basic/r/mcs153_win_frame_bit_and.result | 4 ++-- .../basic/r/mcs154_win_frame_bit_or.result | 4 ++-- .../basic/r/mcs155_win_frame_bit_xor.result | 4 ++-- .../basic/r/mcs156_win_frame_count.result | 22 +++++++++++++++++-- .../basic/r/mcs157_win_frame_lead.result | 11 +++++++++- .../basic/r/mcs158_win_frame_max.result | 22 +++++++++++++++++-- .../basic/r/mcs159_win_frame_min.result | 22 +++++++++++++++++-- .../basic/r/mcs161_win_frame_std.result | 22 +++++++++++++++++-- .../basic/r/mcs162_win_frame_stddev.result | 22 +++++++++++++++++-- .../r/mcs163_win_frame_stddev_pop.result | 22 +++++++++++++++++-- .../r/mcs164_win_frame_stddev_samp.result | 22 +++++++++++++++++-- .../basic/r/mcs165_win_frame_sum.result | 22 +++++++++++++++++-- .../basic/r/mcs166_win_frame_var_pop.result | 22 +++++++++++++++++-- .../basic/r/mcs167_win_frame_var_samp.result | 22 +++++++++++++++++-- .../basic/r/mcs168_win_frame_variance.result | 22 +++++++++++++++++-- .../basic/r/mcs254_hex_function.result | 2 +- .../t/mcs127_window_function_last_value.test | 3 --- .../t/mcs129_window_function_nth_value.test | 3 --- .../basic/t/mcs152_win_frame_avg.test | 3 --- .../basic/t/mcs153_win_frame_bit_and.test | 5 ++--- .../basic/t/mcs154_win_frame_bit_or.test | 5 ++--- .../basic/t/mcs155_win_frame_bit_xor.test | 4 ++-- .../basic/t/mcs156_win_frame_count.test | 6 ++--- .../basic/t/mcs157_win_frame_lead.test | 1 - .../basic/t/mcs158_win_frame_max.test | 3 --- .../basic/t/mcs159_win_frame_min.test | 3 --- .../basic/t/mcs161_win_frame_std.test | 3 --- .../basic/t/mcs162_win_frame_stddev.test | 3 --- .../basic/t/mcs163_win_frame_stddev_pop.test | 3 --- .../basic/t/mcs164_win_frame_stddev_samp.test | 3 --- .../basic/t/mcs165_win_frame_sum.test | 5 +---- .../basic/t/mcs166_win_frame_var_pop.test | 3 --- .../basic/t/mcs167_win_frame_var_samp.test | 3 --- .../basic/t/mcs168_win_frame_variance.test | 3 --- 39 files changed, 318 insertions(+), 101 deletions(-) diff --git a/mysql-test/columnstore/autopilot/r/mcs4463_function_CNX_HEX_NS.result b/mysql-test/columnstore/autopilot/r/mcs4463_function_CNX_HEX_NS.result index f30a30155..94261359d 100644 --- a/mysql-test/columnstore/autopilot/r/mcs4463_function_CNX_HEX_NS.result +++ b/mysql-test/columnstore/autopilot/r/mcs4463_function_CNX_HEX_NS.result @@ -5,16 +5,16 @@ cidx CBIGINT HEX(CBIGINT) 1 -72036854775806 FFFFBE7B9CDC4002 select cidx, CDECIMAL1, HEX(CDECIMAL1) from datatypetestm order by cidx; cidx CDECIMAL1 HEX(CDECIMAL1) -1 -9 FFFF +1 -9 FFFFFFFFFFFFFFF7 select cidx, CDECIMAL4, HEX(CDECIMAL4) from datatypetestm order by cidx; cidx CDECIMAL4 HEX(CDECIMAL4) -1 -999 FFFFFFFFFF +1 -999 FFFFFFFFFFFFFC19 select cidx, CDECIMAL4_2, HEX(CDECIMAL4_2) from datatypetestm order by cidx; cidx CDECIMAL4_2 HEX(CDECIMAL4_2) -1 -9.99 FFFFFFFFFFFF +1 -9.99 FFFFFFFFFFFFFFF6 select cidx, CDECIMAL5, HEX(CDECIMAL5) from datatypetestm order by cidx; cidx CDECIMAL5 HEX(CDECIMAL5) -1 -999 FFFFFFFFFFFF +1 -999 FFFFFFFFFFFFFC19 select cidx, CDECIMAL9, HEX(CDECIMAL9) from datatypetestm order by cidx; cidx CDECIMAL9 HEX(CDECIMAL9) 1 -999999 FFFFFFFFFFF0BDC1 @@ -35,10 +35,10 @@ cidx CINTEGER HEX(CINTEGER) 1 -7483646 FFFFFFFFFF8DCF02 select cidx, CSMALLINT, HEX(CSMALLINT) from datatypetestm order by cidx; cidx CSMALLINT HEX(CSMALLINT) -1 -766 FFFFFFFFFFFF +1 -766 FFFFFFFFFFFFFD02 select cidx, CTINYINT, HEX(CTINYINT) from datatypetestm order by cidx; cidx CTINYINT HEX(CTINYINT) -1 -26 FFFFFFFF +1 -26 FFFFFFFFFFFFFFE6 select cidx, CDOUBLE, HEX(CDOUBLE) from datatypetestm order by cidx; cidx CDOUBLE HEX(CDOUBLE) 1 -1.797693231e108 FFFFFFFFFFFFFFFF diff --git a/mysql-test/columnstore/autopilot/r/mcs4471_function_CNXasPP_HEX_NS.result b/mysql-test/columnstore/autopilot/r/mcs4471_function_CNXasPP_HEX_NS.result index 853356335..51da9f511 100644 --- a/mysql-test/columnstore/autopilot/r/mcs4471_function_CNXasPP_HEX_NS.result +++ b/mysql-test/columnstore/autopilot/r/mcs4471_function_CNXasPP_HEX_NS.result @@ -5,16 +5,16 @@ cidx CBIGINT HEX(CBIGINT) 1 -72036854775806 FFFFBE7B9CDC4002 select cidx, CDECIMAL1, HEX(CDECIMAL1) from datatypetestm order by cidx; cidx CDECIMAL1 HEX(CDECIMAL1) -1 -9 FFFF +1 -9 FFFFFFFFFFFFFFF7 select cidx, CDECIMAL4, HEX(CDECIMAL4) from datatypetestm order by cidx; cidx CDECIMAL4 HEX(CDECIMAL4) -1 -999 FFFFFFFFFF +1 -999 FFFFFFFFFFFFFC19 select cidx, CDECIMAL4_2, HEX(CDECIMAL4_2) from datatypetestm order by cidx; cidx CDECIMAL4_2 HEX(CDECIMAL4_2) -1 -9.99 FFFFFFFFFFFF +1 -9.99 FFFFFFFFFFFFFFF6 select cidx, CDECIMAL5, HEX(CDECIMAL5) from datatypetestm order by cidx; cidx CDECIMAL5 HEX(CDECIMAL5) -1 -999 FFFFFFFFFFFF +1 -999 FFFFFFFFFFFFFC19 select cidx, CDECIMAL9, HEX(CDECIMAL9) from datatypetestm order by cidx; cidx CDECIMAL9 HEX(CDECIMAL9) 1 -999999 FFFFFFFFFFF0BDC1 @@ -35,10 +35,10 @@ cidx CINTEGER HEX(CINTEGER) 1 -7483646 FFFFFFFFFF8DCF02 select cidx, CSMALLINT, HEX(CSMALLINT) from datatypetestm order by cidx; cidx CSMALLINT HEX(CSMALLINT) -1 -766 FFFFFFFFFFFF +1 -766 FFFFFFFFFFFFFD02 select cidx, CTINYINT, HEX(CTINYINT) from datatypetestm order by cidx; cidx CTINYINT HEX(CTINYINT) -1 -26 FFFFFFFF +1 -26 FFFFFFFFFFFFFFE6 select cidx, CDOUBLE, HEX(CDOUBLE) from datatypetestm order by cidx; cidx CDOUBLE HEX(CDOUBLE) 1 -1.797693231e108 FFFFFFFFFFFFFFFF diff --git a/mysql-test/columnstore/basic/r/mcs127_window_function_last_value.result b/mysql-test/columnstore/basic/r/mcs127_window_function_last_value.result index 999c12007..b6f1c41b2 100644 --- a/mysql-test/columnstore/basic/r/mcs127_window_function_last_value.result +++ b/mysql-test/columnstore/basic/r/mcs127_window_function_last_value.result @@ -147,7 +147,25 @@ c 1861 1861 d 10701 1071 d 1071 1071 SELECT a, b, LAST_VALUE(b) OVER(PARTITION BY a ORDER BY a DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b last_value +NULL NULL NULL +a 92 123 +a 1 123 +a 123 123 +b 123 123 +c 1991 1861 +c 1861 1861 +d 1071 10701 +d 10701 10701 SELECT a, b, LAST_VALUE(b) OVER(PARTITION BY b ORDER BY a DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b last_value +NULL NULL NULL +a 1 1 +a 92 92 +b 123 123 +a 123 123 +d 1071 1071 +c 1861 1861 +c 1991 1991 +d 10701 10701 DROP DATABASE mcs127_db; diff --git a/mysql-test/columnstore/basic/r/mcs129_window_function_nth_value.result b/mysql-test/columnstore/basic/r/mcs129_window_function_nth_value.result index c4b57ff25..ebaca3bdb 100644 --- a/mysql-test/columnstore/basic/r/mcs129_window_function_nth_value.result +++ b/mysql-test/columnstore/basic/r/mcs129_window_function_nth_value.result @@ -213,7 +213,25 @@ c 1861 NULL c 1991 NULL d 10701 NULL SELECT a, b, NTH_VALUE(b, 2) OVER(PARTITION BY b ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_value FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b second_value +NULL NULL NULL +a 1 NULL +a 92 NULL +a 123 123 +b 123 123 +d 1071 NULL +c 1861 NULL +c 1991 NULL +d 10701 NULL SELECT a, b, NTH_VALUE(a, 2) OVER(PARTITION BY a ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_value FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b second_value +NULL NULL NULL +a 92 a +a 1 a +a 123 a +b 123 NULL +c 1991 c +c 1861 c +d 1071 d +d 10701 d DROP DATABASE mcs129_db; diff --git a/mysql-test/columnstore/basic/r/mcs152_win_frame_avg.result b/mysql-test/columnstore/basic/r/mcs152_win_frame_avg.result index 7e662dc3b..948d92442 100644 --- a/mysql-test/columnstore/basic/r/mcs152_win_frame_avg.result +++ b/mysql-test/columnstore/basic/r/mcs152_win_frame_avg.result @@ -224,7 +224,25 @@ d 17 17.0000 b 18 18.0000 a 19 19.0000 SELECT a, b, AVG(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) avg FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b avg +NULL NULL NULL +a 12 12.0000 +a 13 13.0000 +b 14 14.0000 +c 15 15.0000 +d 16 16.0000 +d 17 17.0000 +b 18 18.0000 +a 19 19.0000 SELECT a, b, AVG(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) avg FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b avg +NULL NULL NULL +a 19 14.6667 +a 13 14.6667 +a 12 14.6667 +b 18 15.2000 +b 14 15.2000 +c 15 15.1667 +d 16 15.5000 +d 17 15.5000 DROP DATABASE mcs152_db; diff --git a/mysql-test/columnstore/basic/r/mcs153_win_frame_bit_and.result b/mysql-test/columnstore/basic/r/mcs153_win_frame_bit_and.result index 465283246..ca816a590 100644 --- a/mysql-test/columnstore/basic/r/mcs153_win_frame_bit_and.result +++ b/mysql-test/columnstore/basic/r/mcs153_win_frame_bit_and.result @@ -14,7 +14,7 @@ ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_AND' is currently no SELECT a, b, BIT_AND(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) bit_and FROM t1; ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_AND' is currently not supported in Columnstore. SELECT a, b, BIT_AND(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) bit_and FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_AND' is currently not supported in Columnstore. SELECT a, b, BIT_AND(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) bit_and FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_AND' is currently not supported in Columnstore. DROP DATABASE mcs153_db; diff --git a/mysql-test/columnstore/basic/r/mcs154_win_frame_bit_or.result b/mysql-test/columnstore/basic/r/mcs154_win_frame_bit_or.result index 54da5dbf2..bcd7f2ca4 100644 --- a/mysql-test/columnstore/basic/r/mcs154_win_frame_bit_or.result +++ b/mysql-test/columnstore/basic/r/mcs154_win_frame_bit_or.result @@ -14,7 +14,7 @@ ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_OR' is currently not SELECT a, b, BIT_OR(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) bit_or FROM t1; ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_OR' is currently not supported in Columnstore. SELECT a, b, BIT_OR(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) bit_or FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_OR' is currently not supported in Columnstore. SELECT a, b, BIT_OR(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) bit_or FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_OR' is currently not supported in Columnstore. DROP DATABASE mcs154_db; diff --git a/mysql-test/columnstore/basic/r/mcs155_win_frame_bit_xor.result b/mysql-test/columnstore/basic/r/mcs155_win_frame_bit_xor.result index 8e8d8022e..044f21043 100644 --- a/mysql-test/columnstore/basic/r/mcs155_win_frame_bit_xor.result +++ b/mysql-test/columnstore/basic/r/mcs155_win_frame_bit_xor.result @@ -14,7 +14,7 @@ ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_XOR' is currently no SELECT a, b, BIT_XOR(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) BIT_XOR FROM t1; ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_XOR' is currently not supported in Columnstore. SELECT a, b, BIT_XOR(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) BIT_XOR FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_XOR' is currently not supported in Columnstore. SELECT a, b, BIT_XOR(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) BIT_XOR FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +ERROR HY000: Internal error: MCS-9018: Window Function 'BIT_XOR' is currently not supported in Columnstore. DROP DATABASE mcs155_db; diff --git a/mysql-test/columnstore/basic/r/mcs156_win_frame_count.result b/mysql-test/columnstore/basic/r/mcs156_win_frame_count.result index 8074d814d..8a8f619b6 100644 --- a/mysql-test/columnstore/basic/r/mcs156_win_frame_count.result +++ b/mysql-test/columnstore/basic/r/mcs156_win_frame_count.result @@ -224,7 +224,25 @@ d 17 1 b 18 1 a 19 1 SELECT a, b, COUNT(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) count FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b count +NULL NULL 0 +a 12 1 +a 13 1 +b 14 1 +c 15 1 +d 16 1 +d 17 1 +b 18 1 +a 19 1 SELECT a, b, COUNT(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) count FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b count +NULL NULL 0 +a 19 3 +a 13 3 +a 12 3 +b 18 5 +b 14 5 +c 15 6 +d 16 8 +d 17 8 DROP DATABASE mcs156_db; diff --git a/mysql-test/columnstore/basic/r/mcs157_win_frame_lead.result b/mysql-test/columnstore/basic/r/mcs157_win_frame_lead.result index aa51027b0..e660441f7 100644 --- a/mysql-test/columnstore/basic/r/mcs157_win_frame_lead.result +++ b/mysql-test/columnstore/basic/r/mcs157_win_frame_lead.result @@ -226,5 +226,14 @@ a 19 NULL SELECT a, b, LEAD(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) lead_value FROM t1; ERROR HY000: No order list in window specification for 'lead' SELECT a, b, LEAD(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) lead_value FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b lead_value +NULL NULL 19 +a 19 13 +a 13 12 +a 12 18 +b 18 14 +b 14 15 +c 15 16 +d 16 17 +d 17 NULL DROP DATABASE mcs157_db; diff --git a/mysql-test/columnstore/basic/r/mcs158_win_frame_max.result b/mysql-test/columnstore/basic/r/mcs158_win_frame_max.result index 85cd9678e..0b19e596d 100644 --- a/mysql-test/columnstore/basic/r/mcs158_win_frame_max.result +++ b/mysql-test/columnstore/basic/r/mcs158_win_frame_max.result @@ -224,7 +224,25 @@ d 17 17 b 18 18 a 19 19 SELECT a, b, MAX(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) max FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b max + NULL NULL +a 12 12 +a 13 13 +b 14 14 +c 15 15 +d 16 16 +d 17 17 +b 18 18 +a 19 19 SELECT a, b, MAX(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) max FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b max + NULL NULL +a 19 19 +a 13 19 +a 12 19 +b 18 19 +b 14 19 +c 15 19 +d 16 19 +d 17 19 DROP DATABASE mcs158_db; diff --git a/mysql-test/columnstore/basic/r/mcs159_win_frame_min.result b/mysql-test/columnstore/basic/r/mcs159_win_frame_min.result index 7852adde4..70e0eb154 100644 --- a/mysql-test/columnstore/basic/r/mcs159_win_frame_min.result +++ b/mysql-test/columnstore/basic/r/mcs159_win_frame_min.result @@ -224,7 +224,25 @@ d 17 17 b 18 18 a 19 19 SELECT a, b, MIN(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) min FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b min + NULL NULL +a 12 12 +a 13 13 +b 14 14 +c 15 15 +d 16 16 +d 17 17 +b 18 18 +a 19 19 SELECT a, b, MIN(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) min FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b min + NULL NULL +a 19 12 +a 13 12 +a 12 12 +b 18 12 +b 14 12 +c 15 12 +d 16 12 +d 17 12 DROP DATABASE mcs159_db; diff --git a/mysql-test/columnstore/basic/r/mcs161_win_frame_std.result b/mysql-test/columnstore/basic/r/mcs161_win_frame_std.result index 29637fec1..2800df775 100644 --- a/mysql-test/columnstore/basic/r/mcs161_win_frame_std.result +++ b/mysql-test/columnstore/basic/r/mcs161_win_frame_std.result @@ -224,7 +224,25 @@ d 17 0.0000 b 18 0.0000 a 19 0.0000 SELECT a, b, STD(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) std FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b std +NULL NULL NULL +a 12 0.0000 +a 13 0.0000 +b 14 0.0000 +c 15 0.0000 +d 16 0.0000 +d 17 0.0000 +b 18 0.0000 +a 19 0.0000 SELECT a, b, STD(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) std FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b std +NULL NULL NULL +a 19 3.0912 +a 13 3.0912 +a 12 3.0912 +b 18 2.7857 +b 14 2.7857 +c 15 2.5441 +d 16 2.2913 +d 17 2.2913 DROP DATABASE mcs161_db; diff --git a/mysql-test/columnstore/basic/r/mcs162_win_frame_stddev.result b/mysql-test/columnstore/basic/r/mcs162_win_frame_stddev.result index b98dea009..247367499 100644 --- a/mysql-test/columnstore/basic/r/mcs162_win_frame_stddev.result +++ b/mysql-test/columnstore/basic/r/mcs162_win_frame_stddev.result @@ -224,7 +224,25 @@ d 17 0.0000 b 18 0.0000 a 19 0.0000 SELECT a, b, STDDEV(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) stddev FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b stddev +NULL NULL NULL +a 12 0.0000 +a 13 0.0000 +b 14 0.0000 +c 15 0.0000 +d 16 0.0000 +d 17 0.0000 +b 18 0.0000 +a 19 0.0000 SELECT a, b, STDDEV(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) stddev FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b stddev +NULL NULL NULL +a 19 3.0912 +a 13 3.0912 +a 12 3.0912 +b 18 2.7857 +b 14 2.7857 +c 15 2.5441 +d 16 2.2913 +d 17 2.2913 DROP DATABASE mcs162_db; diff --git a/mysql-test/columnstore/basic/r/mcs163_win_frame_stddev_pop.result b/mysql-test/columnstore/basic/r/mcs163_win_frame_stddev_pop.result index cd97a6746..d21fb845a 100644 --- a/mysql-test/columnstore/basic/r/mcs163_win_frame_stddev_pop.result +++ b/mysql-test/columnstore/basic/r/mcs163_win_frame_stddev_pop.result @@ -224,7 +224,25 @@ d 17 0.0000 b 18 0.0000 a 19 0.0000 SELECT a, b, STDDEV_POP(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) STDDEV_POP FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b STDDEV_POP +NULL NULL NULL +a 12 0.0000 +a 13 0.0000 +b 14 0.0000 +c 15 0.0000 +d 16 0.0000 +d 17 0.0000 +b 18 0.0000 +a 19 0.0000 SELECT a, b, STDDEV_POP(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) STDDEV_POP FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b STDDEV_POP +NULL NULL NULL +a 19 3.0912 +a 13 3.0912 +a 12 3.0912 +b 18 2.7857 +b 14 2.7857 +c 15 2.5441 +d 16 2.2913 +d 17 2.2913 DROP DATABASE mcs163_db; diff --git a/mysql-test/columnstore/basic/r/mcs164_win_frame_stddev_samp.result b/mysql-test/columnstore/basic/r/mcs164_win_frame_stddev_samp.result index 30514b254..1aa0ef5c0 100644 --- a/mysql-test/columnstore/basic/r/mcs164_win_frame_stddev_samp.result +++ b/mysql-test/columnstore/basic/r/mcs164_win_frame_stddev_samp.result @@ -224,7 +224,25 @@ d 17 NULL b 18 NULL a 19 NULL SELECT a, b, STDDEV_SAMP(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) STDDEV_SAMP FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b STDDEV_SAMP +NULL NULL NULL +a 12 NULL +a 13 NULL +b 14 NULL +c 15 NULL +d 16 NULL +d 17 NULL +b 18 NULL +a 19 NULL SELECT a, b, STDDEV_SAMP(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) STDDEV_SAMP FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b STDDEV_SAMP +NULL NULL NULL +a 19 3.7859 +a 13 3.7859 +a 12 3.7859 +b 18 3.1145 +b 14 3.1145 +c 15 2.7869 +d 16 2.4495 +d 17 2.4495 DROP DATABASE mcs164_db; diff --git a/mysql-test/columnstore/basic/r/mcs165_win_frame_sum.result b/mysql-test/columnstore/basic/r/mcs165_win_frame_sum.result index 1e7581f20..d70093448 100644 --- a/mysql-test/columnstore/basic/r/mcs165_win_frame_sum.result +++ b/mysql-test/columnstore/basic/r/mcs165_win_frame_sum.result @@ -224,7 +224,25 @@ d 17 17 b 18 18 a 19 19 SELECT a, b, SUM(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) sum FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b sum +NULL NULL NULL +a 12 12 +a 13 13 +b 14 14 +c 15 15 +d 16 16 +d 17 17 +b 18 18 +a 19 19 SELECT a, b, SUM(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) sum FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b sum +NULL NULL NULL +a 19 44 +a 13 44 +a 12 44 +b 18 76 +b 14 76 +c 15 91 +d 16 124 +d 17 124 DROP DATABASE mcs165_db; diff --git a/mysql-test/columnstore/basic/r/mcs166_win_frame_var_pop.result b/mysql-test/columnstore/basic/r/mcs166_win_frame_var_pop.result index 1a0e83b10..f0a255d59 100644 --- a/mysql-test/columnstore/basic/r/mcs166_win_frame_var_pop.result +++ b/mysql-test/columnstore/basic/r/mcs166_win_frame_var_pop.result @@ -224,7 +224,25 @@ d 17 0.0000 b 18 0.0000 a 19 0.0000 SELECT a, b, VAR_POP(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VAR_POP FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b VAR_POP +NULL NULL NULL +a 12 0.0000 +a 13 0.0000 +b 14 0.0000 +c 15 0.0000 +d 16 0.0000 +d 17 0.0000 +b 18 0.0000 +a 19 0.0000 SELECT a, b, VAR_POP(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VAR_POP FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b VAR_POP +NULL NULL NULL +a 19 9.5556 +a 13 9.5556 +a 12 9.5556 +b 18 7.7600 +b 14 7.7600 +c 15 6.4722 +d 16 5.2500 +d 17 5.2500 DROP DATABASE mcs166_db; diff --git a/mysql-test/columnstore/basic/r/mcs167_win_frame_var_samp.result b/mysql-test/columnstore/basic/r/mcs167_win_frame_var_samp.result index 343220918..3b13aa56a 100644 --- a/mysql-test/columnstore/basic/r/mcs167_win_frame_var_samp.result +++ b/mysql-test/columnstore/basic/r/mcs167_win_frame_var_samp.result @@ -224,7 +224,25 @@ d 17 NULL b 18 NULL a 19 NULL SELECT a, b, VAR_SAMP(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VAR_SAMP FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b VAR_SAMP +NULL NULL NULL +a 12 NULL +a 13 NULL +b 14 NULL +c 15 NULL +d 16 NULL +d 17 NULL +b 18 NULL +a 19 NULL SELECT a, b, VAR_SAMP(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VAR_SAMP FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b VAR_SAMP +NULL NULL NULL +a 19 14.3333 +a 13 14.3333 +a 12 14.3333 +b 18 9.7000 +b 14 9.7000 +c 15 7.7667 +d 16 6.0000 +d 17 6.0000 DROP DATABASE mcs167_db; diff --git a/mysql-test/columnstore/basic/r/mcs168_win_frame_variance.result b/mysql-test/columnstore/basic/r/mcs168_win_frame_variance.result index 36ec5142f..e623b0c23 100644 --- a/mysql-test/columnstore/basic/r/mcs168_win_frame_variance.result +++ b/mysql-test/columnstore/basic/r/mcs168_win_frame_variance.result @@ -224,7 +224,25 @@ d 17 0.0000 b 18 0.0000 a 19 0.0000 SELECT a, b, VARIANCE(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VARIANCE FROM t1; -ERROR HY000: RANGE-type frame requires ORDER BY clause with single sort key +a b VARIANCE +NULL NULL NULL +a 12 0.0000 +a 13 0.0000 +b 14 0.0000 +c 15 0.0000 +d 16 0.0000 +d 17 0.0000 +b 18 0.0000 +a 19 0.0000 SELECT a, b, VARIANCE(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VARIANCE FROM t1; -ERROR HY000: Numeric datatype is required for RANGE-type frame +a b VARIANCE +NULL NULL NULL +a 19 9.5556 +a 13 9.5556 +a 12 9.5556 +b 18 7.7600 +b 14 7.7600 +c 15 6.4722 +d 16 5.2500 +d 17 5.2500 DROP DATABASE mcs168_db; diff --git a/mysql-test/columnstore/basic/r/mcs254_hex_function.result b/mysql-test/columnstore/basic/r/mcs254_hex_function.result index 6cfd64440..acec00252 100644 --- a/mysql-test/columnstore/basic/r/mcs254_hex_function.result +++ b/mysql-test/columnstore/basic/r/mcs254_hex_function.result @@ -18,7 +18,7 @@ HEX(10) A SELECT HEX(-1) FROM t1 LIMIT 1; HEX(-1) -FFFF +FFFFFFFFFFFFFFFF SELECT HEX('a') FROM t1 LIMIT 1; HEX('a') 61 diff --git a/mysql-test/columnstore/basic/t/mcs127_window_function_last_value.test b/mysql-test/columnstore/basic/t/mcs127_window_function_last_value.test index e9a204170..cafca5052 100644 --- a/mysql-test/columnstore/basic/t/mcs127_window_function_last_value.test +++ b/mysql-test/columnstore/basic/t/mcs127_window_function_last_value.test @@ -27,10 +27,7 @@ SELECT a, b, LAST_VALUE(b) OVER(ORDER BY b RANGE BETWEEN UNBOUNDED PRECEDING AND SELECT a, b, LAST_VALUE(b) OVER(PARTITION BY b ORDER BY b DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value FROM t1; SELECT a, b, LAST_VALUE(a) OVER(PARTITION BY b ORDER BY b DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value FROM t1; SELECT a, b, LAST_VALUE(b) OVER(PARTITION BY a ORDER BY b DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value FROM t1; - ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, LAST_VALUE(b) OVER(PARTITION BY a ORDER BY a DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, LAST_VALUE(b) OVER(PARTITION BY b ORDER BY a DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs129_window_function_nth_value.test b/mysql-test/columnstore/basic/t/mcs129_window_function_nth_value.test index 137cb27ef..53d7c4514 100644 --- a/mysql-test/columnstore/basic/t/mcs129_window_function_nth_value.test +++ b/mysql-test/columnstore/basic/t/mcs129_window_function_nth_value.test @@ -33,10 +33,7 @@ SELECT a, b, NTH_VALUE(a, 2) OVER(PARTITION BY a ORDER BY b RANGE BETWEEN UNBOUN SELECT a, b, NTH_VALUE(b, 2) OVER(PARTITION BY a ORDER BY b RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_value FROM t1; SELECT a, b, NTH_VALUE(a, 2) OVER(PARTITION BY b ORDER BY b RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_value FROM t1; SELECT a, b, NTH_VALUE(b, 2) OVER(PARTITION BY b ORDER BY b RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_value FROM t1; - ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, NTH_VALUE(b, 2) OVER(PARTITION BY b ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_value FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, NTH_VALUE(a, 2) OVER(PARTITION BY a ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_value FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs152_win_frame_avg.test b/mysql-test/columnstore/basic/t/mcs152_win_frame_avg.test index f18571572..f6f062dfa 100644 --- a/mysql-test/columnstore/basic/t/mcs152_win_frame_avg.test +++ b/mysql-test/columnstore/basic/t/mcs152_win_frame_avg.test @@ -36,10 +36,7 @@ SELECT a, b, AVG(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND SELECT a, b, AVG(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) avg FROM t1; SELECT a, b, AVG(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) avg FROM t1; SELECT a, b, AVG(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) avg FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, AVG(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) avg FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, AVG(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) avg FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs153_win_frame_bit_and.test b/mysql-test/columnstore/basic/t/mcs153_win_frame_bit_and.test index f9a270202..e94651b40 100644 --- a/mysql-test/columnstore/basic/t/mcs153_win_frame_bit_and.test +++ b/mysql-test/columnstore/basic/t/mcs153_win_frame_bit_and.test @@ -24,10 +24,9 @@ SELECT a, b, BIT_AND(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND CURRENT ROW SELECT a, b, BIT_AND(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) bit_and FROM t1; --error 1815 SELECT a, b, BIT_AND(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) bit_and FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY +--error 1815 SELECT a, b, BIT_AND(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) bit_and FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME +--error 1815 SELECT a, b, BIT_AND(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) bit_and FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs154_win_frame_bit_or.test b/mysql-test/columnstore/basic/t/mcs154_win_frame_bit_or.test index e510c7525..eb4962f39 100644 --- a/mysql-test/columnstore/basic/t/mcs154_win_frame_bit_or.test +++ b/mysql-test/columnstore/basic/t/mcs154_win_frame_bit_or.test @@ -24,10 +24,9 @@ SELECT a, b, BIT_OR(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) SELECT a, b, BIT_OR(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) bit_or FROM t1; --error 1815 SELECT a, b, BIT_OR(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) bit_or FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY +--error 1815 SELECT a, b, BIT_OR(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) bit_or FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME +--error 1815 SELECT a, b, BIT_OR(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) bit_or FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs155_win_frame_bit_xor.test b/mysql-test/columnstore/basic/t/mcs155_win_frame_bit_xor.test index 7fe24a6b0..fb6797dc2 100644 --- a/mysql-test/columnstore/basic/t/mcs155_win_frame_bit_xor.test +++ b/mysql-test/columnstore/basic/t/mcs155_win_frame_bit_xor.test @@ -25,9 +25,9 @@ SELECT a, b, BIT_XOR(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING --error 1815 SELECT a, b, BIT_XOR(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) BIT_XOR FROM t1; ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY +--error 1815 SELECT a, b, BIT_XOR(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) BIT_XOR FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME +--error 1815 SELECT a, b, BIT_XOR(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) BIT_XOR FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs156_win_frame_count.test b/mysql-test/columnstore/basic/t/mcs156_win_frame_count.test index 740cc6169..e713d377b 100644 --- a/mysql-test/columnstore/basic/t/mcs156_win_frame_count.test +++ b/mysql-test/columnstore/basic/t/mcs156_win_frame_count.test @@ -19,13 +19,13 @@ SELECT a, b, COUNT(b) OVER(ORDER BY b RANGE BETWEEN CURRENT ROW AND UNBOUNDED FO SELECT a, b, COUNT(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) count FROM t1; SELECT a, b, COUNT(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) count FROM t1; SELECT a, b, COUNT(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) count FROM t1; - + SELECT a, b, COUNT(a) OVER(ORDER BY b DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) count FROM t1; SELECT a, b, COUNT(a) OVER(ORDER BY b DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) count FROM t1; SELECT a, b, COUNT(a) OVER(ORDER BY b DESC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) count FROM t1; SELECT a, b, COUNT(a) OVER(ORDER BY b DESC ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) count FROM t1; SELECT a, b, COUNT(a) OVER(ORDER BY b DESC ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) count FROM t1; - + SELECT a, b, COUNT(a) OVER(PARTITION BY a ORDER BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) count FROM t1; SELECT a, b, COUNT(a) OVER(PARTITION BY b ORDER BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) count FROM t1; SELECT a, b, COUNT(a) OVER(PARTITION BY a ORDER BY b RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) count FROM t1; @@ -37,9 +37,7 @@ SELECT a, b, COUNT(a) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AN SELECT a, b, COUNT(a) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) count FROM t1; SELECT a, b, COUNT(a) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) count FROM t1; ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, COUNT(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) count FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, COUNT(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) count FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs157_win_frame_lead.test b/mysql-test/columnstore/basic/t/mcs157_win_frame_lead.test index c956467f2..86a3afdf5 100644 --- a/mysql-test/columnstore/basic/t/mcs157_win_frame_lead.test +++ b/mysql-test/columnstore/basic/t/mcs157_win_frame_lead.test @@ -39,7 +39,6 @@ SELECT a, b, LEAD(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND --error ER_NO_ORDER_LIST_IN_WINDOW_SPEC SELECT a, b, LEAD(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) lead_value FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, LEAD(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) lead_value FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs158_win_frame_max.test b/mysql-test/columnstore/basic/t/mcs158_win_frame_max.test index e0d105ade..47a122df2 100644 --- a/mysql-test/columnstore/basic/t/mcs158_win_frame_max.test +++ b/mysql-test/columnstore/basic/t/mcs158_win_frame_max.test @@ -36,10 +36,7 @@ SELECT a, b, MAX(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND SELECT a, b, MAX(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) max FROM t1; SELECT a, b, MAX(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) max FROM t1; SELECT a, b, MAX(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) max FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, MAX(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) max FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, MAX(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) max FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs159_win_frame_min.test b/mysql-test/columnstore/basic/t/mcs159_win_frame_min.test index 46dd81f5a..48f0b2d54 100644 --- a/mysql-test/columnstore/basic/t/mcs159_win_frame_min.test +++ b/mysql-test/columnstore/basic/t/mcs159_win_frame_min.test @@ -36,10 +36,7 @@ SELECT a, b, MIN(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND SELECT a, b, MIN(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) min FROM t1; SELECT a, b, MIN(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) min FROM t1; SELECT a, b, MIN(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) min FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, MIN(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) min FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, MIN(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) min FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs161_win_frame_std.test b/mysql-test/columnstore/basic/t/mcs161_win_frame_std.test index f3dfb1263..a99e4f744 100644 --- a/mysql-test/columnstore/basic/t/mcs161_win_frame_std.test +++ b/mysql-test/columnstore/basic/t/mcs161_win_frame_std.test @@ -36,10 +36,7 @@ SELECT a, b, STD(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND SELECT a, b, STD(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) std FROM t1; SELECT a, b, STD(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) std FROM t1; SELECT a, b, STD(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) std FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, STD(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) std FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, STD(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) std FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs162_win_frame_stddev.test b/mysql-test/columnstore/basic/t/mcs162_win_frame_stddev.test index 31da001ee..9d5bcd7e5 100644 --- a/mysql-test/columnstore/basic/t/mcs162_win_frame_stddev.test +++ b/mysql-test/columnstore/basic/t/mcs162_win_frame_stddev.test @@ -36,10 +36,7 @@ SELECT a, b, STDDEV(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING A SELECT a, b, STDDEV(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) stddev FROM t1; SELECT a, b, STDDEV(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) stddev FROM t1; SELECT a, b, STDDEV(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) stddev FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, STDDEV(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) stddev FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, STDDEV(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) stddev FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs163_win_frame_stddev_pop.test b/mysql-test/columnstore/basic/t/mcs163_win_frame_stddev_pop.test index adeec7f75..04c77c25b 100644 --- a/mysql-test/columnstore/basic/t/mcs163_win_frame_stddev_pop.test +++ b/mysql-test/columnstore/basic/t/mcs163_win_frame_stddev_pop.test @@ -36,10 +36,7 @@ SELECT a, b, STDDEV_POP(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDI SELECT a, b, STDDEV_POP(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) STDDEV_POP FROM t1; SELECT a, b, STDDEV_POP(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) STDDEV_POP FROM t1; SELECT a, b, STDDEV_POP(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) STDDEV_POP FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, STDDEV_POP(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) STDDEV_POP FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, STDDEV_POP(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) STDDEV_POP FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs164_win_frame_stddev_samp.test b/mysql-test/columnstore/basic/t/mcs164_win_frame_stddev_samp.test index 48da6aa6a..5f28e3d21 100644 --- a/mysql-test/columnstore/basic/t/mcs164_win_frame_stddev_samp.test +++ b/mysql-test/columnstore/basic/t/mcs164_win_frame_stddev_samp.test @@ -36,10 +36,7 @@ SELECT a, b, STDDEV_SAMP(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECED SELECT a, b, STDDEV_SAMP(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) STDDEV_SAMP FROM t1; SELECT a, b, STDDEV_SAMP(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) STDDEV_SAMP FROM t1; SELECT a, b, STDDEV_SAMP(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) STDDEV_SAMP FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, STDDEV_SAMP(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) STDDEV_SAMP FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, STDDEV_SAMP(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) STDDEV_SAMP FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs165_win_frame_sum.test b/mysql-test/columnstore/basic/t/mcs165_win_frame_sum.test index f26e7dbc3..ef0e04187 100644 --- a/mysql-test/columnstore/basic/t/mcs165_win_frame_sum.test +++ b/mysql-test/columnstore/basic/t/mcs165_win_frame_sum.test @@ -19,7 +19,7 @@ SELECT a, b, SUM(b) OVER(ORDER BY b RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLL SELECT a, b, SUM(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) sum FROM t1; SELECT a, b, SUM(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) sum FROM t1; SELECT a, b, SUM(b) OVER(ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) sum FROM t1; - + SELECT a, b, SUM(b) OVER(ORDER BY b DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) sum FROM t1; SELECT a, b, SUM(b) OVER(ORDER BY b DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) sum FROM t1; SELECT a, b, SUM(b) OVER(ORDER BY b DESC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) sum FROM t1; @@ -36,10 +36,7 @@ SELECT a, b, SUM(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND SELECT a, b, SUM(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) sum FROM t1; SELECT a, b, SUM(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) sum FROM t1; SELECT a, b, SUM(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) sum FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, SUM(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) sum FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, SUM(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) sum FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs166_win_frame_var_pop.test b/mysql-test/columnstore/basic/t/mcs166_win_frame_var_pop.test index 5e52031b1..7a1aa4595 100644 --- a/mysql-test/columnstore/basic/t/mcs166_win_frame_var_pop.test +++ b/mysql-test/columnstore/basic/t/mcs166_win_frame_var_pop.test @@ -36,10 +36,7 @@ SELECT a, b, VAR_POP(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING SELECT a, b, VAR_POP(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) VAR_POP FROM t1; SELECT a, b, VAR_POP(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) VAR_POP FROM t1; SELECT a, b, VAR_POP(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) VAR_POP FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, VAR_POP(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VAR_POP FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, VAR_POP(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VAR_POP FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs167_win_frame_var_samp.test b/mysql-test/columnstore/basic/t/mcs167_win_frame_var_samp.test index 07eee777f..c4875b402 100644 --- a/mysql-test/columnstore/basic/t/mcs167_win_frame_var_samp.test +++ b/mysql-test/columnstore/basic/t/mcs167_win_frame_var_samp.test @@ -36,10 +36,7 @@ SELECT a, b, VAR_SAMP(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING SELECT a, b, VAR_SAMP(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) VAR_SAMP FROM t1; SELECT a, b, VAR_SAMP(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) VAR_SAMP FROM t1; SELECT a, b, VAR_SAMP(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) VAR_SAMP FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, VAR_SAMP(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VAR_SAMP FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, VAR_SAMP(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VAR_SAMP FROM t1; # Clean UP diff --git a/mysql-test/columnstore/basic/t/mcs168_win_frame_variance.test b/mysql-test/columnstore/basic/t/mcs168_win_frame_variance.test index bb05445a7..5b06de80b 100644 --- a/mysql-test/columnstore/basic/t/mcs168_win_frame_variance.test +++ b/mysql-test/columnstore/basic/t/mcs168_win_frame_variance.test @@ -36,10 +36,7 @@ SELECT a, b, VARIANCE(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING SELECT a, b, VARIANCE(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) VARIANCE FROM t1; SELECT a, b, VARIANCE(b) OVER(PARTITION BY a ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) VARIANCE FROM t1; SELECT a, b, VARIANCE(b) OVER(PARTITION BY b ORDER BY b ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) VARIANCE FROM t1; - ---error ER_RANGE_FRAME_NEEDS_SIMPLE_ORDERBY SELECT a, b, VARIANCE(b) OVER(PARTITION BY b RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VARIANCE FROM t1; ---error ER_WRONG_TYPE_FOR_RANGE_FRAME SELECT a, b, VARIANCE(b) OVER(ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) VARIANCE FROM t1; # Clean UP From 31aaa7df5019e06c2bb39077e99f9982b83d4b81 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Tue, 10 Sep 2024 17:44:25 +0300 Subject: [PATCH 07/65] fix(logging): MCOL-5789 Add logs on failure. (#3308) --- versioning/BRM/dbrm.cpp | 6 +++++- versioning/BRM/masterdbrmnode.cpp | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index 27e36067a..82d3eeba4 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -3198,7 +3198,11 @@ void DBRM::rolledback(TxnID& txnid) if (tmp != ERR_OK) { if (getSystemReady() != 0) - log("DBRM: error: SessionManager::rolledback() failed (valid error code)", logging::LOG_TYPE_ERROR); + { + std::stringstream errorStream; + errorStream << "DBRM: error: SessionManager::rolledback() failed (error code " << tmp << ")"; + log(errorStream.str(), logging::LOG_TYPE_ERROR); + } } } diff --git a/versioning/BRM/masterdbrmnode.cpp b/versioning/BRM/masterdbrmnode.cpp index 418a82c76..08b24a4ae 100644 --- a/versioning/BRM/masterdbrmnode.cpp +++ b/versioning/BRM/masterdbrmnode.cpp @@ -1633,8 +1633,12 @@ void MasterDBRMNode::doRolledBack(ByteStream& msg, ThreadParams* p) #endif sm.rolledback(txnid); } - catch (exception&) + catch (exception& ex) { + ostringstream errStream; + errStream << "doRolledBack() failed: " << ex.what(); + log(errStream.str()); + reply << (uint8_t)ERR_FAILURE; try From 4f49c7870afcbc04eb20b06297eb24eaa82c8d2d Mon Sep 17 00:00:00 2001 From: Alexey Antipovsky Date: Tue, 10 Sep 2024 16:50:59 +0200 Subject: [PATCH 08/65] fix(SM): check for duplicates (#3314) Co-authored-by: Roman Nozdrin --- storage-manager/src/PrefixCache.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/storage-manager/src/PrefixCache.cpp b/storage-manager/src/PrefixCache.cpp index cb1faaee8..220346aed 100644 --- a/storage-manager/src/PrefixCache.cpp +++ b/storage-manager/src/PrefixCache.cpp @@ -137,15 +137,24 @@ void PrefixCache::populate() const bf::path& p = dir->path(); if (bf::is_regular_file(p)) { - lru.push_back(p.filename().string()); + auto fileName = p.filename().string(); + if (m_lru.find(fileName) != m_lru.end()) + { + logger->log(LOG_WARNING, "Cache: found a duplicate in the cache '%s'", p.string().c_str()); + ++dir; + continue; + } + lru.push_back(fileName); auto last = lru.end(); m_lru.insert(--last); currentCacheSize += bf::file_size(*dir); newObjects.push_back(p.filename().string()); } else if (p != cachePrefix / downloader->getTmpPath()) + { logger->log(LOG_WARNING, "Cache: found something in the cache that does not belong '%s'", p.string().c_str()); + } ++dir; } sync->newObjects(firstDir, newObjects); @@ -471,7 +480,6 @@ void PrefixCache::_makeSpace(size_t size) if (!bf::exists(cachePrefix / *it)) logger->log(LOG_WARNING, "PrefixCache::makeSpace(): doesn't exist, %s/%s", cachePrefix.string().c_str(), ((string)(*it)).c_str()); - assert(bf::exists(cachePrefix / *it)); /* tell Synchronizer that this key will be evicted delete the file From 3489b2c542033f77b4f86218719e13b0c1301791 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Tue, 10 Sep 2024 19:10:42 +0300 Subject: [PATCH 09/65] fix(logging): Add setddldebuglevel command (#3312) --- dbcon/ddlpackage/CMakeLists.txt | 1 + dbcon/ddlpackage/ddlpkg.h | 29 +- dbcon/ddlpackage/debugstatement.cpp | 40 +++ dbcon/ddlpackage/serialize.cpp | 15 + .../ddlpackageproc/droppartitionprocessor.cpp | 25 +- dbcon/mysql/ha_mcs_client_udfs.cpp | 273 +++++++++++------- dbcon/mysql/ha_mcs_partition.cpp | 64 ++-- dbcon/mysql/install_mcs_mysql.sh.in | 1 + ddlproc/ddlprocessor.cpp | 34 ++- ddlproc/ddlprocessor.h | 1 + 10 files changed, 345 insertions(+), 138 deletions(-) create mode 100644 dbcon/ddlpackage/debugstatement.cpp diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 1c986eced..7c639bbf3 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -32,6 +32,7 @@ ADD_LIBRARY(ddlpackage SHARED markpartition.cpp restorepartition.cpp droppartition.cpp + debugstatement.cpp ${BISON_ddl_gram_OUTPUTS} ${FLEX_ddl_scan_OUTPUTS} ) diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 59238a899..e1564c4cb 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -332,7 +332,8 @@ enum DDL_SERIAL_TYPE DDL_TRUNC_TABLE_STATEMENT, DDL_MARK_PARTITION_STATEMENT, DDL_RESTORE_PARTITION_STATEMENT, - DDL_DROP_PARTITION_STATEMENT + DDL_DROP_PARTITION_STATEMENT, + DDL_DEBUG_STATEMENT }; /** @brief An abstract base for TableDef, ColumnDef, ... @@ -1319,8 +1320,10 @@ struct AlterTableStatement : public SqlStatement QualifiedName* fTableName; AlterTableActionList fActions; + private: long fTimeZone; + public: }; @@ -1441,6 +1444,30 @@ struct DropTableStatement : public SqlStatement bool fCascade; }; +/** @brief DebugStatement + */ +struct DebugDDLStatement : public SqlStatement +{ + /** @brief Deserialize from ByteStream */ + EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + + /** @brief Serialize to ByteStream */ + EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + + DebugDDLStatement(uint32_t debugLevel); + + EXPORT DebugDDLStatement(); + + /** @brief Dump to stdout. */ + EXPORT std::ostream& put(std::ostream& os) const; + + virtual ~DebugDDLStatement() + { + } + + uint32_t fDebugLevel; +}; + /** @brief TruncTableStatement represents the drop table operation */ struct TruncTableStatement : public SqlStatement diff --git a/dbcon/ddlpackage/debugstatement.cpp b/dbcon/ddlpackage/debugstatement.cpp new file mode 100644 index 000000000..36a241f67 --- /dev/null +++ b/dbcon/ddlpackage/debugstatement.cpp @@ -0,0 +1,40 @@ +/* Copyright (C) 2024 MariaDB Corporation + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#define DDLPKG_DLLEXPORT +#include "ddlpkg.h" +#undef DDLPKG_DLLEXPORT + +using namespace std; + +namespace ddlpackage +{ +DebugDDLStatement::DebugDDLStatement(uint32_t debugLevel) +: fDebugLevel(debugLevel) +{ +} + +DebugDDLStatement::DebugDDLStatement() +: fDebugLevel(0) +{ +} + +ostream& DebugDDLStatement::put(ostream& os) const +{ + os << "DDL debug level: " << fDebugLevel << endl; + return os; +} +} // namespace ddlpackage diff --git a/dbcon/ddlpackage/serialize.cpp b/dbcon/ddlpackage/serialize.cpp index 887eaae2a..6dff88362 100644 --- a/dbcon/ddlpackage/serialize.cpp +++ b/dbcon/ddlpackage/serialize.cpp @@ -429,6 +429,21 @@ int DropTableStatement::serialize(ByteStream& bytestream) return ret; } +/** @brief Construct from Bytestream */ +int DebugDDLStatement::unserialize(ByteStream& bytestream) +{ + bytestream >> fDebugLevel; + return 1; +} + +/** @brief Serialize to ByteStream */ +int DebugDDLStatement::serialize(ByteStream& bytestream) +{ + bytestream << (quadbyte)DDL_DEBUG_STATEMENT; + bytestream << fDebugLevel; + return 1; +} + /////////////////////////////////////// /// TruncTableStatement Serialization /////////////////////////////////////// diff --git a/dbcon/ddlpackageproc/droppartitionprocessor.cpp b/dbcon/ddlpackageproc/droppartitionprocessor.cpp index f01fa1ade..ed13c5708 100644 --- a/dbcon/ddlpackageproc/droppartitionprocessor.cpp +++ b/dbcon/ddlpackageproc/droppartitionprocessor.cpp @@ -122,6 +122,7 @@ DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackageInternal auto stmt = formatStatementString(dropPartitionStmt->fSql, dropPartitionStmt->fTableName->fSchema, dropPartitionStmt->fTableName->fName, dropPartitionStmt->fPartitions); SQLLogger logger(stmt, fDDLLoggingId, sessionID, txnID.id); + ostringstream debugLog; try { @@ -226,6 +227,8 @@ DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackageInternal return result; } } + debugLog << "DropPartitionProcessor: got table lock (id) " << uniqueID << " for table (OID)" + << roPair.objnum << endl; // 1. Get the OIDs for the columns // 2. Get the OIDs for the dictionaries @@ -241,9 +244,18 @@ DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackageInternal tableColRidList = systemCatalogPtr->columnRIDs(userTableName); tableAuxColOid = systemCatalogPtr->tableAUXColumnOID(userTableName); - dictOIDList = systemCatalogPtr->dictOIDs(userTableName); + debugLog << "DropPartitionProcessor:" << endl; + debugLog << "column RIDS: "; + for (const auto& rid : tableColRidList) + debugLog << rid.objnum << " "; + debugLog << endl; + debugLog << "dict OIDS: "; + for (const auto& dictOid : dictOIDList) + debugLog << dictOid.dictOID << " "; + debugLog << endl; + // Save qualified tablename, all column, dictionary OIDs, and transaction ID into a file in ASCII format for (unsigned i = 0; i < tableColRidList.size(); i++) { @@ -271,6 +283,12 @@ DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackageInternal { throw std::runtime_error(emsg); } + debugLog << "DropPartitionProcessor: marked partitions for deletion:" << endl; + for (const auto& partition : dropPartitionStmt->fPartitions) + debugLog << "dbroot: " << partition.dbroot << " physical parition: " << partition.pp + << " segment: " << partition.seg << endl; + VERBOSE_INFO(debugLog.str()); + debugLog.clear(); set markedPartitions; set outOfServicePartitions; @@ -298,14 +316,16 @@ DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackageInternal // Save the oids to a file createWritePartitionLogFile(roPair.objnum, markedPartitions, oidList, uniqueId); - VERBOSE_INFO("Removing files"); + VERBOSE_INFO("Removing parition files"); removePartitionFiles(oidList, markedPartitions, uniqueId); // Flush PrimProc cache for those lbids + VERBOSE_INFO("Flushing paritions"); rc = cacheutils::flushPartition(oidList, markedPartitions); // Remove the partition from extent map emsg.clear(); rc = fDbrm->deletePartition(oidList, dropPartitionStmt->fPartitions, emsg); + VERBOSE_INFO("DropPartitionProcessor: partitions removed"); if (rc != 0) throw std::runtime_error(emsg); @@ -390,6 +410,7 @@ DropPartitionProcessor::DDLResult DropPartitionProcessor::processPackageInternal } fSessionManager.committed(txnID); + VERBOSE_INFO("DropPartitionProcessor:: commited"); return result; } diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index a8cd7c02e..e9085e63d 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -38,6 +38,9 @@ using namespace oam; #include "errorids.h" using namespace logging; +#include "ddlpkg.h" +#include "calpontsystemcatalog.h" + // #include "resourcemanager.h" #include "columnstoreversion.h" @@ -111,8 +114,8 @@ extern "C" return 0; } - const char* mcssetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* mcssetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { char parameter[MAXSTRINGLENGTH]; char valuestr[MAXSTRINGLENGTH]; @@ -180,29 +183,27 @@ extern "C" return result; } - - my_bool mcssetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcssetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return setparms_init(initid, args, message, "MCSSETPARMS"); } - void mcssetparms_deinit(UDF_INIT* initid) + void mcssetparms_deinit(UDF_INIT* initid) { } - - const char* calsetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calsetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { return mcssetparms(initid, args, result, length, is_null, error); } - my_bool calsetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calsetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return setparms_init(initid, args, message, "CALSETPARMS"); } - void calsetparms_deinit(UDF_INIT* initid) + void calsetparms_deinit(UDF_INIT* initid) { } @@ -220,8 +221,8 @@ extern "C" return 0; } - const char* mcsgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* mcsgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) { @@ -247,28 +248,27 @@ extern "C" return result; } - - my_bool mcsgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcsgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return getstats_init(initid, args, message, "MCSGETSTATS"); } - void mcsgetstats_deinit(UDF_INIT* initid) + void mcsgetstats_deinit(UDF_INIT* initid) { } - const char* calgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { return mcsgetstats(initid, args, result, length, is_null, error); } - my_bool calgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return getstats_init(initid, args, message, "CALGETSTATS"); } - void calgetstats_deinit(UDF_INIT* initid) + void calgetstats_deinit(UDF_INIT* initid) { } @@ -283,7 +283,7 @@ extern "C" return 0; } - long long mcssettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long mcssettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) { @@ -301,31 +301,31 @@ extern "C" return oldTrace; } - my_bool mcssettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcssettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return settrace_init(initid, args, message, "MCSSETTRACE"); } - void mcssettrace_deinit(UDF_INIT* initid) + void mcssettrace_deinit(UDF_INIT* initid) { } - long long calsettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long calsettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { return mcssettrace(initid, args, is_null, error); } - my_bool calsettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calsettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return settrace_init(initid, args, message, "CALSETTRACE"); } - void calsettrace_deinit(UDF_INIT* initid) + void calsettrace_deinit(UDF_INIT* initid) { } - // Return 1 if system is ready for reads or 0 if not. - long long mcssystemready(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + // Return 1 if system is ready for reads or 0 if not. + long long mcssystemready(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { long long rtn = 0; Oam oam; @@ -346,17 +346,17 @@ extern "C" return rtn; } - my_bool mcssystemready_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcssystemready_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return 0; } - void mcssystemready_deinit(UDF_INIT* initid) + void mcssystemready_deinit(UDF_INIT* initid) { } - // Return non-zero if system is read only; 0 if writeable - long long mcssystemreadonly(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + // Return non-zero if system is read only; 0 if writeable + long long mcssystemreadonly(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { long long rtn = 0; DBRM dbrm(true); @@ -382,17 +382,17 @@ extern "C" return rtn; } - my_bool mcssystemreadonly_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcssystemreadonly_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return 0; } - void mcssystemreadonly_deinit(UDF_INIT* initid) + void mcssystemreadonly_deinit(UDF_INIT* initid) { } - // Return non-zero if this is the primary UM; 0 if not primary - long long mcssystemprimary(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + // Return non-zero if this is the primary UM; 0 if not primary + long long mcssystemprimary(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { long long rtn = 0; Oam oam; @@ -424,12 +424,12 @@ extern "C" return rtn; } - my_bool mcssystemprimary_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcssystemprimary_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return 0; } - void mcssystemprimary_deinit(UDF_INIT* initid) + void mcssystemprimary_deinit(UDF_INIT* initid) { } @@ -462,13 +462,13 @@ extern "C" return 0; } - my_bool mcsviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcsviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return viewtablelock_init(initid, args, message, "MCSVIEWTABLELOCK"); } - const char* mcsviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* mcsviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { THD* thd = current_thd; @@ -522,22 +522,22 @@ extern "C" return result; } - void mcsviewtablelock_deinit(UDF_INIT* initid) + void mcsviewtablelock_deinit(UDF_INIT* initid) { } - my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return viewtablelock_init(initid, args, message, "CALVIEWTABLELOCK"); } - const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { return mcsviewtablelock(initid, args, result, length, is_null, error); } - void calviewtablelock_deinit(UDF_INIT* initid) + void calviewtablelock_deinit(UDF_INIT* initid) { } @@ -555,14 +555,88 @@ extern "C" return 0; } + my_bool mcs_set_ddldebug_level_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT) || + *reinterpret_cast(args->args[0]) > 3) + { + sprintf(message, "%s() requires one integer `debug level` ", funcname); + return 1; + } - my_bool mcscleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + initid->maybe_null = 0; + initid->max_length = 5; + + return 0; + } + + const char* mcs_set_ddldebug_level(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length) + { + uint32_t level = *reinterpret_cast(args->args[0]); + std::unique_ptr stmt = + std::make_unique(level); + stmt->fSessionID = execplan::CalpontSystemCatalog::idb_tid2sid(current_thd->thread_id); + MessageQueueClient mq("DDLProc"); + + ByteStream bytestream; + bytestream << (uint32_t)stmt->fSessionID; + stmt->serialize(bytestream); + ByteStream::byte b = 0; + THD* thd = current_thd; + string emsg; + mq.write(bytestream); + + try + { + bytestream = mq.read(); + + if (bytestream.length() == 0) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); + std::memcpy(result, "Error", 5); + *length = 5; + return result; + } + else + { + bytestream >> b; + bytestream >> emsg; + } + } + catch (runtime_error&) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Lost connection to DDLProc"); + std::memcpy(result, "Error", 5); + *length = 5; + return result; + } + catch (...) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_INTERNAL_ERROR, "Unknown error caught"); + std::memcpy(result, "Error", 5); + *length = 5; + return result; + } + + *length = 2; + std::memcpy(result, "Ok", 2); + return result; + } + + void mcs_set_ddldebug_level_deinit(UDF_INIT* initid) + { + } + + my_bool mcscleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return cleartablelock_init(initid, args, message, "MCSCLEARTABLELOCK"); } - const char* mcscleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* mcscleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) { @@ -588,22 +662,22 @@ extern "C" return result; } - void mcscleartablelock_deinit(UDF_INIT* initid) + void mcscleartablelock_deinit(UDF_INIT* initid) { } - my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return cleartablelock_init(initid, args, message, "CALCLEARTABLELOCK"); } - const char* calcleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calcleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { return mcscleartablelock(initid, args, result, length, is_null, error); } - void calcleartablelock_deinit(UDF_INIT* initid) + void calcleartablelock_deinit(UDF_INIT* initid) { } @@ -636,13 +710,12 @@ extern "C" return 0; } - - my_bool mcslastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcslastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return lastinsertid_init(initid, args, message, "MCSLASTINSERTID"); } - long long mcslastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long mcslastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { THD* thd = current_thd; @@ -702,21 +775,21 @@ extern "C" return (nextVal - 1); } - void mcslastinsertid_deinit(UDF_INIT* initid) + void mcslastinsertid_deinit(UDF_INIT* initid) { } - my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return lastinsertid_init(initid, args, message, "CALLASTINSERTID"); } - long long callastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long callastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { return mcslastinsertid(initid, args, is_null, error); } - void callastinsertid_deinit(UDF_INIT* initid) + void callastinsertid_deinit(UDF_INIT* initid) { } @@ -731,41 +804,41 @@ extern "C" return 0; } - void mcsflushcache_deinit(UDF_INIT* initid) + void mcsflushcache_deinit(UDF_INIT* initid) { } - long long mcsflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long mcsflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { return static_cast(cacheutils::flushPrimProcCache()); } - my_bool mcsflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcsflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return flushcache_init(initid, args, message, "MCSFLUSHCACHE"); } - void calflushcache_deinit(UDF_INIT* initid) + void calflushcache_deinit(UDF_INIT* initid) { } - long long calflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long calflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { return mcsflushcache(initid, args, is_null, error); } - my_bool calflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return flushcache_init(initid, args, message, "CALFLUSHCACHE"); } static const unsigned long TraceSize = 16 * 1024; -// mysqld will call this with only 766 bytes available in result no matter what we asked for in -// calgettrace_init() -// if we return a pointer that is not result, mysqld will take our pointer and use it, freeing up result - const char* mcsgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + // mysqld will call this with only 766 bytes available in result no matter what we asked for in + // calgettrace_init() + // if we return a pointer that is not result, mysqld will take our pointer and use it, freeing up result + const char* mcsgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { const std::string* msgp; int flags = 0; @@ -808,7 +881,7 @@ extern "C" return msgp->c_str(); } - my_bool gettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + my_bool gettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { #if 0 @@ -825,27 +898,27 @@ extern "C" return 0; } - my_bool mcsgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcsgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return gettrace_init(initid, args, message, "MCSGETTRACE"); } - void mcsgettrace_deinit(UDF_INIT* initid) + void mcsgettrace_deinit(UDF_INIT* initid) { } - const char* calgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { return mcsgettrace(initid, args, result, length, is_null, error); } - my_bool calgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return gettrace_init(initid, args, message, "CALGETTRACE"); } - void calgettrace_deinit(UDF_INIT* initid) + void calgettrace_deinit(UDF_INIT* initid) { } @@ -860,8 +933,8 @@ extern "C" return 0; } - const char* mcsgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* mcsgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { std::string version(columnstore_version); *length = version.size(); @@ -869,27 +942,27 @@ extern "C" return result; } - my_bool mcsgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcsgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return getversion_init(initid, args, message, "MCSGETVERSION"); } - void mcsgetversion_deinit(UDF_INIT* initid) + void mcsgetversion_deinit(UDF_INIT* initid) { } - const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { return mcsgetversion(initid, args, result, length, is_null, error); } - my_bool calgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return getversion_init(initid, args, message, "CALGETVERSION"); } - void calgetversion_deinit(UDF_INIT* initid) + void calgetversion_deinit(UDF_INIT* initid) { } @@ -904,8 +977,8 @@ extern "C" return 0; } - const char* mcsgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* mcsgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) { @@ -946,37 +1019,37 @@ extern "C" return result; } - my_bool mcsgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcsgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return getstats_init(initid, args, message, "MCSGETSQLCOUNT"); } - void mcsgetsqlcount_deinit(UDF_INIT* initid) + void mcsgetsqlcount_deinit(UDF_INIT* initid) { } - const char* calgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { return mcsgetsqlcount(initid, args, result, length, is_null, error); } - my_bool calgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { return getstats_init(initid, args, message, "CALGETSQLCOUNT"); } - void calgetsqlcount_deinit(UDF_INIT* initid) + void calgetsqlcount_deinit(UDF_INIT* initid) { } - long long mcs_emindex_size(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long mcs_emindex_size(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { DBRM dbrm; return dbrm.EMIndexShmemSize(); } - my_bool mcs_emindex_size_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcs_emindex_size_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { if (args->arg_count != 0) { @@ -987,17 +1060,17 @@ extern "C" return 0; } - void mcs_emindex_size_deinit(UDF_INIT* initid) + void mcs_emindex_size_deinit(UDF_INIT* initid) { } - long long mcs_emindex_free(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long mcs_emindex_free(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { DBRM dbrm; return dbrm.EMIndexShmemFree(); } - my_bool mcs_emindex_free_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcs_emindex_free_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { if (args->arg_count != 0) { @@ -1008,7 +1081,7 @@ extern "C" return 0; } - void mcs_emindex_free_deinit(UDF_INIT* initid) + void mcs_emindex_free_deinit(UDF_INIT* initid) { } diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index d817b464b..bfb677718 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -546,7 +546,7 @@ extern "C" * CalShowPartitions */ - my_bool calshowpartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calshowpartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { if (args->arg_count < 2 || args->arg_count > 3 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || (args->arg_count == 3 && args->arg_type[2] != STRING_RESULT)) @@ -567,13 +567,13 @@ extern "C" return 0; } - void calshowpartitions_deinit(UDF_INIT* initid) + void calshowpartitions_deinit(UDF_INIT* initid) { delete[] initid->ptr; } - const char* calshowpartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calshowpartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { BRM::DBRM::refreshShm(); DBRM em; @@ -701,7 +701,7 @@ extern "C" * CalDisablePartitions */ - my_bool caldisablepartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool caldisablepartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { bool err = false; @@ -735,8 +735,8 @@ extern "C" return 0; } - const char* caldisablepartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* caldisablepartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { CalpontSystemCatalog::TableName tableName; set partitionNums; @@ -772,7 +772,7 @@ extern "C" return result; } - void caldisablepartitions_deinit(UDF_INIT* initid) + void caldisablepartitions_deinit(UDF_INIT* initid) { } @@ -780,7 +780,7 @@ extern "C" * CalEnablePartitions */ - my_bool calenablepartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calenablepartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { bool err = false; @@ -814,8 +814,8 @@ extern "C" return 0; } - const char* calenablepartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* calenablepartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { CalpontSystemCatalog::TableName tableName; string errMsg; @@ -851,7 +851,7 @@ extern "C" return result; } - void calenablepartitions_deinit(UDF_INIT* initid) + void calenablepartitions_deinit(UDF_INIT* initid) { } @@ -859,7 +859,7 @@ extern "C" * CalDropPartitions */ - my_bool caldroppartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool caldroppartitions_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { bool err = false; @@ -893,8 +893,8 @@ extern "C" return 0; } - const char* caldroppartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, - char* is_null, char* error) + const char* caldroppartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { CalpontSystemCatalog::TableName tableName; string errMsg; @@ -930,7 +930,7 @@ extern "C" return result; } - void caldroppartitions_deinit(UDF_INIT* initid) + void caldroppartitions_deinit(UDF_INIT* initid) { } @@ -938,11 +938,11 @@ extern "C" * CalDropPartitionsByValue */ - void caldroppartitionsbyvalue_deinit(UDF_INIT* initid) + void caldroppartitionsbyvalue_deinit(UDF_INIT* initid) { } - my_bool caldroppartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool caldroppartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { bool err = false; @@ -979,8 +979,8 @@ extern "C" return 0; } - const char* caldroppartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, - unsigned long* length, char* is_null, char* error) + const char* caldroppartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { string msg; CalpontSystemCatalog::TableName tableName; @@ -1005,11 +1005,11 @@ extern "C" * CalDisablePartitionsByValue */ - void caldisablepartitionsbyvalue_deinit(UDF_INIT* initid) + void caldisablepartitionsbyvalue_deinit(UDF_INIT* initid) { } - my_bool caldisablepartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool caldisablepartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { bool err = false; @@ -1042,8 +1042,8 @@ extern "C" return 0; } - const char* caldisablepartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, - unsigned long* length, char* is_null, char* error) + const char* caldisablepartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long* length, char* is_null, char* error) { string msg; set partSet; @@ -1067,11 +1067,11 @@ extern "C" /** * CalEnablePartitionsByValue */ - void calenablepartitionsbyvalue_deinit(UDF_INIT* initid) + void calenablepartitionsbyvalue_deinit(UDF_INIT* initid) { } - my_bool calenablepartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calenablepartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { bool err = false; @@ -1104,8 +1104,8 @@ extern "C" return 0; } - const char* calenablepartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, - unsigned long* length, char* is_null, char* error) + const char* calenablepartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long* length, char* is_null, char* error) { string msg; set partSet; @@ -1129,7 +1129,7 @@ extern "C" /** * CalShowPartitionsByValue */ - my_bool calshowpartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calshowpartitionsbyvalue_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { bool err = false; @@ -1162,13 +1162,13 @@ extern "C" return 0; } - void calshowpartitionsbyvalue_deinit(UDF_INIT* initid) + void calshowpartitionsbyvalue_deinit(UDF_INIT* initid) { delete[] initid->ptr; } - const char* calshowpartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, - unsigned long* length, char* is_null, char* error) + const char* calshowpartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) { BRM::DBRM::refreshShm(); DBRM em; diff --git a/dbcon/mysql/install_mcs_mysql.sh.in b/dbcon/mysql/install_mcs_mysql.sh.in index 7d3913cd7..ff80a18cf 100755 --- a/dbcon/mysql/install_mcs_mysql.sh.in +++ b/dbcon/mysql/install_mcs_mysql.sh.in @@ -79,6 +79,7 @@ CREATE OR REPLACE FUNCTION caldisablepartitionsbyvalue RETURNS STRING SONAME 'ha CREATE OR REPLACE FUNCTION calenablepartitionsbyvalue RETURNS STRING SONAME 'ha_columnstore.so'; CREATE OR REPLACE FUNCTION calshowpartitionsbyvalue RETURNS STRING SONAME 'ha_columnstore.so'; CREATE OR REPLACE AGGREGATE FUNCTION moda RETURNS STRING SONAME 'libregr_mysql.so'; +CREATE OR REPLACE FUNCTION mcs_set_ddldebug_level RETURNS STRING SONAME 'ha_columnstore.so'; CREATE DATABASE IF NOT EXISTS infinidb_querystats; CREATE TABLE IF NOT EXISTS infinidb_querystats.querystats diff --git a/ddlproc/ddlprocessor.cpp b/ddlproc/ddlprocessor.cpp index 330c537f3..3b812a442 100644 --- a/ddlproc/ddlprocessor.cpp +++ b/ddlproc/ddlprocessor.cpp @@ -75,8 +75,9 @@ void cleanPMSysCache() class PackageHandler { public: - PackageHandler(QueryTeleClient qtc, DBRM* dbrm, messageqcpp::IOSocket& ios, bool concurrentSupport) - : fIos(ios), fDbrm(dbrm), fQtc(qtc), fConcurrentSupport(concurrentSupport) + PackageHandler(QueryTeleClient qtc, DBRM* dbrm, messageqcpp::IOSocket& ios, bool concurrentSupport, + uint32_t* debugLevel) + : fIos(ios), fDbrm(dbrm), fQtc(qtc), fConcurrentSupport(concurrentSupport), fDebugLevel(debugLevel) { } @@ -490,6 +491,8 @@ class PackageHandler boost::scoped_ptr processor(new CreateTableProcessor(fDbrm)); processor->fTxnid.id = fTxnid.id; processor->fTxnid.valid = true; + if (fDebugLevel) + processor->setDebugLevel(static_cast(*fDebugLevel)); // cout << "create table using txnid " << fTxnid.id << endl; QueryTeleStats qts; @@ -524,6 +527,8 @@ class PackageHandler boost::scoped_ptr processor(new AlterTableProcessor(fDbrm)); processor->fTxnid.id = fTxnid.id; processor->fTxnid.valid = true; + if (fDebugLevel) + processor->setDebugLevel(static_cast(*fDebugLevel)); QueryTeleStats qts; qts.query_uuid = QueryTeleClient::genUUID(); @@ -560,6 +565,8 @@ class PackageHandler processor->fTxnid.id = fTxnid.id; processor->fTxnid.valid = true; + if (fDebugLevel) + processor->setDebugLevel(static_cast(*fDebugLevel)); QueryTeleStats qts; qts.query_uuid = QueryTeleClient::genUUID(); @@ -595,6 +602,8 @@ class PackageHandler processor->fTxnid.id = fTxnid.id; processor->fTxnid.valid = true; + if (fDebugLevel) + processor->setDebugLevel(static_cast(*fDebugLevel)); QueryTeleStats qts; qts.query_uuid = QueryTeleClient::genUUID(); @@ -628,6 +637,9 @@ class PackageHandler boost::scoped_ptr processor(new MarkPartitionProcessor(fDbrm)); (processor->fTxnid).id = fTxnid.id; (processor->fTxnid).valid = true; + if (fDebugLevel) + processor->setDebugLevel(static_cast(*fDebugLevel)); + result = processor->processPackage(&markPartitionStmt); systemCatalogPtr->removeCalpontSystemCatalog(markPartitionStmt.fSessionID); systemCatalogPtr->removeCalpontSystemCatalog(markPartitionStmt.fSessionID | 0x80000000); @@ -643,6 +655,9 @@ class PackageHandler boost::scoped_ptr processor(new RestorePartitionProcessor(fDbrm)); (processor->fTxnid).id = fTxnid.id; (processor->fTxnid).valid = true; + if (fDebugLevel) + processor->setDebugLevel(static_cast(*fDebugLevel)); + result = processor->processPackage(&restorePartitionStmt); systemCatalogPtr->removeCalpontSystemCatalog(restorePartitionStmt.fSessionID); systemCatalogPtr->removeCalpontSystemCatalog(restorePartitionStmt.fSessionID | 0x80000000); @@ -658,12 +673,24 @@ class PackageHandler boost::scoped_ptr processor(new DropPartitionProcessor(fDbrm)); (processor->fTxnid).id = fTxnid.id; (processor->fTxnid).valid = true; + if (fDebugLevel) + processor->setDebugLevel(static_cast(*fDebugLevel)); + result = processor->processPackage(&dropPartitionStmt); systemCatalogPtr->removeCalpontSystemCatalog(dropPartitionStmt.fSessionID); systemCatalogPtr->removeCalpontSystemCatalog(dropPartitionStmt.fSessionID | 0x80000000); } break; + case ddlpackage::DDL_DEBUG_STATEMENT: + { + DebugDDLStatement stmt; + stmt.unserialize(fByteStream); + if (fDebugLevel) + *fDebugLevel = stmt.fDebugLevel; + } + break; + default: throw UNRECOGNIZED_PACKAGE_TYPE; break; } @@ -728,6 +755,7 @@ class PackageHandler QueryTeleClient fQtc; oam::OamCache* fOamCache = nullptr; bool fConcurrentSupport; + uint32_t* fDebugLevel{nullptr}; }; } // namespace @@ -782,7 +810,7 @@ void DDLProcessor::process() try { ios = fMqServer.accept(); - fDdlPackagepool.invoke(PackageHandler(fQtc, &dbrm, ios, concurrentSupport)); + fDdlPackagepool.invoke(PackageHandler(fQtc, &dbrm, ios, concurrentSupport, &debugLevel)); } catch (...) { diff --git a/ddlproc/ddlprocessor.h b/ddlproc/ddlprocessor.h index b5af43dd7..82bf22d2c 100644 --- a/ddlproc/ddlprocessor.h +++ b/ddlproc/ddlprocessor.h @@ -93,6 +93,7 @@ class DDLProcessor WriteEngine::WEClients* fWEClient; uint32_t fPMCount; querytele::QueryTeleClient fQtc; + uint32_t debugLevel{0}; }; } // namespace ddlprocessor From bd9dfdf38c3b74ab2ce90aaa401b0314b691ac4d Mon Sep 17 00:00:00 2001 From: Allen Herrera <82840027+mariadb-AllenHerrera@users.noreply.github.com> Date: Wed, 11 Sep 2024 04:50:41 -0700 Subject: [PATCH 10/65] feat(installation): MCOL-4886 skip chown if top folder permissions good (#3093) * MCOL-4886 skip chown if top folder permissions good * remove '-type f' check permissions of directories too --- oam/install_scripts/columnstore-post-install.in | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 8222c1a72..a25a2eccc 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -375,7 +375,17 @@ fi if [ $(running_systemd) -eq 0 ]; then chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ @ENGINE_LOGDIR@ chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ /etc/columnstore - chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ @ENGINE_DATADIR@ + findcmd=`which find 2>/dev/null` + if [ -n "$findcmd" ]; then + if [[ $($findcmd @ENGINE_DATADIR@ -maxdepth 3 ! -user @DEFAULT_USER@ -exec echo {} \; -quit 2>/dev/null) ]]; then + echo "At least one file is not owned by @DEFAULT_USER@ in @ENGINE_DATADIR@. Running chown..." + chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ @ENGINE_DATADIR@ + else + echo "Confirmed top @ENGINE_DATADIR@ folders owned by @DEFAULT_USER@" + fi + else + chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ @ENGINE_DATADIR@ + fi chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ $tmpDir chmod 777 /dev/shm fi From 42be2cb7e0c484a02caa434e35b21c350c6f199f Mon Sep 17 00:00:00 2001 From: Aleksei Antipovskii Date: Fri, 18 Oct 2024 19:13:47 +0200 Subject: [PATCH 11/65] fix(dbcon) MCOL-5812 server crash related to stored functions Using the stored function's return value as an argument for another function was handled incorrectly, leading to a server crash. --- dbcon/mysql/ha_mcs_pushdown.cpp | 4 + .../columnstore/bugfixes/mcol-5812.result | 202 ++++++++++++++++++ .../columnstore/bugfixes/mcol-5812.test | 121 +++++++++++ 3 files changed, 327 insertions(+) create mode 100644 mysql-test/columnstore/bugfixes/mcol-5812.result create mode 100644 mysql-test/columnstore/bugfixes/mcol-5812.test diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index 94181e261..cbf4b66d6 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -364,6 +364,10 @@ void item_check(Item* item, bool* unsupported_feature) bool check_user_var(SELECT_LEX* select_lex) { + if (!select_lex) { + // There are definitely no user vars if select_lex is null + return false; + } List_iterator_fast it(select_lex->item_list); Item* item; bool is_user_var_func = false; diff --git a/mysql-test/columnstore/bugfixes/mcol-5812.result b/mysql-test/columnstore/bugfixes/mcol-5812.result new file mode 100644 index 000000000..59b4ab152 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-5812.result @@ -0,0 +1,202 @@ +DROP DATABASE IF EXISTS mcol5812; +CREATE DATABASE mcol5812; +USE mcol5812; +CREATE TABLE `cs_test_exclude` ( +`username` varchar(128) NOT NULL DEFAULT '', +`location_id` int(11) NOT NULL, +`inclusive` tinyint(1) NOT NULL DEFAULT 0, +`gbd_round_id` int(11) NOT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; +CREATE TABLE `cs_test_lhh` ( +`location_set_version_id` int(11) NOT NULL, +`location_set_id` int(11) NOT NULL, +`location_id` int(11) NOT NULL, +`parent_id` int(11) NOT NULL, +`top_prnt` varchar(200) DEFAULT NULL, +`level` int(11) NOT NULL, +`is_estimate` tinyint(1) NOT NULL DEFAULT 0, +`most_detailed` tinyint(4) DEFAULT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; +INSERT INTO `cs_test_exclude` VALUES +('Value1',62,0,7),('Value1',63,0,7),('Value1',72,0,7),('Value1',4841,1,7),('Value1',4842,1,7), +('Value1',4843,1,7),('Value1',4844,1,7),('Value1',4846,1,7),('Value1',4849,1,7), +('Value1',4850,1,7),('Value1',4851,1,7),('Value1',4852,1,7),('Value1',4853,1,7), +('Value1',4854,1,7),('Value1',4855,1,7),('Value1',4856,1,7),('Value1',4857,1,7), +('Value1',4859,1,7),('Value1',4860,1,7),('Value1',4861,1,7),('Value1',4862,1,7), +('Value1',4863,1,7),('Value1',4864,1,7),('Value1',4865,1,7),('Value1',4867,1,7), +('Value1',4868,1,7),('Value1',4869,1,7),('Value1',4870,1,7),('Value1',4871,1,7), +('Value1',4872,1,7),('Value1',4873,1,7),('Value1',4874,1,7),('Value1',4875,1,7), +('Value1',44538,1,7); +INSERT INTO `cs_test_lhh` VALUES +(1,958664876,246190364,1249,'62',2,0,0),(2,203385377,71291823,1248,'4875',4,0,0), +(3,330636575,599349854,1248,'72',2,1,0),(4,83193844,69601420,1248,'4841',2,1,2), +(5,880576199,919233821,1251,'62',2,0,1),(6,423710106,372141638,1250,'113',4,0,0), +(7,964870752,706911487,1250,'63',5,0,1),(8,811505779,288644012,1248,'72',8,1,1), +(9,863527754,867131640,1248,'63',4,1,1),(10,997630965,903920363,1250,'114',1,0,0), +(11,67092268,643249095,1248,'63',7,1,1),(12,3886045,783815976,1251,'72',2,0,1), +(13,358564822,831193421,1248,'114',3,1,0),(14,204755399,806303011,1249,'111',1,0,0), +(15,547308388,662836201,1248,'4842',8,0,0),(16,218564918,481884787,1248,'4842',4,1,0), +(17,639801213,326560225,1248,'44538',8,0,1),(18,319975567,930174978,1248,'111',3,0,2), +(19,228530438,909067697,1251,'44538',3,1,2),(20,989807004,760825441,1251,'114',9,0,2), +(21,71143451,705834119,1247,'4843',4,0,0),(22,298278074,557207242,1251,'113',3,1,1), +(23,18903111,627537731,1248,'4875',4,0,1),(24,858596664,687567389,1251,'4841',6,1,1), +(25,925293899,954442412,1251,'63',6,1,2),(26,754240682,449170390,1251,'44538',9,1,0), +(27,622033315,581225374,1248,'4843',2,0,1),(28,253668752,416962235,1247,'4842',8,0,1), +(29,614116441,218551159,1247,'44538',3,0,0),(30,376548495,224463308,1251,'4841',5,0,0), +(31,330713421,600474630,1248,'4841',2,0,0),(32,988170605,257929822,1247,'114',3,1,2), +(33,392917009,680986028,1247,'63',7,1,2),(34,248884530,727794041,1251,'4841',7,1,2), +(35,762392593,389261523,1250,'63',6,1,1),(36,288099880,107542410,1248,'62',2,1,1), +(37,927146744,882593710,1250,'4875',6,1,1),(38,469238982,704069569,1248,'4843',9,0,2), +(39,467061012,323832344,1247,'63',9,0,0),(40,917004206,684981061,1248,'4842',5,1,2), +(41,688679370,705739286,1249,'72',6,0,0),(42,408049260,30613069,1251,'44538',9,0,1), +(43,54000735,273162722,1247,'72',4,0,2),(44,164671784,211068500,1250,'72',2,0,2), +(45,488378905,578003329,1249,'4843',7,0,0),(46,758198951,55172361,1248,'113',2,0,1), +(47,266894388,746743719,1251,'4843',3,0,1),(48,71736102,694360233,1247,'72',3,1,0), +(49,957245979,914867965,1248,'112',7,0,1),(50,301352564,397696525,1248,'72',9,1,0), +(51,512235534,718530555,1248,'63',5,1,0),(52,839568220,712650810,1248,'63',3,0,0), +(53,575910842,926645552,1251,'112',1,1,1),(54,628010357,106713826,1250,'115',7,1,1), +(55,500726159,757662497,1247,'72',9,0,0),(56,435533157,345284473,1249,'62',1,0,1), +(57,216127503,408342278,1249,'112',5,0,2),(58,632536551,494012796,1250,'4842',2,1,1), +(59,808450911,944931619,1247,'111',4,0,0),(60,401138482,966265756,1250,'4841',3,1,2), +(61,306189266,971603360,1251,'113',1,0,1),(62,383512515,97432722,1249,'4843',9,1,0), +(63,823002329,380225321,1249,'62',8,1,1),(64,850675079,272242917,1248,'72',7,1,1), +(65,546386018,339705923,1248,'4841',2,0,2),(66,972614758,839392045,1247,'114',5,0,0), +(67,754887111,844479876,1251,'4841',4,0,0),(68,96863327,157587773,1249,'62',6,1,2), +(69,359206490,874831340,1247,'114',4,0,0),(70,568416298,378659531,1247,'113',5,1,0), +(71,770107723,799396801,1248,'62',2,0,0),(72,267553306,348071087,1251,'111',4,0,1), +(73,352593522,980460449,1251,'4841',8,1,0),(74,733381379,666045858,1248,'111',8,0,1), +(75,412482330,927782012,1249,'72',9,1,1),(76,592328197,401156054,1247,'115',1,0,1), +(77,564885656,439254004,1250,'72',5,1,0),(78,453881920,646915758,1251,'4843',5,0,1), +(79,7041781,245431073,1247,'4841',8,0,1),(80,389362773,540027369,1250,'62',6,1,1), +(81,859161520,920208461,1248,'4842',7,1,1),(82,702478077,414598228,1251,'44538',1,0,1), +(83,82773827,410201619,1248,'113',5,0,1),(84,881657037,174790010,1247,'111',4,0,0), +(85,163860768,214199912,1250,'4841',5,1,0),(86,950914751,901637761,1250,'44538',9,1,0), +(87,34289392,352757936,1250,'4841',3,1,1),(88,597054039,821408137,1247,'63',6,1,0), +(89,114114883,334611378,1247,'111',3,0,0),(90,735863059,223069230,1251,'114',6,1,1), +(91,636187531,343234972,1248,'62',6,0,2),(92,765890679,85421062,1248,'4843',6,1,2), +(93,172538451,891634438,1251,'62',3,1,2),(94,9107635,10400866,1248,'63',4,1,0), +(95,995075180,943586298,1248,'113',9,0,1),(96,646629630,886808768,1249,'113',6,0,1), +(97,885926569,634024443,1250,'111',7,1,0),(98,320024878,276165077,1249,'4841',2,1,1), +(99,193259578,966379021,1247,'4842',1,0,1),(100,798259742,56493273,1251,'4841',7,1,0); +CREATE FUNCTION `cs_test_function`(p_username VARCHAR(128), p_gbd_round_id INT(11)) RETURNS text CHARSET utf8mb3 COLLATE utf8mb3_general_ci +READS SQL DATA +DETERMINISTIC +BEGIN +DECLARE exclude_set TEXT; +SELECT GROUP_CONCAT(CONCAT(location_id,IF(inclusive,'($|,)',',')) SEPARATOR '|') +INTO exclude_set +FROM cs_test_exclude +WHERE username=p_username +AND gbd_round_id=p_gbd_round_id +GROUP BY username; +IF exclude_set IS NOT NULL THEN +RETURN(CONCAT(',(',exclude_set,')')); +END IF; +RETURN('^$'); +END $$ +SELECT location_id +FROM cs_test_lhh +WHERE (top_prnt NOT REGEXP cs_test_function('Value1', 7)); +location_id +246190364 +71291823 +599349854 +69601420 +919233821 +372141638 +706911487 +288644012 +867131640 +903920363 +643249095 +783815976 +831193421 +806303011 +662836201 +481884787 +326560225 +930174978 +909067697 +760825441 +705834119 +557207242 +627537731 +687567389 +954442412 +449170390 +581225374 +416962235 +218551159 +224463308 +600474630 +257929822 +680986028 +727794041 +389261523 +107542410 +882593710 +704069569 +323832344 +684981061 +705739286 +30613069 +273162722 +211068500 +578003329 +55172361 +746743719 +694360233 +914867965 +397696525 +718530555 +712650810 +926645552 +106713826 +757662497 +345284473 +408342278 +494012796 +944931619 +966265756 +971603360 +97432722 +380225321 +272242917 +339705923 +839392045 +844479876 +157587773 +874831340 +378659531 +799396801 +348071087 +980460449 +666045858 +927782012 +401156054 +439254004 +646915758 +245431073 +540027369 +920208461 +414598228 +410201619 +174790010 +214199912 +901637761 +352757936 +821408137 +334611378 +223069230 +343234972 +85421062 +891634438 +10400866 +943586298 +886808768 +634024443 +276165077 +966379021 +56493273 +DROP FUNCTION IF EXISTS `cs_test_function`; +DROP DATABASE mcol5812; diff --git a/mysql-test/columnstore/bugfixes/mcol-5812.test b/mysql-test/columnstore/bugfixes/mcol-5812.test new file mode 100644 index 000000000..a28080659 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-5812.test @@ -0,0 +1,121 @@ +--source ../include/have_columnstore.inc + +--disable_warnings +DROP DATABASE IF EXISTS mcol5812; +--enable_warnings + +CREATE DATABASE mcol5812; +USE mcol5812; + +# DROP TABLE IF EXISTS `cs_test_exclude`; +CREATE TABLE `cs_test_exclude` ( + `username` varchar(128) NOT NULL DEFAULT '', + `location_id` int(11) NOT NULL, + `inclusive` tinyint(1) NOT NULL DEFAULT 0, + `gbd_round_id` int(11) NOT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +# DROP TABLE IF EXISTS `cs_test_lhh`; +CREATE TABLE `cs_test_lhh` ( + `location_set_version_id` int(11) NOT NULL, + `location_set_id` int(11) NOT NULL, + `location_id` int(11) NOT NULL, + `parent_id` int(11) NOT NULL, + `top_prnt` varchar(200) DEFAULT NULL, + `level` int(11) NOT NULL, + `is_estimate` tinyint(1) NOT NULL DEFAULT 0, + `most_detailed` tinyint(4) DEFAULT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; + +INSERT INTO `cs_test_exclude` VALUES + ('Value1',62,0,7),('Value1',63,0,7),('Value1',72,0,7),('Value1',4841,1,7),('Value1',4842,1,7), + ('Value1',4843,1,7),('Value1',4844,1,7),('Value1',4846,1,7),('Value1',4849,1,7), + ('Value1',4850,1,7),('Value1',4851,1,7),('Value1',4852,1,7),('Value1',4853,1,7), + ('Value1',4854,1,7),('Value1',4855,1,7),('Value1',4856,1,7),('Value1',4857,1,7), + ('Value1',4859,1,7),('Value1',4860,1,7),('Value1',4861,1,7),('Value1',4862,1,7), + ('Value1',4863,1,7),('Value1',4864,1,7),('Value1',4865,1,7),('Value1',4867,1,7), + ('Value1',4868,1,7),('Value1',4869,1,7),('Value1',4870,1,7),('Value1',4871,1,7), + ('Value1',4872,1,7),('Value1',4873,1,7),('Value1',4874,1,7),('Value1',4875,1,7), + ('Value1',44538,1,7); + +INSERT INTO `cs_test_lhh` VALUES + (1,958664876,246190364,1249,'62',2,0,0),(2,203385377,71291823,1248,'4875',4,0,0), + (3,330636575,599349854,1248,'72',2,1,0),(4,83193844,69601420,1248,'4841',2,1,2), + (5,880576199,919233821,1251,'62',2,0,1),(6,423710106,372141638,1250,'113',4,0,0), + (7,964870752,706911487,1250,'63',5,0,1),(8,811505779,288644012,1248,'72',8,1,1), + (9,863527754,867131640,1248,'63',4,1,1),(10,997630965,903920363,1250,'114',1,0,0), + (11,67092268,643249095,1248,'63',7,1,1),(12,3886045,783815976,1251,'72',2,0,1), + (13,358564822,831193421,1248,'114',3,1,0),(14,204755399,806303011,1249,'111',1,0,0), + (15,547308388,662836201,1248,'4842',8,0,0),(16,218564918,481884787,1248,'4842',4,1,0), + (17,639801213,326560225,1248,'44538',8,0,1),(18,319975567,930174978,1248,'111',3,0,2), + (19,228530438,909067697,1251,'44538',3,1,2),(20,989807004,760825441,1251,'114',9,0,2), + (21,71143451,705834119,1247,'4843',4,0,0),(22,298278074,557207242,1251,'113',3,1,1), + (23,18903111,627537731,1248,'4875',4,0,1),(24,858596664,687567389,1251,'4841',6,1,1), + (25,925293899,954442412,1251,'63',6,1,2),(26,754240682,449170390,1251,'44538',9,1,0), + (27,622033315,581225374,1248,'4843',2,0,1),(28,253668752,416962235,1247,'4842',8,0,1), + (29,614116441,218551159,1247,'44538',3,0,0),(30,376548495,224463308,1251,'4841',5,0,0), + (31,330713421,600474630,1248,'4841',2,0,0),(32,988170605,257929822,1247,'114',3,1,2), + (33,392917009,680986028,1247,'63',7,1,2),(34,248884530,727794041,1251,'4841',7,1,2), + (35,762392593,389261523,1250,'63',6,1,1),(36,288099880,107542410,1248,'62',2,1,1), + (37,927146744,882593710,1250,'4875',6,1,1),(38,469238982,704069569,1248,'4843',9,0,2), + (39,467061012,323832344,1247,'63',9,0,0),(40,917004206,684981061,1248,'4842',5,1,2), + (41,688679370,705739286,1249,'72',6,0,0),(42,408049260,30613069,1251,'44538',9,0,1), + (43,54000735,273162722,1247,'72',4,0,2),(44,164671784,211068500,1250,'72',2,0,2), + (45,488378905,578003329,1249,'4843',7,0,0),(46,758198951,55172361,1248,'113',2,0,1), + (47,266894388,746743719,1251,'4843',3,0,1),(48,71736102,694360233,1247,'72',3,1,0), + (49,957245979,914867965,1248,'112',7,0,1),(50,301352564,397696525,1248,'72',9,1,0), + (51,512235534,718530555,1248,'63',5,1,0),(52,839568220,712650810,1248,'63',3,0,0), + (53,575910842,926645552,1251,'112',1,1,1),(54,628010357,106713826,1250,'115',7,1,1), + (55,500726159,757662497,1247,'72',9,0,0),(56,435533157,345284473,1249,'62',1,0,1), + (57,216127503,408342278,1249,'112',5,0,2),(58,632536551,494012796,1250,'4842',2,1,1), + (59,808450911,944931619,1247,'111',4,0,0),(60,401138482,966265756,1250,'4841',3,1,2), + (61,306189266,971603360,1251,'113',1,0,1),(62,383512515,97432722,1249,'4843',9,1,0), + (63,823002329,380225321,1249,'62',8,1,1),(64,850675079,272242917,1248,'72',7,1,1), + (65,546386018,339705923,1248,'4841',2,0,2),(66,972614758,839392045,1247,'114',5,0,0), + (67,754887111,844479876,1251,'4841',4,0,0),(68,96863327,157587773,1249,'62',6,1,2), + (69,359206490,874831340,1247,'114',4,0,0),(70,568416298,378659531,1247,'113',5,1,0), + (71,770107723,799396801,1248,'62',2,0,0),(72,267553306,348071087,1251,'111',4,0,1), + (73,352593522,980460449,1251,'4841',8,1,0),(74,733381379,666045858,1248,'111',8,0,1), + (75,412482330,927782012,1249,'72',9,1,1),(76,592328197,401156054,1247,'115',1,0,1), + (77,564885656,439254004,1250,'72',5,1,0),(78,453881920,646915758,1251,'4843',5,0,1), + (79,7041781,245431073,1247,'4841',8,0,1),(80,389362773,540027369,1250,'62',6,1,1), + (81,859161520,920208461,1248,'4842',7,1,1),(82,702478077,414598228,1251,'44538',1,0,1), + (83,82773827,410201619,1248,'113',5,0,1),(84,881657037,174790010,1247,'111',4,0,0), + (85,163860768,214199912,1250,'4841',5,1,0),(86,950914751,901637761,1250,'44538',9,1,0), + (87,34289392,352757936,1250,'4841',3,1,1),(88,597054039,821408137,1247,'63',6,1,0), + (89,114114883,334611378,1247,'111',3,0,0),(90,735863059,223069230,1251,'114',6,1,1), + (91,636187531,343234972,1248,'62',6,0,2),(92,765890679,85421062,1248,'4843',6,1,2), + (93,172538451,891634438,1251,'62',3,1,2),(94,9107635,10400866,1248,'63',4,1,0), + (95,995075180,943586298,1248,'113',9,0,1),(96,646629630,886808768,1249,'113',6,0,1), + (97,885926569,634024443,1250,'111',7,1,0),(98,320024878,276165077,1249,'4841',2,1,1), + (99,193259578,966379021,1247,'4842',1,0,1),(100,798259742,56493273,1251,'4841',7,1,0); + +DELIMITER $$; +CREATE FUNCTION `cs_test_function`(p_username VARCHAR(128), p_gbd_round_id INT(11)) RETURNS text CHARSET utf8mb3 COLLATE utf8mb3_general_ci + READS SQL DATA + DETERMINISTIC +BEGIN + DECLARE exclude_set TEXT; + + SELECT GROUP_CONCAT(CONCAT(location_id,IF(inclusive,'($|,)',',')) SEPARATOR '|') + INTO exclude_set + FROM cs_test_exclude + WHERE username=p_username + AND gbd_round_id=p_gbd_round_id + GROUP BY username; + + IF exclude_set IS NOT NULL THEN + RETURN(CONCAT(',(',exclude_set,')')); + END IF; + + RETURN('^$'); +END $$ +DELIMITER ;$$ + +SELECT location_id +FROM cs_test_lhh +WHERE (top_prnt NOT REGEXP cs_test_function('Value1', 7)); + +# cleanup +DROP FUNCTION IF EXISTS `cs_test_function`; +DROP DATABASE mcol5812; From 842a3c8a40c4ece404b3d3bfe8f819af27a18d20 Mon Sep 17 00:00:00 2001 From: Alexey Antipovsky Date: Fri, 8 Nov 2024 18:44:20 +0100 Subject: [PATCH 12/65] =?UTF-8?q?fix(PrimProc):=20MCOL-5651=20Add=20a=20wo?= =?UTF-8?q?rkaround=20to=20avoid=20choosing=20an=20incorr=E2=80=A6=20(#332?= =?UTF-8?q?0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(PrimProc): MCOL-5651 Add a workaround to avoid choosing an incorrect TupleHashJoinStep as a joiner --- dbcon/joblist/jlf_execplantojoblist.cpp | 2 +- dbcon/joblist/jlf_tuplejoblist.cpp | 188 +++++++++--------- dbcon/joblist/tuplehashjoin.cpp | 4 +- .../columnstore/bugfixes/mcol-5651.result | 63 ++++++ .../columnstore/bugfixes/mcol-5651.test | 56 ++++++ .../primproc/batchprimitiveprocessor.cpp | 43 ++-- utils/joiner/tuplejoiner.cpp | 4 +- 7 files changed, 242 insertions(+), 118 deletions(-) create mode 100644 mysql-test/columnstore/bugfixes/mcol-5651.result create mode 100644 mysql-test/columnstore/bugfixes/mcol-5651.test diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index f6d4f686a..c43fc15b2 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1478,7 +1478,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) { JobStepVector jsv; - if (sf == 0) + if (sf == nullptr) return jsv; // cout << "doSimpleFilter " << endl; diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index ea095587c..0413a8bae 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -3078,7 +3078,7 @@ void createPostJoinFilters(const JobInfo& jobInfo, TableInfoMap& tableInfoMap, if (jobInfo.trace) { - if (postJoinFilters.size()) + if (!postJoinFilters.empty()) { cout << "Post join filters created." << endl; for (auto* filter : postJoinFilters) @@ -3100,37 +3100,37 @@ SP_JoinInfo joinToLargeTable(uint32_t large, TableInfoMap& tableInfoMap, JobInfo set& tableSet = tableInfoMap[large].fJoinedTables; vector& adjList = tableInfoMap[large].fAdjacentList; uint32_t prevLarge = (uint32_t)getPrevLarge(large, tableInfoMap); - bool root = (prevLarge == (uint32_t)-1) ? true : false; + bool root = (prevLarge == (uint32_t)-1); uint32_t link = large; uint32_t cId = -1; // Get small sides ready. - for (vector::iterator i = adjList.begin(); i != adjList.end(); i++) + for (unsigned int& adj : adjList) { - if (tableInfoMap[*i].fVisited == false) + if (!tableInfoMap[adj].fVisited) { - cId = *i; - smallSides.push_back(joinToLargeTable(*i, tableInfoMap, jobInfo, joinOrder, joinEdgesToRestore)); + cId = adj; + smallSides.push_back(joinToLargeTable(adj, tableInfoMap, jobInfo, joinOrder, joinEdgesToRestore)); - tableSet.insert(tableInfoMap[*i].fJoinedTables.begin(), tableInfoMap[*i].fJoinedTables.end()); + tableSet.insert(tableInfoMap[adj].fJoinedTables.begin(), tableInfoMap[adj].fJoinedTables.end()); } } // Join with its small sides, if not a leaf node. - if (smallSides.size() > 0) + if (!smallSides.empty()) { // non-leaf node, need a join SJSTEP spjs = tableInfoMap[large].fQuerySteps.back(); - BatchPrimitive* bps = dynamic_cast(spjs.get()); - SubAdapterStep* tsas = dynamic_cast(spjs.get()); - TupleHashJoinStep* thjs = dynamic_cast(spjs.get()); + auto* bps = dynamic_cast(spjs.get()); + auto* tsas = dynamic_cast(spjs.get()); + auto* thjs = dynamic_cast(spjs.get()); // @bug6158, try to put BPS on large side if possible if (tsas && smallSides.size() == 1) { SJSTEP sspjs = tableInfoMap[cId].fQuerySteps.back(); - BatchPrimitive* sbps = dynamic_cast(sspjs.get()); - TupleHashJoinStep* sthjs = dynamic_cast(sspjs.get()); + auto* sbps = dynamic_cast(sspjs.get()); + auto* sthjs = dynamic_cast(sspjs.get()); if (sbps || (sthjs && sthjs->tokenJoin() == cId)) { @@ -3143,7 +3143,7 @@ SP_JoinInfo joinToLargeTable(uint32_t large, TableInfoMap& tableInfoMap, JobInfo largeJoinInfo->fDl = tableInfoMap[large].fDl; largeJoinInfo->fRowGroup = tableInfoMap[large].fRowGroup; - TableJoinMap::iterator mit = jobInfo.tableJoinMap.find(make_pair(large, cId)); + auto mit = jobInfo.tableJoinMap.find(make_pair(large, cId)); if (mit == jobInfo.tableJoinMap.end()) throw runtime_error("Join step not found."); @@ -3158,7 +3158,7 @@ SP_JoinInfo joinToLargeTable(uint32_t large, TableInfoMap& tableInfoMap, JobInfo bps = sbps; thjs = sthjs; - tsas = NULL; + tsas = nullptr; } } @@ -3173,7 +3173,7 @@ SP_JoinInfo joinToLargeTable(uint32_t large, TableInfoMap& tableInfoMap, JobInfo size_t dcf = 0; // for dictionary column filters, 0 if thjs is null. RowGroup largeSideRG = tableInfoMap[large].fRowGroup; - if (thjs && thjs->tokenJoin() == large) + if (thjs && thjs->tokenJoin() == large && thjs->tupleId1() != thjs->tupleId2()) { dcf = thjs->getLargeKeys().size(); largeSideRG = thjs->getLargeRowGroup(); @@ -3195,9 +3195,9 @@ SP_JoinInfo joinToLargeTable(uint32_t large, TableInfoMap& tableInfoMap, JobInfo vector> smallKeyIndices; vector> largeKeyIndices; - for (vector::iterator i = smallSides.begin(); i != smallSides.end(); i++) + for (auto& smallSide : smallSides) { - JoinInfo* info = i->get(); + JoinInfo* info = smallSide.get(); smallSideDLs.push_back(info->fDl); smallSideRGs.push_back(info->fRowGroup); jointypes.push_back(info->fJoinData.fTypes[0]); @@ -3207,8 +3207,8 @@ SP_JoinInfo joinToLargeTable(uint32_t large, TableInfoMap& tableInfoMap, JobInfo vector largeIndices; const vector& keys1 = info->fJoinData.fLeftKeys; const vector& keys2 = info->fJoinData.fRightKeys; - vector::const_iterator k1 = keys1.begin(); - vector::const_iterator k2 = keys2.begin(); + auto k1 = keys1.begin(); + auto k2 = keys2.begin(); uint32_t stid = getTableKey(jobInfo, *k1); tableNames.push_back(jobInfo.keyInfo->tupleKeyVec[stid].fTable); @@ -3267,7 +3267,8 @@ SP_JoinInfo joinToLargeTable(uint32_t large, TableInfoMap& tableInfoMap, JobInfo traces.push_back(oss.str()); } - if (bps || tsas) + // If the tupleIDs are the same it's not a join, so a new TupleHashJoinStep must be created + if (bps || tsas || (thjs && thjs->tupleId1() == thjs->tupleId2())) { thjs = new TupleHashJoinStep(jobInfo); thjs->tableOid1(smallSides[0]->fTableOid); @@ -4379,10 +4380,14 @@ inline void joinTables(JobStepVector& joinSteps, TableInfoMap& tableInfoMap, Job { uint32_t largestTable = getLargestTable(jobInfo, tableInfoMap, overrideLargeSideEstimate); - if (jobInfo.outerOnTable.size() == 0) + if (jobInfo.outerOnTable.empty()) + { joinToLargeTable(largestTable, tableInfoMap, jobInfo, joinOrder, jobInfo.joinEdgesToRestore); + } else + { joinTablesInOrder(largestTable, joinSteps, tableInfoMap, jobInfo, joinOrder); + } } void makeNoTableJobStep(JobStepVector& querySteps, JobStepVector& projectSteps, @@ -4407,14 +4412,14 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte const boost::shared_ptr& keyInfo = jobInfo.keyInfo; cout << "query steps:" << endl; - for (JobStepVector::iterator i = querySteps.begin(); i != querySteps.end(); ++i) + for (const auto& step: querySteps) { - TupleHashJoinStep* thjs = dynamic_cast(i->get()); + auto* thjs = dynamic_cast(step.get()); - if (thjs == NULL) + if (thjs == nullptr) { - int64_t id = ((*i)->tupleId() != (uint64_t)-1) ? (*i)->tupleId() : -1; - cout << typeid(*(i->get())).name() << ": " << (*i)->oid() << " " << id << " " + int64_t id = (step->tupleId() != (uint64_t)-1) ? step->tupleId() : -1; + cout << typeid(step.get()).name() << ": " << step->oid() << " " << id << " " << (int)((id != -1) ? getTableKey(jobInfo, id) : -1) << endl; } else @@ -4430,16 +4435,18 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte cout << "project steps:" << endl; - for (JobStepVector::iterator i = projectSteps.begin(); i != projectSteps.end(); ++i) + for (const auto& prStep: projectSteps) { - cout << typeid(*(i->get())).name() << ": " << (*i)->oid() << " " << (*i)->tupleId() << " " - << getTableKey(jobInfo, (*i)->tupleId()) << endl; + cout << typeid(prStep.get()).name() << ": " << prStep->oid() << " " << prStep->tupleId() << " " + << getTableKey(jobInfo, prStep->tupleId()) << endl; } cout << "delivery steps:" << endl; - for (DeliveredTableMap::iterator i = deliverySteps.begin(); i != deliverySteps.end(); ++i) - cout << typeid(*(i->second.get())).name() << endl; + for (const auto& [_, value]: deliverySteps) + { + cout << typeid(value.get()).name() << endl; + } cout << "\nTable Info: (key oid name alias view sub)" << endl; @@ -4452,7 +4459,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte CalpontSystemCatalog::OID oid = keyInfo->tupleKeyVec[i].fId; string alias = keyInfo->tupleKeyVec[i].fTable; - if (alias.length() < 1) + if (alias.empty()) alias = "N/A"; string name = keyInfo->keyName[i]; @@ -4462,10 +4469,10 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte string view = keyInfo->tupleKeyVec[i].fView; - if (view.length() < 1) + if (view.empty()) view = "N/A"; - int sid = keyInfo->tupleKeyVec[i].fSubId; + auto sid = keyInfo->tupleKeyVec[i].fSubId; cout << i << "\t" << oid << "\t" << name << "\t" << alias << "\t" << view << "\t" << hex << sid << dec << endl; } @@ -4479,7 +4486,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte int64_t tid = jobInfo.keyInfo->colKeyToTblKey[i]; string alias = keyInfo->tupleKeyVec[i].fTable; - if (alias.length() < 1) + if (alias.empty()) alias = "N/A"; // Expression IDs are borrowed from systemcatalog IDs, which are not used in tuple. @@ -4502,10 +4509,10 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte string view = keyInfo->tupleKeyVec[i].fView; - if (view.length() < 1) + if (view.empty()) view = "N/A"; - int sid = keyInfo->tupleKeyVec[i].fSubId; + auto sid = keyInfo->tupleKeyVec[i].fSubId; cout << i << "\t" << oid << "\t" << tid << "\t" << name << "\t" << alias << "\t" << view << "\t" << hex << sid << dec << endl; } @@ -4514,7 +4521,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte } // @bug 2771, handle no table select query - if (jobInfo.tableList.size() < 1) + if (jobInfo.tableList.empty()) { makeNoTableJobStep(querySteps, projectSteps, deliverySteps, jobInfo); return; @@ -4537,33 +4544,33 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte } // Set of the columns being projected. - for (TupleInfoVector::iterator i = jobInfo.pjColList.begin(); i != jobInfo.pjColList.end(); i++) + for (auto i = jobInfo.pjColList.begin(); i != jobInfo.pjColList.end(); i++) jobInfo.returnColSet.insert(i->key); // Strip constantbooleanquerySteps for (uint64_t i = 0; i < querySteps.size();) { - TupleConstantBooleanStep* bs = dynamic_cast(querySteps[i].get()); - ExpressionStep* es = dynamic_cast(querySteps[i].get()); + auto* bs = dynamic_cast(querySteps[i].get()); + auto* es = dynamic_cast(querySteps[i].get()); - if (bs != NULL) + if (bs != nullptr) { // cosntant step - if (bs->boolValue() == false) + if (!bs->boolValue()) jobInfo.constantFalse = true; querySteps.erase(querySteps.begin() + i); } - else if (es != NULL && es->tableKeys().size() == 0) + else if (es != nullptr && es->tableKeys().empty()) { // constant expression ParseTree* p = es->expressionFilter(); // filter - if (p != NULL) + if (p != nullptr) { Row r; // dummy row - if (funcexp::FuncExp::instance()->evaluate(r, p) == false) + if (!funcexp::FuncExp::instance()->evaluate(r, p)) jobInfo.constantFalse = true; querySteps.erase(querySteps.begin() + i); @@ -4582,7 +4589,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte { bool exist = false; - for (JobStepVector::iterator j = steps.begin(); j != steps.end() && !exist; ++j) + for (auto j = steps.begin(); j != steps.end() && !exist; ++j) { if (jobInfo.functionJoins[i] == j->get()) exist = true; @@ -4597,37 +4604,37 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte // Make sure each query step has an output DL // This is necessary for toString() method on most steps - for (JobStepVector::iterator it = steps.begin(); it != steps.end(); ++it) + for (auto& step: steps) { // if (dynamic_cast(it->get())) // continue; - if (it->get()->outputAssociation().outSize() == 0) + if (step->outputAssociation().outSize() == 0) { JobStepAssociation jsa; AnyDataListSPtr adl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - dl->OID(it->get()->oid()); + auto* dl = new RowGroupDL(1, jobInfo.fifoSize); + dl->OID(step->oid()); adl->rowGroupDL(dl); jsa.outAdd(adl); - it->get()->outputAssociation(jsa); + step->outputAssociation(jsa); } } // Populate the TableInfo map with the job steps keyed by table ID. JobStepVector joinSteps; JobStepVector& expSteps = jobInfo.crossTableExpressions; - JobStepVector::iterator it = querySteps.begin(); - JobStepVector::iterator end = querySteps.end(); + auto it = querySteps.begin(); + auto end = querySteps.end(); while (it != end) { // Separate table joins from other predicates. - TupleHashJoinStep* thjs = dynamic_cast(it->get()); - ExpressionStep* exps = dynamic_cast(it->get()); - SubAdapterStep* subs = dynamic_cast(it->get()); + auto* thjs = dynamic_cast(it->get()); + auto* exps = dynamic_cast(it->get()); + auto* subs = dynamic_cast(it->get()); - if (thjs != NULL && thjs->tupleId1() != thjs->tupleId2()) + if (thjs && thjs->tupleId1() != thjs->tupleId2()) { // simple column and constant column semi join if (thjs->tableOid2() == 0 && thjs->schema2().empty()) @@ -4685,8 +4692,8 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte // keep a join map pair tablePair(tid1, tid2); - TableJoinMap::iterator m1 = jobInfo.tableJoinMap.find(tablePair); - TableJoinMap::iterator m2 = jobInfo.tableJoinMap.end(); + auto m1 = jobInfo.tableJoinMap.find(tablePair); + auto m2 = jobInfo.tableJoinMap.end(); if (m1 == jobInfo.tableJoinMap.end()) { @@ -4782,17 +4789,17 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte m1->second.fJoinId = m2->second.fJoinId = thjs->joinId(); } // Separate the expressions - else if (exps != NULL && subs == NULL) + else if (exps && !subs) { const vector& tables = exps->tableKeys(); const vector& columns = exps->columnKeys(); bool tableInOuterQuery = false; set tableSet; // involved unique tables - for (uint64_t i = 0; i < tables.size(); ++i) + for (unsigned int table: tables) { - if (find(jobInfo.tableList.begin(), jobInfo.tableList.end(), tables[i]) != jobInfo.tableList.end()) - tableSet.insert(tables[i]); + if (find(jobInfo.tableList.begin(), jobInfo.tableList.end(), table) != jobInfo.tableList.end()) + tableSet.insert(table); else tableInOuterQuery = true; } @@ -4814,10 +4821,10 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte tableInfoMap[tables[i]].fProjectCols.push_back(c); jobInfo.pjColList.push_back(getTupleInfo(c, jobInfo)); jobInfo.returnColSet.insert(c); - const SimpleColumn* sc = dynamic_cast(exps->columns()[i]); + const auto* sc = dynamic_cast(exps->columns()[i]); - if (sc != NULL) - jobInfo.deliveredCols.push_back(SRCP(sc->clone())); + if (sc) + jobInfo.deliveredCols.emplace_back(sc->clone()); } } @@ -4831,8 +4838,8 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte // single table and not in join on clause uint32_t tid = tables[0]; - for (uint64_t i = 0; i < columns.size(); ++i) - tableInfoMap[tid].fColsInExp1.push_back(columns[i]); + for (unsigned int column : columns) + tableInfoMap[tid].fColsInExp1.push_back(column); tableInfoMap[tid].fOneTableExpSteps.push_back(*it); } @@ -4848,9 +4855,8 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte } // resolve after join: cross table or on clause conditions - for (uint64_t i = 0; i < columns.size(); ++i) + for (unsigned int cid : columns) { - uint32_t cid = columns[i]; uint32_t tid = getTableKey(jobInfo, cid); tableInfoMap[tid].fColsInExp2.push_back(cid); } @@ -4887,7 +4893,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte outjoinPredicateAdjust(tableInfoMap, jobInfo); // @bug4021, make sure there is real column to scan - for (TableInfoMap::iterator it = tableInfoMap.begin(); it != tableInfoMap.end(); it++) + for (auto it = tableInfoMap.begin(); it != tableInfoMap.end(); it++) { uint32_t tableUid = it->first; @@ -4895,8 +4901,8 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte continue; JobStepVector& steps = tableInfoMap[tableUid].fQuerySteps; - JobStepVector::iterator s = steps.begin(); - JobStepVector::iterator p = steps.end(); + auto s = steps.begin(); + auto p = steps.end(); for (; s != steps.end(); s++) { @@ -4910,7 +4916,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte if (s == steps.end()) { - map::iterator t = jobInfo.tableColMap.find(tableUid); + auto t = jobInfo.tableColMap.find(tableUid); if (t == jobInfo.tableColMap.end()) { @@ -4919,7 +4925,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte throw runtime_error(msg); } - SimpleColumn* sc = dynamic_cast(t->second.get()); + auto* sc = dynamic_cast(t->second.get()); CalpontSystemCatalog::OID oid = sc->oid(); CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); CalpontSystemCatalog::ColType ct = sc->colType(); @@ -4946,30 +4952,30 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte } // @bug3767, error out scalar subquery with aggregation and correlated additional comparison. - if (jobInfo.hasAggregation && (jobInfo.correlateSteps.size() > 0)) + if (jobInfo.hasAggregation && (!jobInfo.correlateSteps.empty())) { // expression filter - ExpressionStep* exp = NULL; + ExpressionStep* exp = nullptr; for (it = jobInfo.correlateSteps.begin(); it != jobInfo.correlateSteps.end(); it++) { - if (((exp = dynamic_cast(it->get())) != NULL) && (!exp->functionJoin())) + if (((exp = dynamic_cast(it->get())) != nullptr) && (!exp->functionJoin())) break; - exp = NULL; + exp = nullptr; } // correlated join step - TupleHashJoinStep* thjs = NULL; + TupleHashJoinStep* thjs = nullptr; for (it = jobInfo.correlateSteps.begin(); it != jobInfo.correlateSteps.end(); it++) { - if ((thjs = dynamic_cast(it->get())) != NULL) + if ((thjs = dynamic_cast(it->get())) != nullptr) break; } // @bug5202, error out not equal correlation and aggregation in subquery. - if ((exp != NULL) && (thjs != NULL) && (thjs->getJoinType() & CORRELATED)) + if (exp && thjs && (thjs->getJoinType() & CORRELATED)) throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_NEQ_AGG_SUB), ERR_NON_SUPPORT_NEQ_AGG_SUB); } @@ -4985,7 +4991,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte it++; } - for (TupleInfoVector::iterator j = jobInfo.pjColList.begin(); j != jobInfo.pjColList.end(); j++) + for (auto j = jobInfo.pjColList.begin(); j != jobInfo.pjColList.end(); j++) { if (jobInfo.keyInfo->tupleKeyVec[j->tkey].fId == CNX_EXP_TABLE_ID) continue; @@ -5000,9 +5006,9 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte for (it = retExp.begin(); it != retExp.end(); ++it) { - ExpressionStep* exp = dynamic_cast(it->get()); + auto* exp = dynamic_cast(it->get()); - if (exp == NULL) + if (exp == nullptr) throw runtime_error("Not an expression."); for (uint64_t i = 0; i < exp->columnKeys().size(); ++i) @@ -5023,7 +5029,7 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte TableInfoMap::iterator mit; for (mit = tableInfoMap.begin(); mit != tableInfoMap.end(); mit++) - if (combineJobStepsByTable(mit, jobInfo) == false) + if (!combineJobStepsByTable(mit, jobInfo)) throw runtime_error("combineJobStepsByTable failed."); // 2. join the combined steps together to form the spanning tree @@ -5031,9 +5037,9 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte joinTables(joinSteps, tableInfoMap, jobInfo, joinOrder, overrideLargeSideEstimate); // 3. put the steps together - for (vector::iterator i = joinOrder.begin(); i != joinOrder.end(); ++i) - querySteps.insert(querySteps.end(), tableInfoMap[*i].fQuerySteps.begin(), - tableInfoMap[*i].fQuerySteps.end()); + for (uint32_t i: joinOrder) + querySteps.insert(querySteps.end(), tableInfoMap[i].fQuerySteps.begin(), + tableInfoMap[i].fQuerySteps.end()); adjustLastStep(querySteps, deliverySteps, jobInfo); // to match the select clause } diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index e6c6b9796..13059e8b2 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -97,10 +97,10 @@ TupleHashJoinStep::TupleHashJoinStep(const JobInfo& jobInfo) fExtendedInfo = "THJS: "; joinType = INIT; joinThreadCount = resourceManager->getJlNumScanReceiveThreads(); - largeBPS = NULL; + largeBPS = nullptr; moreInput = true; fQtc.stepParms().stepType = StepTeleStats::T_HJS; - outputDL = NULL; + outputDL = nullptr; ownsOutputDL = false; djsSmallUsage = jobInfo.smallSideUsage; djsSmallLimit = jobInfo.smallSideLimit; diff --git a/mysql-test/columnstore/bugfixes/mcol-5651.result b/mysql-test/columnstore/bugfixes/mcol-5651.result new file mode 100644 index 000000000..f14843e2a --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-5651.result @@ -0,0 +1,63 @@ +DROP DATABASE IF EXISTS `mcol_5651`; +CREATE DATABASE `mcol_5651`; +USE `mcol_5651`; +CREATE TABLE mcs (id INTEGER NOT NULL, type VARCHAR(10) NOT NULL, sn VARCHAR(30), descr VARCHAR(100), sdate DATETIME) ENGINE=Columnstore; +INSERT INTO mcs VALUES +(1,'a','1',NULL,'2024-01-11 18:36:01'), +(2,'a','1',NULL,'2024-01-11 18:36:03'), +(3,'a','1',NULL,'2024-01-11 18:36:04'), +(4,'a','1',NULL,'2024-01-11 18:36:06'), +(5,'a','1',NULL,'2024-01-11 18:36:07'), +(6,'a','1',NULL,'2024-01-12 13:04:15'), +(7,'a','1',NULL,'2024-01-12 13:04:17'), +(8,'a','1',NULL,'2024-01-12 13:04:18'), +(9,'a','1',NULL,'2024-01-12 13:04:20'), +(10,'a','1',NULL,'2024-01-11 18:35:58'), +(11,'a','1',NULL,'2024-01-11 18:30:00'), +(12,'a','1',NULL,'2024-01-11 18:30:00'), +(13,'a','1',NULL,'2024-01-11 18:30:03'), +(14,'a','1',NULL,'2024-01-11 18:30:03'), +(15,'a','1','a','2024-01-11 18:30:02'), +(16,'a','1',NULL,'2024-01-11 18:30:03'), +(17,'a','1',NULL,'2024-01-11 18:30:03'), +(18,'a','1',NULL,'2024-01-11 18:30:03'), +(19,'a','1',NULL,'2024-01-12 18:53:02'), +(20,'a','1',NULL,'2024-01-12 18:53:02'), +(21,'a','1',NULL,'2024-01-12 19:27:47'), +(22,'a','1',NULL,'2024-01-12 19:27:48'), +(23,'a','1',NULL,'2024-01-13 01:47:26'), +(24,'a','1',NULL,'2024-01-13 01:47:26'), +(25,'a','1',NULL,'2024-01-13 01:47:26'), +(26,'a','1',NULL,'2024-01-13 01:47:26'), +(27,'a','1',NULL,'2024-01-13 01:47:26'), +(28,'a','1',NULL,'2024-01-13 01:47:26'); +SELECT s1.id, count(*) +FROM mcs AS s1 +WHERE s1.type = 'a' + AND s1.sdate BETWEEN '2026-01-05 16:21:00' - INTERVAL 24 MONTH AND '2026-01-05 16:21:00' + AND EXISTS ( +SELECT s.SN, s.sdate +FROM mcs AS s +WHERE s.type = 'a' + AND s.sdate BETWEEN '2026-01-05 16:21:00' - INTERVAL 24 MONTH AND '2026-01-05 16:21:00' + AND s.descr = 'a' + AND s.SN = s1.sn +AND s1.sdate BETWEEN s.sdate - INTERVAL 10 HOUR AND s.sdate + INTERVAL 10 HOUR +) +GROUP BY 1 ORDER BY 2,1; +id count(*) +1 1 +2 1 +3 1 +4 1 +5 1 +10 1 +11 1 +12 1 +13 1 +14 1 +15 1 +16 1 +17 1 +18 1 +DROP DATABASE `mcol_5651`; diff --git a/mysql-test/columnstore/bugfixes/mcol-5651.test b/mysql-test/columnstore/bugfixes/mcol-5651.test new file mode 100644 index 000000000..d91b8fe5d --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-5651.test @@ -0,0 +1,56 @@ +--source ../include/have_columnstore.inc +--disable_warnings +DROP DATABASE IF EXISTS `mcol_5651`; +--enable_warnings +CREATE DATABASE `mcol_5651`; +USE `mcol_5651`; + +CREATE TABLE mcs (id INTEGER NOT NULL, type VARCHAR(10) NOT NULL, sn VARCHAR(30), descr VARCHAR(100), sdate DATETIME) ENGINE=Columnstore; + +INSERT INTO mcs VALUES + (1,'a','1',NULL,'2024-01-11 18:36:01'), + (2,'a','1',NULL,'2024-01-11 18:36:03'), + (3,'a','1',NULL,'2024-01-11 18:36:04'), + (4,'a','1',NULL,'2024-01-11 18:36:06'), + (5,'a','1',NULL,'2024-01-11 18:36:07'), + (6,'a','1',NULL,'2024-01-12 13:04:15'), + (7,'a','1',NULL,'2024-01-12 13:04:17'), + (8,'a','1',NULL,'2024-01-12 13:04:18'), + (9,'a','1',NULL,'2024-01-12 13:04:20'), + (10,'a','1',NULL,'2024-01-11 18:35:58'), + (11,'a','1',NULL,'2024-01-11 18:30:00'), + (12,'a','1',NULL,'2024-01-11 18:30:00'), + (13,'a','1',NULL,'2024-01-11 18:30:03'), + (14,'a','1',NULL,'2024-01-11 18:30:03'), + (15,'a','1','a','2024-01-11 18:30:02'), + (16,'a','1',NULL,'2024-01-11 18:30:03'), + (17,'a','1',NULL,'2024-01-11 18:30:03'), + (18,'a','1',NULL,'2024-01-11 18:30:03'), + (19,'a','1',NULL,'2024-01-12 18:53:02'), + (20,'a','1',NULL,'2024-01-12 18:53:02'), + (21,'a','1',NULL,'2024-01-12 19:27:47'), + (22,'a','1',NULL,'2024-01-12 19:27:48'), + (23,'a','1',NULL,'2024-01-13 01:47:26'), + (24,'a','1',NULL,'2024-01-13 01:47:26'), + (25,'a','1',NULL,'2024-01-13 01:47:26'), + (26,'a','1',NULL,'2024-01-13 01:47:26'), + (27,'a','1',NULL,'2024-01-13 01:47:26'), + (28,'a','1',NULL,'2024-01-13 01:47:26'); + +SELECT s1.id, count(*) +FROM mcs AS s1 +WHERE s1.type = 'a' + AND s1.sdate BETWEEN '2026-01-05 16:21:00' - INTERVAL 24 MONTH AND '2026-01-05 16:21:00' + AND EXISTS ( + SELECT s.SN, s.sdate + FROM mcs AS s + WHERE s.type = 'a' + AND s.sdate BETWEEN '2026-01-05 16:21:00' - INTERVAL 24 MONTH AND '2026-01-05 16:21:00' + AND s.descr = 'a' + AND s.SN = s1.sn + AND s1.sdate BETWEEN s.sdate - INTERVAL 10 HOUR AND s.sdate + INTERVAL 10 HOUR + ) +GROUP BY 1 ORDER BY 2,1; + +# cleanup +DROP DATABASE `mcol_5651`; diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index e917756dc..57b8111c9 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -39,7 +39,7 @@ #include #include #include "serviceexemgr.h" -#include +#include using namespace std; #include @@ -148,7 +148,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() pp.setLogicalBlockMode(true); pp.setBlockPtr((int*)blockData); pp.setBlockPtrAux((int*)blockDataAux); - pthread_mutex_init(&objLock, NULL); + pthread_mutex_init(&objLock, nullptr); } BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, @@ -209,7 +209,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, pp.setBlockPtr((int*)blockData); pp.setBlockPtrAux((int*)blockDataAux); sendThread = bppst; - pthread_mutex_init(&objLock, NULL); + pthread_mutex_init(&objLock, nullptr); initBPP(b); } @@ -417,7 +417,6 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs) for (i = 0; i < joinerCount; i++) { smallSideRowLengths[i] = smallSideRGs[i].getRowSize(); - ; smallSideRowData[i] = RGData(smallSideRGs[i], tJoinerSizes[i]); smallSideRGs[i].setData(&smallSideRowData[i]); smallSideRGs[i].resetRowGroup(0); @@ -467,7 +466,7 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs) { hasFilterStep = true; - if (dynamic_cast(filterSteps[i].get()) != NULL) + if (dynamic_cast(filterSteps[i].get()) != nullptr) filtOnString = true; } else if (type == Command::DICT_STEP || type == Command::RID_TO_STRING) @@ -498,10 +497,9 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs) bs >> *(fAggregator.get()); // If there's UDAF involved, set up for PM processing - for (uint64_t i = 0; i < fAggregator->getAggFunctions().size(); i++) + for (const auto & pcol : fAggregator->getAggFunctions()) { - RowUDAFFunctionCol* rowUDAF = - dynamic_cast(fAggregator->getAggFunctions()[i].get()); + auto* rowUDAF = dynamic_cast(pcol.get()); if (rowUDAF) { @@ -553,10 +551,10 @@ void BatchPrimitiveProcessor::resetBPP(ByteStream& bs, const SP_UM_MUTEX& w, con ridMap = 0; baseRid = absRids[0] & 0xffffffffffffe000ULL; - for (uint32_t i = 0; i < ridCount; i++) + for (uint32_t j = 0; j < ridCount; j++) { - relRids[i] = absRids[i] - baseRid; - ridMap |= 1 << (relRids[i] >> 9); + relRids[j] = absRids[j] - baseRid; + ridMap |= 1 << (relRids[j] >> 9); } } else @@ -583,7 +581,7 @@ void BatchPrimitiveProcessor::resetBPP(ByteStream& bs, const SP_UM_MUTEX& w, con projectSteps[i]->resetCommand(bs); } - idbassert(bs.length() == 0); + idbassert(bs.empty()); /* init vars not part of the BS */ currentBlockOffset = 0; @@ -1098,7 +1096,7 @@ void BatchPrimitiveProcessor::initProcessor() } } - if (fAggregator.get() != NULL) + if (fAggregator.get() != nullptr) { fAggRowGroupData.reinit(fAggregateRG); fAggregateRG.setData(&fAggRowGroupData); @@ -1164,7 +1162,6 @@ uint32_t BatchPrimitiveProcessor::executeTupleJoin(uint32_t startRid, RowGroup& for (j = 0; j < joinerCount; j++) { bool found; - if (UNLIKELY(joinTypes[j] & ANTI)) { if (joinTypes[j] & WITHFCNEXP) @@ -1184,7 +1181,7 @@ uint32_t BatchPrimitiveProcessor::executeTupleJoin(uint32_t startRid, RowGroup& largeKey = oldRow.getIntField(colIndex); uint bucket = bucketPicker((char*)&largeKey, 8, bpSeed) & ptMask; - bool joinerIsEmpty = tJoiners[j][bucket]->empty() ? true : false; + bool joinerIsEmpty = tJoiners[j][bucket]->empty(); found = (tJoiners[j][bucket]->find(largeKey) != tJoiners[j][bucket]->end()); isNull = oldRow.isNullValue(colIndex); @@ -1221,8 +1218,8 @@ uint32_t BatchPrimitiveProcessor::executeTupleJoin(uint32_t startRid, RowGroup& { bool hasNull = false; - for (uint32_t z = 0; z < tlLargeSideKeyColumns[j].size(); z++) - if (oldRow.isNullValue(tlLargeSideKeyColumns[j][z])) + for (unsigned int column: tlLargeSideKeyColumns[j]) + if (oldRow.isNullValue(column)) { hasNull = true; break; @@ -1396,7 +1393,7 @@ void BatchPrimitiveProcessor::execute() { ColumnCommand* col = dynamic_cast(filterSteps[0].get()); - if ((col != NULL) && (col->getFilterCount() == 0) && (col->getLBID() != 0)) + if ((col != nullptr) && (col->getFilterCount() == 0) && (col->getLBID() != 0)) { // stored in last pos in relLBID[] and asyncLoaded[] uint64_t p = projectCount; @@ -2452,7 +2449,7 @@ SBPP BatchPrimitiveProcessor::duplicate() for (i = 0; i < projectCount; ++i) bpp->projectSteps[i] = projectSteps[i]->duplicate(); - if (fAggregator.get() != NULL) + if (fAggregator.get() != nullptr) { bpp->fAggregateRG = fAggregateRG; bpp->fAggregator.reset(new RowAggregation(fAggregator->getGroupByCols(), fAggregator->getAggFunctions())); @@ -2551,7 +2548,7 @@ void BatchPrimitiveProcessor::asyncLoadProjectColumns() // only care about column commands ColumnCommand* col = dynamic_cast(projectSteps[i].get()); - if (col != NULL) + if (col != nullptr) { asyncLoaded[i] = asyncLoaded[i] && (relLBID[i] % blocksReadAhead != 0); relLBID[i] += col->getWidth(); @@ -2706,12 +2703,14 @@ inline void BatchPrimitiveProcessor::getJoinResults(const Row& r, uint32_t jInde { bool hasNullValue = false; - for (uint32_t i = 0; i < tlLargeSideKeyColumns[jIndex].size(); i++) - if (r.isNullValue(tlLargeSideKeyColumns[jIndex][i])) + for (unsigned int column: tlLargeSideKeyColumns[jIndex]) + { + if (r.isNullValue(column)) { hasNullValue = true; break; } + } if (hasNullValue) { diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 0db8f88db..bd7046efa 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -599,7 +599,7 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin if (UNLIKELY(inUM() && (joinType & MATCHNULLS) && !isNull && !typelessJoin)) { - if (largeRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) + if (largeRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE && ld) { uint bucket = bucketPicker((char*)&(joblist::LONGDOUBLENULL), sizeof(joblist::LONGDOUBLENULL), bpSeed) & bucketMask; @@ -608,7 +608,7 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin for (; range.first != range.second; ++range.first) matches->push_back(range.first->second); } - else if (!largeRG.usesStringTable()) + else if (!smallRG.usesStringTable()) { auto nullVal = getJoinNullValue(); uint bucket = bucketPicker((char*)&nullVal, sizeof(nullVal), bpSeed) & bucketMask; From 8ae5a3da40910a70a73a7236c62a94fb63531fcd Mon Sep 17 00:00:00 2001 From: drrtuy Date: Sat, 9 Nov 2024 19:44:02 +0000 Subject: [PATCH 13/65] Fix/mcol 5787 rgdata buffer max size dev (#3325) * fix(rowgroup): RGData now uses uint64_t counter for the fixed sizes columns data buf. The buffer can utilize > 4GB RAM that is necessary for PM side join. RGData ctor uses uint32_t allocating data buffer. This fact causes implicit heap overflow. * feat(bytestream,serdes): BS buffer size type is uint64_t This necessary to handle 64bit RGData, that comes as a separate patch. The pair of patches would allow to have PM joins when SmallSide size > 4GB. * feat(bytestream,serdes): Distribute BS buf size data type change to avoid implicit data type narrowing * feat(rowgroup): this returns bits lost during cherry-pick. The bits lost caused the first RGData::serialize to crash a process --- dbcon/joblist/batchprimitiveprocessor-jl.cpp | 55 +- dbcon/joblist/batchprimitiveprocessor-jl.h | 1 - dbcon/joblist/lbidlist.cpp | 3 +- dbcon/joblist/primitivemsg.h | 2 + dbcon/joblist/resourcemanager.cpp | 1 + dbcon/joblist/rowestimator.cpp | 4 +- dbcon/joblist/tuplehashjoin.cpp | 1 + dbcon/mysql/ha_mcs_execplan.cpp | 4 +- primitives/primproc/columncommand.h | 2 + primitives/primproc/dictstep.h | 2 + tests/CMakeLists.txt | 4 + tests/bytestream.cpp | 922 +++++++++++++++++++ utils/cloudio/SocketPool.cpp | 5 +- utils/joiner/joinpartition.cpp | 3 +- utils/messageqcpp/bytestream.cpp | 78 +- utils/messageqcpp/bytestream.h | 46 +- utils/messageqcpp/compressed_iss.cpp | 6 +- utils/messageqcpp/inetstreamsocket.cpp | 3 + utils/rowgroup/rowgroup.cpp | 103 +-- utils/rowgroup/rowgroup.h | 33 +- utils/rowgroup/rowstorage.cpp | 3 +- utils/windowfunction/idborderby.cpp | 2 +- versioning/BRM/slavecomm.cpp | 6 +- writeengine/server/we_dataloader.cpp | 2 +- writeengine/server/we_dataloader.h | 49 +- writeengine/splitter/we_sdhandler.cpp | 2 +- writeengine/splitter/we_splclient.cpp | 4 +- writeengine/splitter/we_splclient.h | 15 +- 28 files changed, 1130 insertions(+), 231 deletions(-) create mode 100644 tests/bytestream.cpp diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 7d94daa8f..1163661e8 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -72,7 +72,6 @@ BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm) , LBIDTrace(false) , tupleLength(0) , status(0) - , sendRowGroups(false) , valueColumn(0) , sendTupleJoinRowGroupData(false) , bop(BOP_AND) @@ -147,7 +146,7 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pDictionaryStep& step) tableOID = step.tableOid(); - if (filterCount == 0 && !sendRowGroups) + if (filterCount == 0) { sendAbsRids = true; sendValues = true; @@ -244,7 +243,7 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& step) if (utils::isWide(cc->getWidth())) wideColumnsWidths |= cc->getWidth(); - if (filterCount == 0 && !sendRowGroups) + if (filterCount == 0) sendValues = true; idbassert(sessionID == step.sessionId()); @@ -283,7 +282,7 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& p, const pDic projectCount++; needStrValues = true; - if (filterCount == 0 && !sendRowGroups) + if (filterCount == 0) { sendValues = true; sendAbsRids = true; @@ -1054,9 +1053,6 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const if (tJoiners.size() > 0) flags |= HAS_JOINER; - if (sendRowGroups) - flags |= HAS_ROWGROUP; - if (sendTupleJoinRowGroupData) flags |= JOIN_ROWGROUP_DATA; @@ -1071,12 +1067,6 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const bs << bop; bs << (uint8_t)(forHJ ? 1 : 0); - if (sendRowGroups) - { - bs << valueColumn; - bs << inputRG; - } - if (ot == ROW_GROUP) { bs << projectionRG; @@ -1248,6 +1238,7 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const * (projection count)x run msgs for projection Commands */ +// The deser counterpart function is BPP::resetBPP void BatchPrimitiveProcessorJL::runBPP(ByteStream& bs, uint32_t pmNum, bool isExeMgrDEC) { ISMPacketHeader ism; @@ -1289,35 +1280,28 @@ void BatchPrimitiveProcessorJL::runBPP(ByteStream& bs, uint32_t pmNum, bool isEx bs << sentByEM; if (_hasScan) + { idbassert(ridCount == 0); - else if (!sendRowGroups) + } + else + { idbassert(ridCount > 0 && (ridMap != 0 || sendAbsRids)); - else - idbassert(inputRG.getRowCount() > 0); - - if (sendRowGroups) - { - uint32_t rgSize = inputRG.getDataSize(); - bs << rgSize; - bs.append(inputRG.getData(), rgSize); } + + bs << ridCount; + + if (sendAbsRids) + bs.append((uint8_t*)absRids.get(), ridCount << 3); else { - bs << ridCount; - - if (sendAbsRids) - bs.append((uint8_t*)absRids.get(), ridCount << 3); - else - { - bs << ridMap; - bs << baseRid; - bs.append((uint8_t*)relRids, ridCount << 1); - } - - if (sendValues) - bs.append((uint8_t*)values, ridCount << 3); + bs << ridMap; + bs << baseRid; + bs.append((uint8_t*)relRids, ridCount << 1); } + if (sendValues) + bs.append((uint8_t*)values, ridCount << 3); + for (i = 0; i < filterCount; i++) filterSteps[i]->runCommand(bs); @@ -1667,7 +1651,6 @@ void BatchPrimitiveProcessorJL::setJoinedRowGroup(const rowgroup::RowGroup& rg) void BatchPrimitiveProcessorJL::setInputRowGroup(const rowgroup::RowGroup& rg) { - sendRowGroups = true; sendAbsRids = false; sendValues = false; inputRG = rg; diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index 365ea4ad0..2b9e400b0 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -343,7 +343,6 @@ class BatchPrimitiveProcessorJL /* for RowGroup return type */ rowgroup::RowGroup inputRG, projectionRG; - bool sendRowGroups; uint32_t valueColumn; /* for PM Aggregation */ diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 34a5f23d5..ac97e49af 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -20,6 +20,7 @@ * ******************************************************************************/ #include +#include "bytestream.h" #include "primitivemsg.h" #include "blocksize.h" #include "lbidlist.h" @@ -700,7 +701,7 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP, bool isDict) { - int length = bs->length(), pos = 0; + messageqcpp::BSSizeType length = bs->length(), pos = 0; const char* MsgDataPtr = (const char*)bs->buf(); bool scan = true; int64_t value = 0; diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index 479e1535a..f06a7e99c 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -282,6 +282,8 @@ struct ISMPacketHeader uint32_t Interleave; uint16_t Flags; uint8_t Command; + // !!! This attribute is used to store a sum which arg type is potentially uint64_t. + // As of 23.02.10 uint32_t here is always enough for the purpose of this attribute though. uint16_t Size; unsigned Type : 4; unsigned MsgCount : 4; diff --git a/dbcon/joblist/resourcemanager.cpp b/dbcon/joblist/resourcemanager.cpp index e162f1a1f..e61da3c4d 100644 --- a/dbcon/joblist/resourcemanager.cpp +++ b/dbcon/joblist/resourcemanager.cpp @@ -368,6 +368,7 @@ bool ResourceManager::getMemory(int64_t amount, boost::shared_ptr& sess return (ret1 && ret2); } // Don't care about session memory +// The amount type is unsafe if amount close to max that is unrealistic in 2024. bool ResourceManager::getMemory(int64_t amount, bool patience) { bool ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0); diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index fa7e8920c..1b2b2f18a 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -19,6 +19,7 @@ * $Id: rowestimator.cpp 5642 2009-08-10 21:04:59Z wweeks $ * ******************************************************************************/ +#include #include #include "primitivemsg.h" #include "blocksize.h" @@ -292,7 +293,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, const m // For example, there are two operations for "col1 > 5 and col1 < 10": // 1) col1 > 5 // 2) col2 < 10 - int length = bs->length(), pos = 0; + messageqcpp::BSSizeType length = bs->length(), pos = 0; const char* msgDataPtr = (const char*)bs->buf(); int64_t value = 0; int128_t bigValue = 0; @@ -301,6 +302,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, const m for (int i = 0; i < comparisonLimit; i++) { + assert(ct.colWidth >= 0); pos += ct.colWidth + 2; // predicate + op + lcf // TODO: Stole this condition from lbidlist. diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 13059e8b2..6e6a60c3d 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -417,6 +417,7 @@ void TupleHashJoinStep::smallRunnerFcn(uint32_t index, uint threadID, uint64_t* smallRG.initRow(&r); try { + // Very unfortunate choice for the type b/c of RM::getMemory type. ssize_t rgSize; bool gotMem; goto next; diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 8616cd4d1..6ac70dc77 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4300,7 +4300,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (from_tzinfo) { serializeTimezoneInfo(bs, from_tzinfo); - uint32_t length = bs.length(); + messageqcpp::BSSizeType length = bs.length(); uint8_t* buf = new uint8_t[length]; bs >> buf; tzinfo = string((char*)buf, length); @@ -4312,7 +4312,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (to_tzinfo) { serializeTimezoneInfo(bs, to_tzinfo); - uint32_t length = bs.length(); + messageqcpp::BSSizeType length = bs.length(); uint8_t* buf = new uint8_t[length]; bs >> buf; tzinfo = string((char*)buf, length); diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index cc08f6edb..be99261c4 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -172,6 +172,8 @@ class ColumnCommand : public Command // the length of base prim msg, which is everything up to the // rid array for the pCol message + // !!! This attribute is used to store a sum which arg type is potentially uint64_t. + // As of 23.02.10 uint32_t here is always enough for the purpose of this attribute though. uint32_t baseMsgLength; uint64_t lbid; diff --git a/primitives/primproc/dictstep.h b/primitives/primproc/dictstep.h index 5200b80c6..9c5120b98 100644 --- a/primitives/primproc/dictstep.h +++ b/primitives/primproc/dictstep.h @@ -151,6 +151,8 @@ class DictStep : public Command int compressionType; messageqcpp::ByteStream filterString; uint32_t filterCount; + // !!! This attribute is used to store a sum which arg type is potentially uint64_t. + // As of 23.02.10 uint32_t here is always enough for the purpose of this attribute though. uint32_t bufferSize; uint32_t charsetNumber; uint16_t inputRidCount; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 079757126..b7632f301 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -80,6 +80,10 @@ if (WITH_UNITTESTS) target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) add_test(NAME columnstore:comparators_tests COMMAND comparators_tests) + add_executable(bytestream bytestream.cpp) + target_link_libraries(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + add_test(NAME columnstore:bytestream COMMAND bytestream) + # standalone EM routines test add_executable(brm_em_standalone brm-em-standalone.cpp) target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) diff --git a/tests/bytestream.cpp b/tests/bytestream.cpp new file mode 100644 index 000000000..76e60826b --- /dev/null +++ b/tests/bytestream.cpp @@ -0,0 +1,922 @@ +/* Copyright (C) 2024 MariaDB Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#include +#include + +using namespace std; +#include +#include +#include +#include +#include + +#include + +#include "bytestream.h" +using namespace messageqcpp; +#include "configcpp.h" +using namespace config; +#include "mcs_decimal.h" + +class ByteStreamTestSuite : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(ByteStreamTestSuite); + + CPPUNIT_TEST(bs_1); + CPPUNIT_TEST(bs_1_1); + CPPUNIT_TEST(bs_1_2); + CPPUNIT_TEST(bs_2); + CPPUNIT_TEST(bs_3); + CPPUNIT_TEST(bs_4); + CPPUNIT_TEST_EXCEPTION(bs_5_1, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_2, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_3, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_4, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_5, std::underflow_error); + CPPUNIT_TEST_EXCEPTION(bs_5_6, std::underflow_error); + CPPUNIT_TEST(bs_6); + CPPUNIT_TEST(bs_7); + CPPUNIT_TEST(bs_8); + CPPUNIT_TEST_EXCEPTION(bs_9, std::underflow_error); + CPPUNIT_TEST(bs_10); + CPPUNIT_TEST(bs_12); + CPPUNIT_TEST(bs_13); + CPPUNIT_TEST(bs_14); + CPPUNIT_TEST(bs_15); + CPPUNIT_TEST(bs_16); + CPPUNIT_TEST_SUITE_END(); + + private: + ByteStream::byte b; + ByteStream::doublebyte d; + ByteStream::quadbyte q; + ByteStream::octbyte o; + + uint8_t u8; + uint16_t u16; + uint32_t u32; + uint64_t u64; + uint128_t u128; + int8_t i8; + int16_t i16; + int32_t i32; + int64_t i64; + int128_t i128; + + ByteStream bs; + ByteStream bs1; + + ByteStream::byte* bap; + ByteStream::byte* bap1; + + int len; + + public: + void setUp() + { + bs.reset(); + bs1.reset(); + bap = 0; + bap1 = 0; + } + + void tearDown() + { + bs.reset(); + bs1.reset(); + delete[] bap; + bap = 0; + delete[] bap1; + bap1 = 0; + } + + void bs_1() + { + bs.reset(); + + o = 0xdeadbeefbadc0ffeLL; + bs << o; + CPPUNIT_ASSERT(bs.length() == 8); + o = 0; + bs >> o; + CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 0); + + q = 0xdeadbeef; + bs << q; + CPPUNIT_ASSERT(bs.length() == 4); + q = 0; + bs >> q; + CPPUNIT_ASSERT(q == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 0); + + d = 0xf00f; + bs << d; + CPPUNIT_ASSERT(bs.length() == 2); + d = 0; + bs >> d; + CPPUNIT_ASSERT(d == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 0); + + b = 0x0f; + bs << b; + CPPUNIT_ASSERT(bs.length() == 1); + b = 0; + bs >> b; + CPPUNIT_ASSERT(b == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + + o = 0xdeadbeefbadc0ffeLL; + bs << o; + CPPUNIT_ASSERT(bs.length() == 8); + o = 0; + + q = 0xdeadbeef; + bs << q; + CPPUNIT_ASSERT(bs.length() == 12); + q = 0; + + d = 0xf00f; + bs << d; + CPPUNIT_ASSERT(bs.length() == 14); + d = 0; + + b = 0x0f; + bs << b; + CPPUNIT_ASSERT(bs.length() == 15); + b = 0; + + bs >> o; + CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 7); + bs >> q; + CPPUNIT_ASSERT(q == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 3); + bs >> d; + CPPUNIT_ASSERT(d == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 1); + bs >> b; + CPPUNIT_ASSERT(b == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_1_1() + { + bs.reset(); + + o = 0xdeadbeefbadc0ffeLL; + bs << o; + CPPUNIT_ASSERT(bs.length() == 8); + o = 0; + + q = 0xdeadbeef; + bs << q; + CPPUNIT_ASSERT(bs.length() == 12); + q = 0; + + d = 0xf00f; + bs << d; + CPPUNIT_ASSERT(bs.length() == 14); + d = 0; + + b = 0x0f; + bs << b; + CPPUNIT_ASSERT(bs.length() == 15); + b = 0; + + ByteStream bbs1; + bbs1 << bs; + CPPUNIT_ASSERT(bbs1.length() == bs.length() + sizeof(messageqcpp::BSSizeType)); + bs.reset(); + bbs1 >> bs; + CPPUNIT_ASSERT(bbs1.length() == 0); + CPPUNIT_ASSERT(bs.length() == 15); + + bs >> o; + CPPUNIT_ASSERT(o == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 7); + bs >> q; + CPPUNIT_ASSERT(q == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 3); + bs >> d; + CPPUNIT_ASSERT(d == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 1); + bs >> b; + CPPUNIT_ASSERT(b == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_1_2() + { + bs.reset(); + + i64 = -2401053089477160962; + bs << i64; + CPPUNIT_ASSERT(bs.length() == 8); + i64 = 0; + + i32 = -559038737; + bs << i32; + CPPUNIT_ASSERT(bs.length() == 12); + i32 = 0; + + i16 = -4081; + bs << i16; + CPPUNIT_ASSERT(bs.length() == 14); + i16 = 0; + + i8 = 15; + bs << i8; + CPPUNIT_ASSERT(bs.length() == 15); + i8 = 0; + + bs >> i64; + CPPUNIT_ASSERT(i64 == -2401053089477160962); + CPPUNIT_ASSERT(bs.length() == 7); + + bs >> i32; + CPPUNIT_ASSERT(i32 == -559038737); + CPPUNIT_ASSERT(bs.length() == 3); + + bs >> i16; + CPPUNIT_ASSERT(i16 == -4081); + CPPUNIT_ASSERT(bs.length() == 1); + + bs >> i8; + CPPUNIT_ASSERT(i8 == 15); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_2() + { + int i; + + bs.reset(); + srand(time(0)); + + for (i = 0; i < 10240; i++) + { + bs << (uint32_t)rand(); + } + + bs1 = bs; + + uint32_t q1; + + for (i = 0; i < 10240; i++) + { + bs >> u32; + bs1 >> q1; + CPPUNIT_ASSERT(u32 == q1); + } + + bs.reset(); + bs1.reset(); + } + + void bs_3() + { + uint8_t ba[1024] = { + 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0, + }; + + bs.load(ba, 8); + CPPUNIT_ASSERT(bs.length() == 8); + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x12); + bs >> u16; + CPPUNIT_ASSERT(u16 == 0x5634); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdebc9a78); + + CPPUNIT_ASSERT(bs.length() == 1); + + bs.reset(); + CPPUNIT_ASSERT(bs.length() == 0); + + bs.load(ba, 8); + len = bs.length(); + CPPUNIT_ASSERT(len == 8); + bap = new ByteStream::byte[len]; + // bs >> bap; + memcpy(bap, bs.buf(), len); + CPPUNIT_ASSERT(memcmp(ba, bap, len) == 0); + delete[] bap; + bap = 0; + + bs.reset(); + + for (u32 = 0; u32 < 20480; u32++) + { + bs << u32; + } + + len = bs.length(); + CPPUNIT_ASSERT(len == (20480 * sizeof(u32))); + bap = new ByteStream::byte[len]; + // bs >> bap; + memcpy(bap, bs.buf(), len); + + bs.reset(); + + for (u32 = 0; u32 < 20480; u32++) + { + bs << u32; + } + + len = bs.length(); + CPPUNIT_ASSERT(len == (20480 * sizeof(q))); + bap1 = new ByteStream::byte[len]; + // bs >> bap1; + memcpy(bap1, bs.buf(), len); + + CPPUNIT_ASSERT(memcmp(bap1, bap, len) == 0); + + delete[] bap; + bap = 0; + delete[] bap1; + bap1 = 0; + bs.reset(); + } + void bs_4() + { + for (i32 = 0; i32 < 20480; i32++) + { + bs << i32; + } + + ByteStream bs2(bs); + len = bs2.length(); + CPPUNIT_ASSERT(len == (20480 * sizeof(i32))); + bap = new ByteStream::byte[len]; + // bs2 >> bap; + memcpy(bap, bs2.buf(), len); + + bs1 = bs2; + len = bs1.length(); + CPPUNIT_ASSERT(len == (20480 * sizeof(i32))); + bap1 = new ByteStream::byte[len]; + // bs1 >> bap1; + memcpy(bap1, bs1.buf(), len); + + CPPUNIT_ASSERT(memcmp(bap1, bap, len) == 0); + delete[] bap; + bap = 0; + delete[] bap1; + bap1 = 0; + bs.reset(); + bs1.reset(); + bs2.reset(); + } + + void bs_5_1() + { + bs.reset(); + + u8 = 0x0f; + bs << u8; + + for (;;) + bs >> u32; + } + + void bs_5_2() + { + bs.reset(); + + u8 = 0x0f; + bs << u8; + + for (;;) + bs >> u16; + } + + void bs_5_3() + { + bs.reset(); + + u8 = 0x0f; + bs << u8; + + for (;;) + bs >> u8; + } + + void bs_5_4() + { + bs.reset(); + + i8 = 0x0f; + bs << i8; + + for (;;) + bs >> i32; + } + + void bs_5_5() + { + bs.reset(); + + i8 = 0x0f; + bs << i8; + + for (;;) + bs >> i16; + } + + void bs_5_6() + { + bs.reset(); + + i8 = 0x0f; + bs << i8; + + for (;;) + bs >> i8; + } + + void bs_6() + { + u8 = 0x1a; + bs << u8; + u8 = 0x2b; + bs << u8; + u8 = 0x3c; + bs << u8; + + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x1a); + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x2b); + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x3c); + + bs.reset(); + + u8 = 12; + bs << u8; + u8 = 3; + bs << u8; + u8 = 0; + bs << u8; + u8 = 2; + bs << u8; + + ByteStream bs3(bs); + + bs3 >> u8; + CPPUNIT_ASSERT(u8 == 12); + bs3 >> u8; + CPPUNIT_ASSERT(u8 == 3); + bs3 >> u8; + CPPUNIT_ASSERT(u8 == 0); + bs3 >> u8; + CPPUNIT_ASSERT(u8 == 2); + } + + void bs_7() + { + size_t i; + + bs.reset(); + bap = new ByteStream::byte[ByteStream::BlockSize * 2]; + ByteStream::byte* bapp; + + for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) + *bapp = 0xa5; + + bs.append(bap, ByteStream::BlockSize); + CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 1)); + + for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) + *bapp = 0x5a; + + bs.append(bap, ByteStream::BlockSize); + CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 2)); + + for (bapp = &bap[0], i = 0; i < ByteStream::BlockSize * 2; bapp++, i++) + *bapp = 0x55; + + bs.append(bap, ByteStream::BlockSize * 2); + CPPUNIT_ASSERT(bs.length() == (ByteStream::BlockSize * 4)); + delete[] bap; + bap = new ByteStream::byte[bs.length()]; + // bs >> bap; + memcpy(bap, bs.buf(), bs.length()); + bap1 = new ByteStream::byte[bs.length()]; + + for (bapp = &bap1[0], i = 0; i < ByteStream::BlockSize; bapp++, i++) + *bapp = 0xa5; + + for (i = 0; i < ByteStream::BlockSize; bapp++, i++) + *bapp = 0x5a; + + for (i = 0; i < ByteStream::BlockSize * 2; bapp++, i++) + *bapp = 0x55; + + CPPUNIT_ASSERT(memcmp(bap, bap1, bs.length()) == 0); + delete[] bap; + bap = 0; + delete[] bap1; + bap1 = 0; + } + + std::string getString() + { + static const std::string s( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore " + "et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut " + "aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse " + "cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in " + "culpa qui officia deserunt mollit anim id est laborum."); + return s; + } + + void bs_8() + { + bs.reset(); + string s; + s = "This is a test"; + bs << s; + string s1; + bs >> s1; + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(bs.length() == 0); + + bs.reset(); + s = getString(); + bs << s; + bs >> s1; + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(bs.length() == 0); + + u8 = 0xa5; + bs << u8; + u16 = 0x5aa5; + bs << u16; + u32 = 0xdeadbeef; + bs << u32; + bs << s; + s += s1; + bs << s; + s += s1; + bs << s; + bs << u32; + bs << u16; + bs << u8; + + bs >> u8; + CPPUNIT_ASSERT(u8 == 0xa5); + bs >> u16; + CPPUNIT_ASSERT(u16 == 0x5aa5); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + bs >> s; + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(s.length() == (s1.length() * 1)); + bs >> s; + CPPUNIT_ASSERT(s.length() == (s1.length() * 2)); + bs >> s; + CPPUNIT_ASSERT(s.length() == (s1.length() * 3)); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + bs >> u16; + CPPUNIT_ASSERT(u16 == 0x5aa5); + bs >> u8; + CPPUNIT_ASSERT(u8 == 0xa5); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_9() + { + bs.reset(); + // Load up a bogus string (too short) + u32 = 100; + bs << u32; + bs.append(reinterpret_cast("This is a test"), 14); + string s; + // Should throw underflow + bs >> s; + } + + void bs_10() + { + bs.reset(); + bs1.reset(); + u32 = 0xdeadbeef; + bs << u32; + CPPUNIT_ASSERT(bs.length() == 4); + CPPUNIT_ASSERT(bs1.length() == 0); + bs.swap(bs1); + CPPUNIT_ASSERT(bs1.length() == 4); + CPPUNIT_ASSERT(bs.length() == 0); + bs1 >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + + bs.reset(); + bs1.reset(); + u32 = 0xdeadbeef; + bs << u32; + bs1 << u32; + bs += bs1; + CPPUNIT_ASSERT(bs.length() == 8); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + + bs.reset(); + bs1.reset(); + ByteStream bs2; + u32 = 0xdeadbeef; + bs1 << u32; + bs2 << u32; + bs = bs1 + bs2; + CPPUNIT_ASSERT(bs.length() == 8); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + } + + void bs_12() + { + bs.reset(); + + i128 = 10 * 100000000000000000000000000000000000000_xxl; + bs << i128; + CPPUNIT_ASSERT(bs.length() == 16); + i128 = 0; + bs >> i128; + CPPUNIT_ASSERT(i128 == static_cast(10 * 100000000000000000000000000000000000000_xxl)); + CPPUNIT_ASSERT(bs.length() == 0); + + u128 = 10 * 100000000000000000000000000000000000000_xxl; + bs << u128; + CPPUNIT_ASSERT(bs.length() == 16); + u128 = 0; + bs >> u128; + CPPUNIT_ASSERT(u128 == 10 * 100000000000000000000000000000000000000_xxl); + CPPUNIT_ASSERT(bs.length() == 0); + + u64 = 0xdeadbeefbadc0ffeLL; + bs << u64; + CPPUNIT_ASSERT(bs.length() == 8); + u64 = 0; + bs.peek(u64); + CPPUNIT_ASSERT(u64 == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 8); + u64 = 0; + bs >> u64; + CPPUNIT_ASSERT(u64 == 0xdeadbeefbadc0ffeLL); + CPPUNIT_ASSERT(bs.length() == 0); + + u16 = 0xf00f; + bs << u16; + CPPUNIT_ASSERT(bs.length() == 2); + u16 = 0; + bs.peek(u16); + CPPUNIT_ASSERT(u16 == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 2); + u16 = 0; + bs >> u16; + CPPUNIT_ASSERT(u16 == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 0); + + u8 = 0x0f; + bs << u8; + CPPUNIT_ASSERT(bs.length() == 1); + u8 = 0; + bs.peek(u8); + CPPUNIT_ASSERT(u8 == 0x0f); + CPPUNIT_ASSERT(bs.length() == 1); + u8 = 0; + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + + u32 = 0xdeadbeef; + bs << u32; + CPPUNIT_ASSERT(bs.length() == 4); + u32 = 0; + + u16 = 0xf00f; + bs << u16; + CPPUNIT_ASSERT(bs.length() == 6); + u16 = 0; + + u8 = 0x0f; + bs << u8; + CPPUNIT_ASSERT(bs.length() == 7); + u8 = 0; + + bs.peek(u32); + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 7); + u32 = 0; + bs >> u32; + CPPUNIT_ASSERT(u32 == 0xdeadbeef); + CPPUNIT_ASSERT(bs.length() == 3); + u16 = 0; + bs.peek(u16); + CPPUNIT_ASSERT(u16 == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 3); + u16 = 0; + bs >> u16; + CPPUNIT_ASSERT(u16 == 0xf00f); + CPPUNIT_ASSERT(bs.length() == 1); + u8 = 0; + bs.peek(u8); + CPPUNIT_ASSERT(u8 == 0x0f); + CPPUNIT_ASSERT(bs.length() == 1); + u8 = 0; + bs >> u8; + CPPUNIT_ASSERT(u8 == 0x0f); + CPPUNIT_ASSERT(bs.length() == 0); + + string s; + s = "This is a test"; + bs << s; + string s1; + bs.peek(s1); + CPPUNIT_ASSERT(s == s1); + CPPUNIT_ASSERT(bs.length() == s1.size() + 4); + CPPUNIT_ASSERT(!s1.empty()); + string s2; + bs >> s2; + CPPUNIT_ASSERT(s == s2); + CPPUNIT_ASSERT(bs.length() == 0); + } + + void bs_13() + { + bs.reset(); + std::string s = getString(); + bs << s; + ofstream of("bs_13.dat"); + of << bs; + of.close(); + + ifstream ifs; + ifs.open("./bs_13.dat"); + ifs.seekg(0, ios::end); + size_t ifs_len1 = ifs.tellg(); + // will be longer than orig file because string length is encoded into stream + CPPUNIT_ASSERT((s.size() + sizeof(ByteStream::quadbyte)) == ifs_len1); + ifs.seekg(0, ios::beg); + std::unique_ptr buf1(new char[ifs_len1]); + bs1.reset(); + ifs >> bs1; + ifs.close(); + CPPUNIT_ASSERT(bs.length() == bs1.length()); + string s1; + bs1 >> s1; + CPPUNIT_ASSERT(s == s1); + } + + void bs_14() + { + ByteStream bs1(0); + ByteStream bs2(bs1); + CPPUNIT_ASSERT(bs2.fBuf == 0); + ByteStream bs3(0); + bs3 = bs1; + CPPUNIT_ASSERT(bs3.fBuf == 0); + } + + void bs_15() + { + ByteStream b1, b2, empty; + uint8_t u8; + + CPPUNIT_ASSERT(b1 == b2); + CPPUNIT_ASSERT(b2 == b1); + CPPUNIT_ASSERT(b2 == empty); + CPPUNIT_ASSERT(b1 == empty); + + CPPUNIT_ASSERT(!(b1 != b2)); + CPPUNIT_ASSERT(!(b2 != b1)); + CPPUNIT_ASSERT(!(b2 != empty)); + CPPUNIT_ASSERT(!(b1 != empty)); + + b1 << "Woo hoo"; + + CPPUNIT_ASSERT(b1 != b2); + CPPUNIT_ASSERT(b2 != b1); + CPPUNIT_ASSERT(b1 != empty); + + CPPUNIT_ASSERT(!(b1 == b2)); + CPPUNIT_ASSERT(!(b2 == b1)); + CPPUNIT_ASSERT(!(b1 == empty)); + + b2 << "Woo hoo"; + + CPPUNIT_ASSERT(b1 == b2); + CPPUNIT_ASSERT(b2 == b1); + CPPUNIT_ASSERT(!(b1 != b2)); + CPPUNIT_ASSERT(!(b2 != b1)); + + b1 >> u8; + + CPPUNIT_ASSERT(b1 != b2); + CPPUNIT_ASSERT(b2 != b1); + CPPUNIT_ASSERT(!(b1 == b2)); + CPPUNIT_ASSERT(!(b2 == b1)); + + b1 << u8; + + CPPUNIT_ASSERT(b1 != b2); + CPPUNIT_ASSERT(b2 != b1); + CPPUNIT_ASSERT(!(b1 == b2)); + CPPUNIT_ASSERT(!(b2 == b1)); + + b2 >> u8; + b2 << u8; + + CPPUNIT_ASSERT(b1 == b2); + CPPUNIT_ASSERT(b2 == b1); + CPPUNIT_ASSERT(!(b1 != b2)); + CPPUNIT_ASSERT(!(b2 != b1)); + } + + void bs_16() + { + int i; + uint32_t len; + + bs.reset(); + srand(time(0)); + + for (i = 0; i < 10240; i++) + { + bs << (ByteStream::quadbyte)rand(); + } + + std::unique_ptr bp(new ByteStream::byte[bs.length()]); + ByteStream::byte* bpp = bp.get(); + std::unique_ptr bp1(new ByteStream::byte[bs.length()]); + ByteStream::byte* bpp1 = bp1.get(); + + len = bs.length(); + CPPUNIT_ASSERT(len == 10240 * 4); + bs.peek(bpp); + CPPUNIT_ASSERT(bs.length() == len); + CPPUNIT_ASSERT(memcmp(bpp, bs.buf(), len) == 0); + + bs >> bpp1; + CPPUNIT_ASSERT(bs.length() == 0); + CPPUNIT_ASSERT(memcmp(bpp, bpp1, len) == 0); + + bs.reset(); + } +}; + +#define TS_NS(x) (x) +#define TS_US(x) ((x) * 1000) +#define TS_MS(x) ((x) * 1000000) + +CPPUNIT_TEST_SUITE_REGISTRATION(ByteStreamTestSuite); + +#include +#include + +#include + +void setupSignalHandlers() +{ + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + + sigaction(SIGPIPE, &ign, 0); +} + +int main(int argc, char** argv) +{ + setupSignalHandlers(); + + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); +} diff --git a/utils/cloudio/SocketPool.cpp b/utils/cloudio/SocketPool.cpp index f461e39f7..bf0c22fbd 100644 --- a/utils/cloudio/SocketPool.cpp +++ b/utils/cloudio/SocketPool.cpp @@ -16,6 +16,7 @@ MA 02110-1301, USA. */ #include "SocketPool.h" +#include "bytestream.h" #include "configcpp.h" #include "logger.h" #include "messageFormat.h" @@ -87,8 +88,8 @@ SocketPool::~SocketPool() int SocketPool::send_recv(messageqcpp::ByteStream& in, messageqcpp::ByteStream* out) { - uint count = 0; - uint length = in.length(); + messageqcpp::BSSizeType count = 0; + messageqcpp::BSSizeType length = in.length(); int sock = -1; const uint8_t* inbuf = in.buf(); ssize_t err = 0; diff --git a/utils/joiner/joinpartition.cpp b/utils/joiner/joinpartition.cpp index bccf89b38..362842b55 100644 --- a/utils/joiner/joinpartition.cpp +++ b/utils/joiner/joinpartition.cpp @@ -16,6 +16,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include "bytestream.h" #define _CRT_RAND_S // for win rand_s #include #include @@ -804,7 +805,7 @@ uint64_t JoinPartition::writeByteStream(int which, ByteStream& bs) } uint64_t ret = 0; - size_t len = bs.length(); + BSSizeType len = bs.length(); idbassert(len != 0); fs.seekp(offset); diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index 6fad9767f..565b6eb1c 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -1,6 +1,6 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2019 MariaDB Corporation + Copyright (C) 2019-2024 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -46,7 +46,7 @@ namespace messageqcpp /* Copies only the data left to be read */ void ByteStream::doCopy(const ByteStream& rhs) { - uint32_t rlen = rhs.length(); + BSSizeType rlen = rhs.length(); if (fMaxLen < rlen) { @@ -94,7 +94,7 @@ ByteStream& ByteStream::operator=(const ByteStream& rhs) return *this; } -ByteStream::ByteStream(uint32_t initSize) : fBuf(0), fCurInPtr(0), fCurOutPtr(0), fMaxLen(0) +ByteStream::ByteStream(BSSizeType initSize) : fBuf(0), fCurInPtr(0), fCurOutPtr(0), fMaxLen(0) { if (initSize > 0) growBuf(initSize); @@ -102,13 +102,13 @@ ByteStream::ByteStream(uint32_t initSize) : fBuf(0), fCurInPtr(0), fCurOutPtr(0) void ByteStream::add(const uint8_t b) { - if (fBuf == 0 || (static_cast(fCurInPtr - fBuf) == fMaxLen + ISSOverhead)) + if (fBuf == 0 || (static_cast(fCurInPtr - fBuf) == fMaxLen + ISSOverhead)) growBuf(); *fCurInPtr++ = b; } -void ByteStream::growBuf(uint32_t toSize) +void ByteStream::growBuf(BSSizeType toSize) { if (fBuf == 0) { @@ -138,8 +138,8 @@ void ByteStream::growBuf(uint32_t toSize) toSize = std::max(toSize, fMaxLen * 2); uint8_t* t = new uint8_t[toSize + ISSOverhead]; - uint32_t curOutOff = fCurOutPtr - fBuf; - uint32_t curInOff = fCurInPtr - fBuf; + BSSizeType curOutOff = fCurOutPtr - fBuf; + BSSizeType curInOff = fCurInPtr - fBuf; memcpy(t, fBuf, fCurInPtr - fBuf); #ifdef ZERO_ON_NEW memset(t + (fCurInPtr - fBuf), 0, (toSize + ISSOverhead) - (fCurInPtr - fBuf)); @@ -169,7 +169,7 @@ void ByteStream::setLongStrings(const std::vector>& o ByteStream& ByteStream::operator<<(const int8_t b) { - if (fBuf == 0 || (fCurInPtr - fBuf + 1U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(b) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); *((int8_t*)fCurInPtr) = b; @@ -187,7 +187,7 @@ ByteStream& ByteStream::operator<<(const uint8_t b) ByteStream& ByteStream::operator<<(const int16_t d) { - if (fBuf == 0 || (fCurInPtr - fBuf + 2U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(d) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); *((int16_t*)fCurInPtr) = d; @@ -198,7 +198,7 @@ ByteStream& ByteStream::operator<<(const int16_t d) ByteStream& ByteStream::operator<<(const uint16_t d) { - if (fBuf == 0 || (fCurInPtr - fBuf + 2U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(d) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); *((uint16_t*)fCurInPtr) = d; @@ -209,7 +209,7 @@ ByteStream& ByteStream::operator<<(const uint16_t d) ByteStream& ByteStream::operator<<(const int32_t q) { - if (fBuf == 0 || (fCurInPtr - fBuf + 4U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(q) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); *((int32_t*)fCurInPtr) = q; @@ -220,7 +220,7 @@ ByteStream& ByteStream::operator<<(const int32_t q) ByteStream& ByteStream::operator<<(const uint32_t q) { - if (fBuf == 0 || (fCurInPtr - fBuf + 4U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(q) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); *((uint32_t*)fCurInPtr) = q; @@ -231,7 +231,7 @@ ByteStream& ByteStream::operator<<(const uint32_t q) ByteStream& ByteStream::operator<<(const int64_t o) { - if (fBuf == 0 || (fCurInPtr - fBuf + 8U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(o) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); *((int64_t*)fCurInPtr) = o; @@ -242,7 +242,7 @@ ByteStream& ByteStream::operator<<(const int64_t o) ByteStream& ByteStream::operator<<(const uint64_t o) { - if (fBuf == 0 || (fCurInPtr - fBuf + 8U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(o) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); *((uint64_t*)fCurInPtr) = o; @@ -251,20 +251,20 @@ ByteStream& ByteStream::operator<<(const uint64_t o) return *this; } -ByteStream& ByteStream::operator<<(const uint128_t& o) +ByteStream& ByteStream::operator<<(const uint128_t& h) { - if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(h) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); - datatypes::TSInt128::storeUnaligned(fCurInPtr, o); + datatypes::TSInt128::storeUnaligned(fCurInPtr, h); fCurInPtr += 16; return *this; } -ByteStream& ByteStream::operator<<(const int128_t& o) +ByteStream& ByteStream::operator<<(const int128_t& h) { - if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + if (fBuf == 0 || (fCurInPtr - fBuf + sizeof(h) > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); - datatypes::TSInt128::storeUnaligned(fCurInPtr, o); + datatypes::TSInt128::storeUnaligned(fCurInPtr, h); fCurInPtr += 16; return *this; } @@ -475,18 +475,18 @@ void ByteStream::peek(uint64_t& o) const o = *((uint64_t*)fCurOutPtr); } -void ByteStream::peek(uint128_t& o) const +void ByteStream::peek(uint128_t& h) const { if (length() < 16) throw underflow_error("ByteStream>uint128_t: not enough data in stream to fill datatype"); - datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); + datatypes::TSInt128::assignPtrPtr(&h, fCurOutPtr); } -void ByteStream::peek(int128_t& o) const +void ByteStream::peek(int128_t& h) const { if (length() < 16) throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); - datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); + datatypes::TSInt128::assignPtrPtr(&h, fCurOutPtr); } void ByteStream::peek(string& s) const @@ -519,7 +519,7 @@ void ByteStream::peek(string& s) const throw logging::ProtocolError("expected a string"); // we know len >= 0 by now... - if (length() < static_cast(len + 4)) + if (length() < static_cast(len + 4)) { #if DEBUG_DUMP_STRINGS_LESS_THAN > 0 cerr << "bs: wanted " << len + 4 << " bytes, but there are only " << length() << " remaining" << endl; @@ -531,13 +531,13 @@ void ByteStream::peek(string& s) const s.assign((char*)&fCurOutPtr[4], len); } -void ByteStream::load(const uint8_t* bp, uint32_t len) +void ByteStream::load(const uint8_t* bp, BSSizeType len) { // Do all the stuff that could throw an exception first if (bp == 0 && len != 0) throw invalid_argument("ByteStream::load: bp cannot equal 0 when len is not equal to 0"); - uint32_t newMaxLen = (len + BlockSize - 1) / BlockSize * BlockSize; + BSSizeType newMaxLen = (len + BlockSize - 1) / BlockSize * BlockSize; if (len > fMaxLen) { @@ -551,7 +551,7 @@ void ByteStream::load(const uint8_t* bp, uint32_t len) fCurInPtr = fBuf + len + ISSOverhead; } -void ByteStream::append(const uint8_t* bp, uint32_t len) +void ByteStream::append(const uint8_t* bp, BSSizeType len) { if (len == 0) return; @@ -559,7 +559,7 @@ void ByteStream::append(const uint8_t* bp, uint32_t len) if (bp == 0) throw invalid_argument("ByteStream::append: bp cannot equal 0 when len is not equal to 0"); - uint32_t newSize = static_cast(fCurInPtr - fBuf + len); + BSSizeType newSize = static_cast(fCurInPtr - fBuf + len); if (fBuf == 0 || (newSize > fMaxLen)) growBuf(newSize); @@ -635,7 +635,7 @@ void ByteStream::serialize(ByteStream& bs) const void ByteStream::deserialize(ByteStream& bs) { - uint32_t len; + BSSizeType len; restart(); bs >> len; @@ -643,9 +643,9 @@ void ByteStream::deserialize(ByteStream& bs) bs.advance(len); } -void ByteStream::needAtLeast(size_t amount) +void ByteStream::needAtLeast(BSSizeType amount) { - size_t currentSpace; + BSSizeType currentSpace; currentSpace = fMaxLen - (fCurInPtr - (fBuf + ISSOverhead)); @@ -656,7 +656,7 @@ void ByteStream::needAtLeast(size_t amount) ByteStream& ByteStream::operator<<(const ByteStream& bs) { - uint32_t len = bs.length(); + BSSizeType len = bs.length(); *this << len; @@ -668,20 +668,20 @@ ByteStream& ByteStream::operator<<(const ByteStream& bs) ByteStream& ByteStream::operator>>(ByteStream& bs) { peek(bs); - fCurOutPtr += 4 + bs.length(); + fCurOutPtr += sizeof(BSSizeType) + bs.length(); return *this; } void ByteStream::peek(ByteStream& bs) const { - uint32_t len; + BSSizeType len; peek(len); if (length() < len) throw underflow_error("ByteStream>ByteStream: not enough data in stream to fill datatype"); - bs.load(&fCurOutPtr[4], len); + bs.load(&fCurOutPtr[sizeof(len)], len); } ByteStream& ByteStream::operator<<(const uuid& u) @@ -707,7 +707,7 @@ void ByteStream::peek(uuid& u) const ByteStream& ByteStream::operator<<(const float f) { - int sz = sizeof(float); + const constexpr BSSizeType sz = sizeof(float); if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); @@ -719,7 +719,7 @@ ByteStream& ByteStream::operator<<(const float f) } ByteStream& ByteStream::operator<<(const double d) { - int sz = sizeof(double); + const constexpr BSSizeType sz = sizeof(double); if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); @@ -731,7 +731,7 @@ ByteStream& ByteStream::operator<<(const double d) } ByteStream& ByteStream::operator<<(const long double d) { - int sz = sizeof(long double); + const constexpr BSSizeType sz = sizeof(long double); if (fBuf == 0 || (fCurInPtr - fBuf + sz > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 872956167..ff9606298 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -45,6 +45,7 @@ class ByteStreamTestSuite; namespace messageqcpp { typedef boost::shared_ptr SBS; +using BSSizeType = uint64_t; /** * @brief A class to marshall bytes as a stream @@ -76,11 +77,11 @@ class ByteStream : public Serializeable /** * default ctor */ - EXPORT explicit ByteStream(uint32_t initSize = 8192); // multiples of pagesize are best + EXPORT explicit ByteStream(BSSizeType initSize = 8192); // multiples of pagesize are best /** * ctor with a uint8_t array and len initializer */ - inline ByteStream(const uint8_t* bp, const uint32_t len); + inline ByteStream(const uint8_t* bp, const BSSizeType len); /** * copy ctor */ @@ -337,12 +338,12 @@ class ByteStream : public Serializeable /** * load the stream from an array. Clears out any previous data. */ - EXPORT void load(const uint8_t* bp, uint32_t len); + EXPORT void load(const uint8_t* bp, BSSizeType len); /** * append bytes to the end of the stream. */ - EXPORT void append(const uint8_t* bp, uint32_t len); + EXPORT void append(const uint8_t* bp, BSSizeType len); /** * equality check on buffer contents. @@ -378,19 +379,19 @@ class ByteStream : public Serializeable * advance the output ptr without having to extract bytes * @warning be careful advancing near 4GB! */ - inline void advance(uint32_t amt); + inline void advance(BSSizeType amt); /** * returns the length of the queue (in bytes) * @warning do not attempt to make a ByteStream bigger than 4GB! */ - inline uint32_t length() const; + inline BSSizeType length() const; inline bool empty() const; /** * returns the length of the queue, including header overhead (in bytes) */ - inline uint32_t lengthWithHdrOverhead() const; + inline BSSizeType lengthWithHdrOverhead() const; /** * clears the stream. Releases any current stream and sets all pointers to 0. The state of the object @@ -422,7 +423,7 @@ class ByteStream : public Serializeable /** * Get the allocated size of the buffer. */ - inline uint32_t getBufferSize() const; + inline BSSizeType getBufferSize() const; /** * Serializeable interface @@ -437,10 +438,10 @@ class ByteStream : public Serializeable /** * memory allocation chunk size */ - EXPORT static const uint32_t BlockSize = 4096; + EXPORT static const BSSizeType BlockSize = 4096; /** size of the space we want in front of the data */ - EXPORT static const uint32_t ISSOverhead = + EXPORT static const BSSizeType ISSOverhead = 3 * sizeof(uint32_t); // space for the BS magic & length & number of long strings. // Methods to get and set `long strings`. @@ -458,7 +459,7 @@ class ByteStream : public Serializeable /** * adds another BlockSize bytes to the internal buffer */ - void growBuf(uint32_t toSize = 0); + void growBuf(BSSizeType toSize = 0); /** * handles member copying from one ByteStream to another */ @@ -476,9 +477,8 @@ class ByteStream : public Serializeable uint8_t* fBuf; /// the start of the allocated buffer uint8_t* fCurInPtr; // the point in fBuf where data is inserted next uint8_t* fCurOutPtr; // the point in fBuf where data is extracted from next - uint32_t fMaxLen; // how big fBuf is currently - // Stores `long strings`. - std::vector> longStrings; + BSSizeType fMaxLen; // how big fBuf is currently + std::vector> longStrings; // Stores `long strings`. }; template @@ -527,7 +527,7 @@ static const uint8_t BS_BLOB = 9; static const uint8_t BS_SERIALIZABLE = 10; static const uint8_t BS_UUID = 11; -inline ByteStream::ByteStream(const uint8_t* bp, const uint32_t len) : fBuf(0), fMaxLen(0) +inline ByteStream::ByteStream(const uint8_t* bp, const BSSizeType len) : fBuf(0), fMaxLen(0) { load(bp, len); } @@ -544,15 +544,15 @@ inline uint8_t* ByteStream::buf() { return fCurOutPtr; } -inline uint32_t ByteStream::length() const +inline BSSizeType ByteStream::length() const { - return (uint32_t)(fCurInPtr - fCurOutPtr); + return static_cast(fCurInPtr - fCurOutPtr); } inline bool ByteStream::empty() const { return (length() == 0); } -inline uint32_t ByteStream::lengthWithHdrOverhead() const +inline BSSizeType ByteStream::lengthWithHdrOverhead() const { return (length() + ISSOverhead); } @@ -570,7 +570,7 @@ inline void ByteStream::rewind() { fCurOutPtr = fBuf + ISSOverhead; } -inline void ByteStream::advance(uint32_t adv) +inline void ByteStream::advance(BSSizeType adv) { // fCurOutPtr is always >= fBuf, so fCurOutPtr - fBuf is >= 0, and this difference is always <= 32 bits // there is an edge condition not detected here: if fCurOutPtr - fBuf is nearly 4GB and you try to @@ -619,7 +619,7 @@ inline ByteStream& ByteStream::operator=(const SBS& rhs) return *this; } -inline uint32_t ByteStream::getBufferSize() const +inline BSSizeType ByteStream::getBufferSize() const { return fMaxLen; } @@ -738,12 +738,6 @@ void deserializeSet(ByteStream& bs, std::set& s) s.insert(tmp); } } -/* -template<> -struct ByteStream::_ByteStreamType<1, ByteStream::byte>> -{ - typedef ByteStream::byte type; -}*/ } // namespace messageqcpp diff --git a/utils/messageqcpp/compressed_iss.cpp b/utils/messageqcpp/compressed_iss.cpp index 82fdf0514..c194a41ad 100644 --- a/utils/messageqcpp/compressed_iss.cpp +++ b/utils/messageqcpp/compressed_iss.cpp @@ -20,6 +20,7 @@ * * ***********************************************************************/ +#include "bytestream.h" #include "mcsconfig.h" #include @@ -117,7 +118,7 @@ const SBS CompressedInetStreamSocket::read(const struct timespec* timeout, bool* void CompressedInetStreamSocket::write(const ByteStream& msg, Stats* stats) { - size_t len = msg.length(); + BSSizeType len = msg.length(); if (useCompression && (len > 512)) { @@ -126,6 +127,9 @@ void CompressedInetStreamSocket::write(const ByteStream& msg, Stats* stats) alg->compress((char*)msg.buf(), len, (char*)smsg.getInputPtr() + HEADER_SIZE, &outLen); // Save original len. + // !!! + // !!! Reducing BS size type from 64bit down to 32 and potentially loosing data. + // !!! *(uint32_t*)smsg.getInputPtr() = len; smsg.advanceInputPtr(outLen + HEADER_SIZE); diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index 1e94e0b66..7df141192 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -572,6 +572,9 @@ void InetStreamSocket::write(SBS msg, Stats* stats) void InetStreamSocket::do_write(const ByteStream& msg, uint32_t whichMagic, Stats* stats) const { + // !!! + // !!! Reducing BS size type from 64bit down to 32 and potentially loosing data. + // !!! uint32_t msglen = msg.length(); uint32_t magic = whichMagic; uint32_t* realBuf; diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index b8f835d7f..69049542a 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -31,7 +31,6 @@ #include using namespace std; - #include #include "bytestream.h" @@ -49,8 +48,6 @@ namespace rowgroup { using cscType = execplan::CalpontSystemCatalog::ColDataType; - - StringStore::~StringStore() { #if 0 @@ -302,47 +299,27 @@ void UserDataStore::deserialize(ByteStream& bs) return; } - RGData::RGData(const RowGroup& rg, uint32_t rowCount) { - // cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; - rowData.reset(new uint8_t[rg.getDataSize(rowCount)]); + RGDataSizeType s = rg.getDataSize(rowCount); + rowData.reset(new uint8_t[s]); if (rg.usesStringTable() && rowCount > 0) strings.reset(new StringStore()); userDataStore.reset(); - - -#ifdef VALGRIND - /* In a PM-join, we can serialize entire tables; not every value has been - * filled in yet. Need to look into that. Valgrind complains that - * those bytes are uninitialized, this suppresses that error. - */ - memset(rowData.get(), 0, rg.getDataSize(rowCount)); // XXXPAT: make valgrind happy temporarily -#endif - memset(rowData.get(), 0, rg.getDataSize(rowCount)); // XXXPAT: make valgrind happy temporarily columnCount = rg.getColumnCount(); rowSize = rg.getRowSize(); } RGData::RGData(const RowGroup& rg) { - // cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; rowData.reset(new uint8_t[rg.getMaxDataSize()]); if (rg.usesStringTable()) strings.reset(new StringStore()); userDataStore.reset(); - -#ifdef VALGRIND - /* In a PM-join, we can serialize entire tables; not every value has been - * filled in yet. Need to look into that. Valgrind complains that - * those bytes are uninitialized, this suppresses that error. - */ - memset(rowData.get(), 0, rg.getMaxDataSize()); -#endif columnCount = rg.getColumnCount(); rowSize = rg.getRowSize(); } @@ -356,14 +333,6 @@ void RGData::reinit(const RowGroup& rg, uint32_t rowCount) strings.reset(new StringStore()); else strings.reset(); - -#ifdef VALGRIND - /* In a PM-join, we can serialize entire tables; not every value has been - * filled in yet. Need to look into that. Valgrind complains that - * those bytes are uninitialized, this suppresses that error. - */ - memset(rowData.get(), 0, rg.getDataSize(rowCount)); -#endif columnCount = rg.getColumnCount(); rowSize = rg.getRowSize(); } @@ -373,11 +342,11 @@ void RGData::reinit(const RowGroup& rg) reinit(rg, 8192); } -void RGData::serialize(ByteStream& bs, uint32_t amount) const +void RGData::serialize(ByteStream& bs, RGDataSizeType amount) const { // cout << "serializing!\n"; bs << (uint32_t)RGDATA_SIG; - bs << (uint32_t)amount; + bs << amount; bs << columnCount; bs << rowSize; bs.append(rowData.get(), amount); @@ -399,9 +368,10 @@ void RGData::serialize(ByteStream& bs, uint32_t amount) const bs << (uint8_t)0; } -void RGData::deserialize(ByteStream& bs, uint32_t defAmount) +void RGData::deserialize(ByteStream& bs, RGDataSizeType defAmount) { - uint32_t amount, sig; + uint32_t sig; + RGDataSizeType amount; uint8_t* buf; uint8_t tmp8; @@ -642,7 +612,7 @@ string Row::toCSV() const void Row::setToNull(uint32_t colIndex) { - setNullMark(colIndex, true); // mark as null. + setNullMark(colIndex, true); // mark as null. switch (types[colIndex]) { case CalpontSystemCatalog::TINYINT: data[offsets[colIndex]] = joblist::TINYINTNULL; break; @@ -665,11 +635,11 @@ void Row::setToNull(uint32_t colIndex) *((int32_t*)&data[offsets[colIndex]]) = static_cast(joblist::DATENULL); break; - case CalpontSystemCatalog::BIGINT: - if (precision[colIndex] != MagicPrecisionForCountAgg) - *((uint64_t*)&data[offsets[colIndex]]) = joblist::BIGINTNULL; - else // work around for count() in outer join result. - *((uint64_t*)&data[offsets[colIndex]]) = 0; + case CalpontSystemCatalog::BIGINT: + if (precision[colIndex] != MagicPrecisionForCountAgg) + *((uint64_t*)&data[offsets[colIndex]]) = joblist::BIGINTNULL; + else // work around for count() in outer join result. + *((uint64_t*)&data[offsets[colIndex]]) = 0; break; @@ -680,9 +650,13 @@ void Row::setToNull(uint32_t colIndex) *((long double*)&data[offsets[colIndex]]) = joblist::LONGDOUBLENULL; break; - case CalpontSystemCatalog::DATETIME: *((uint64_t*)&data[offsets[colIndex]]) = joblist::DATETIMENULL; break; + case CalpontSystemCatalog::DATETIME: + *((uint64_t*)&data[offsets[colIndex]]) = joblist::DATETIMENULL; + break; - case CalpontSystemCatalog::TIMESTAMP: *((uint64_t*)&data[offsets[colIndex]]) = joblist::TIMESTAMPNULL; break; + case CalpontSystemCatalog::TIMESTAMP: + *((uint64_t*)&data[offsets[colIndex]]) = joblist::TIMESTAMPNULL; + break; case CalpontSystemCatalog::TIME: *((uint64_t*)&data[offsets[colIndex]]) = joblist::TIMENULL; break; @@ -716,9 +690,7 @@ void Row::setToNull(uint32_t colIndex) case 7: case 8: *((uint64_t*)&data[offsets[colIndex]]) = joblist::CHAR8NULL; break; - default: - setNullMark(colIndex, true); - break; + default: setNullMark(colIndex, true); break; } break; @@ -751,7 +723,9 @@ void Row::setToNull(uint32_t colIndex) case CalpontSystemCatalog::UTINYINT: data[offsets[colIndex]] = joblist::UTINYINTNULL; break; - case CalpontSystemCatalog::USMALLINT: *((uint16_t*)&data[offsets[colIndex]]) = joblist::USMALLINTNULL; break; + case CalpontSystemCatalog::USMALLINT: + *((uint16_t*)&data[offsets[colIndex]]) = joblist::USMALLINTNULL; + break; case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: *((uint32_t*)&data[offsets[colIndex]]) = joblist::UINTNULL; break; @@ -760,8 +734,8 @@ void Row::setToNull(uint32_t colIndex) default: ostringstream os; - os << "Row::initToNull(): got bad column type (" << types[colIndex] << "). Width=" << getColumnWidth(colIndex) - << endl; + os << "Row::initToNull(): got bad column type (" << types[colIndex] + << "). Width=" << getColumnWidth(colIndex) << endl; os << toString(); throw logic_error(os.str()); } @@ -870,8 +844,8 @@ bool Row::isNullValue(uint32_t colIndex) const return strings->isNullValue(offset); } -// if (data[offsets[colIndex]] == 0) // empty string -// return true; + // if (data[offsets[colIndex]] == 0) // empty string + // return true; switch (len) { @@ -1126,7 +1100,6 @@ RowGroup::RowGroup(const RowGroup& r) offsets = &stOffsets[0]; else if (!useStringTable && !oldOffsets.empty()) offsets = &oldOffsets[0]; - } RowGroup& RowGroup::operator=(const RowGroup& r) @@ -1241,27 +1214,28 @@ void RowGroup::serializeRGData(ByteStream& bs) const rgData->serialize(bs, getDataSize()); } -uint32_t RowGroup::getDataSize() const +RGDataSizeType RowGroup::getDataSize() const { return getDataSize(getRowCount()); } -uint32_t RowGroup::getDataSize(uint64_t n) const +RGDataSizeType RowGroup::getDataSize(uint64_t n) const { - return headerSize + (n * getRowSize()); + return headerSize + (n * static_cast(getRowSize())); } -uint32_t RowGroup::getMaxDataSize() const +RGDataSizeType RowGroup::getMaxDataSize() const { - return headerSize + (8192 * getRowSize()); + return headerSize + (static_cast(rgCommonSize) * static_cast(getRowSize())); } -uint32_t RowGroup::getMaxDataSizeWithStrings() const +RGDataSizeType RowGroup::getMaxDataSizeWithStrings() const { - return headerSize + (8192 * (oldOffsets[columnCount] + columnCount)); + return headerSize + + (static_cast(rgCommonSize) * static_cast(getRowSizeWithStrings())); } -uint32_t RowGroup::getEmptySize() const +RGDataSizeType RowGroup::getEmptySize() const { return headerSize; } @@ -1331,9 +1305,8 @@ string RowGroup::toString(const std::vector& used) const os << "rowcount = " << getRowCount() << endl; if (!used.empty()) { - uint64_t cnt = - std::accumulate(used.begin(), used.end(), 0ULL, - [](uint64_t a, uint64_t bits) { return a + __builtin_popcountll(bits); }); + uint64_t cnt = std::accumulate(used.begin(), used.end(), 0ULL, [](uint64_t a, uint64_t bits) + { return a + __builtin_popcountll(bits); }); os << "sparse row count = " << cnt << endl; } os << "base rid = " << getBaseRid() << endl; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 113f0ae6e..62fd3a306 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -62,6 +62,7 @@ namespace rowgroup { const int16_t rgCommonSize = 8192; +using RGDataSizeType = uint64_t; /* The RowGroup family of classes encapsulate the data moved through the @@ -270,14 +271,14 @@ class RGData // amount should be the # returned by RowGroup::getDataSize() - void serialize(messageqcpp::ByteStream&, uint32_t amount) const; + void serialize(messageqcpp::ByteStream&, RGDataSizeType amount) const; // the 'hasLengthField' is there b/c PM aggregation (and possibly others) currently sends // inline data with a length field. Once that's converted to string table format, that // option can go away. - void deserialize(messageqcpp::ByteStream&, uint32_t amount = 0); // returns the # of bytes read + void deserialize(messageqcpp::ByteStream&, RGDataSizeType amount = 0); // returns the # of bytes read - inline uint64_t getStringTableMemUsage(); + inline RGDataSizeType getStringTableMemUsage(); void clear(); void reinit(const RowGroup& rg); void reinit(const RowGroup& rg, uint32_t rowCount); @@ -1499,15 +1500,15 @@ class RowGroup : public messageqcpp::Serializeable uint32_t getDBRoot() const; void setDBRoot(uint32_t); - uint32_t getDataSize() const; - uint32_t getDataSize(uint64_t n) const; - uint32_t getMaxDataSize() const; - uint32_t getMaxDataSizeWithStrings() const; - uint32_t getEmptySize() const; + RGDataSizeType getDataSize() const; + RGDataSizeType getDataSize(uint64_t n) const; + RGDataSizeType getMaxDataSize() const; + RGDataSizeType getMaxDataSizeWithStrings() const; + RGDataSizeType getEmptySize() const; // this returns the size of the row data with the string table - inline uint64_t getSizeWithStrings() const; - inline uint64_t getSizeWithStrings(uint64_t n) const; + inline RGDataSizeType getSizeWithStrings() const; + inline RGDataSizeType getSizeWithStrings(uint64_t n) const; // sets the row count to 0 and the baseRid to something // effectively initializing whatever chunk of memory @@ -1628,11 +1629,11 @@ class RowGroup : public messageqcpp::Serializeable uint32_t sTableThreshold = 20; std::shared_ptr forceInline; - static const uint32_t headerSize = 18; - static const uint32_t rowCountOffset = 0; - static const uint32_t baseRidOffset = 4; - static const uint32_t statusOffset = 12; - static const uint32_t dbRootOffset = 14; + static const uint64_t headerSize = 18; + static const uint64_t rowCountOffset = 0; + static const uint64_t baseRidOffset = 4; + static const uint64_t statusOffset = 12; + static const uint64_t dbRootOffset = 14; }; inline uint64_t convertToRid(const uint32_t& partNum, const uint16_t& segNum, const uint8_t& extentNum, @@ -1778,7 +1779,7 @@ inline uint32_t RowGroup::getRowSizeWithStrings() const return oldOffsets[columnCount] + columnCount; } -inline uint64_t RowGroup::getSizeWithStrings(uint64_t n) const +inline RGDataSizeType RowGroup::getSizeWithStrings(uint64_t n) const { if (strings == nullptr) return getDataSize(n); diff --git a/utils/rowgroup/rowstorage.cpp b/utils/rowgroup/rowstorage.cpp index fe07fe8b7..3c9a3aee0 100644 --- a/utils/rowgroup/rowstorage.cpp +++ b/utils/rowgroup/rowstorage.cpp @@ -689,6 +689,7 @@ class RowGroupStorage if (fRGDatas[rgid]) { fRowGroupOut->setData(fRGDatas[rgid].get()); + // An implicit s2u type cast. int64_t memSz = fRowGroupOut->getSizeWithStrings(fMaxRows); if (!fMM->acquire(memSz)) { @@ -965,7 +966,7 @@ class RowGroupStorage while (rgid >= fRGDatas.size()) { - int64_t memSz = fRowGroupOut->getSizeWithStrings(fMaxRows); + auto memSz = fRowGroupOut->getSizeWithStrings(fMaxRows); if (!fMM->acquire(memSz)) { throw logging::IDBExcept( diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index ec4008062..1c6fd9528 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -753,7 +753,7 @@ void IdbOrderBy::initialize(const RowGroup& rg) // initialize rows IdbCompare::initialize(rg); - uint64_t newSize = rg.getSizeWithStrings(fRowsPerRG); + auto newSize = rg.getSizeWithStrings(fRowsPerRG); if (fRm && !fRm->getMemory(newSize, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index b4f648eb7..27e0eece0 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -400,8 +400,7 @@ void SlaveComm::do_createStripeColumnExtents(ByteStream& msg) if (printOnly) { - cout << "createStripeColumnExtents(). " - << "DBRoot=" << dbRoot << "; Part#=" << partitionNum << endl; + cout << "createStripeColumnExtents(). " << "DBRoot=" << dbRoot << "; Part#=" << partitionNum << endl; for (uint32_t i = 0; i < cols.size(); i++) cout << "StripeColExt arg " << i + 1 << ": oid=" << cols[i].oid << " width=" << cols[i].width << endl; @@ -2183,6 +2182,9 @@ void SlaveComm::saveDelta() { try { + // !!! + // !!! Reducing BS size type from 64bit down to 32 and potentially loosing data. + // !!! const uint32_t deltaLen = delta.length(); const uint32_t bufferSize = sizeof(deltaLen) + deltaLen; std::unique_ptr buffer(new char[bufferSize]); diff --git a/writeengine/server/we_dataloader.cpp b/writeengine/server/we_dataloader.cpp index 25ae1fbe8..ce96f28ee 100644 --- a/writeengine/server/we_dataloader.cpp +++ b/writeengine/server/we_dataloader.cpp @@ -437,7 +437,7 @@ void WEDataLoader::pushData2Cpimport(ByteStream& Ibs) { if (Ibs.length() > 0) { - int aLen = Ibs.length(); + messageqcpp::BSSizeType aLen = Ibs.length(); char* pStart = reinterpret_cast(Ibs.buf()); char* pEnd = pStart + aLen; char* pPtr = pStart; diff --git a/writeengine/server/we_dataloader.h b/writeengine/server/we_dataloader.h index 86ce2904d..ad43609c7 100644 --- a/writeengine/server/we_dataloader.h +++ b/writeengine/server/we_dataloader.h @@ -30,6 +30,7 @@ #pragma once +#include "bytestream.h" #include "rwlock_local.h" #include "resourcemanager.h" @@ -58,32 +59,32 @@ class WEDataLoader : public Observer public: bool setupCpimport(); // fork the cpimport void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267 - void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue + void pushData2Cpimport(messageqcpp::ByteStream& Ibs); // push data to cpimport from the queue void closeWritePipe(); void str2Argv(std::string CmdLine, std::vector& V); public: - void onReceiveKeepAlive(ByteStream& Ibs); - void onReceiveData(ByteStream& Ibs); - void onReceiveEod(ByteStream& Ibs); // end of data - void onReceiveMode(ByteStream& Ibs); + void onReceiveKeepAlive(messageqcpp::ByteStream& Ibs); + void onReceiveData(messageqcpp::ByteStream& Ibs); + void onReceiveEod(messageqcpp::ByteStream& Ibs); // end of data + void onReceiveMode(messageqcpp::ByteStream& Ibs); // void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs); - void onReceiveCmd(ByteStream& bs); // {(ByteStream& Ibs); - void onReceiveAck(ByteStream& Ibs); - void onReceiveNak(ByteStream& Ibs); - void onReceiveError(ByteStream& Ibs); + void onReceiveCmd(messageqcpp::ByteStream& bs); // {(ByteStream& Ibs); + void onReceiveAck(messageqcpp::ByteStream& Ibs); + void onReceiveNak(messageqcpp::ByteStream& Ibs); + void onReceiveError(messageqcpp::ByteStream& Ibs); - void onReceiveJobId(ByteStream& Ibs); - void onReceiveJobData(ByteStream& Ibs); - void onReceiveImportFileName(ByteStream& Ibs); - void onReceiveCmdLineArgs(ByteStream& Ibs); + void onReceiveJobId(messageqcpp::ByteStream& Ibs); + void onReceiveJobData(messageqcpp::ByteStream& Ibs); + void onReceiveImportFileName(messageqcpp::ByteStream& Ibs); + void onReceiveCmdLineArgs(messageqcpp::ByteStream& Ibs); void onReceiveStartCpimport(); - void onReceiveBrmRptFileName(ByteStream& Ibs); - void onReceiveCleanup(ByteStream& Ibs); - void onReceiveRollback(ByteStream& Ibs); + void onReceiveBrmRptFileName(messageqcpp::ByteStream& Ibs); + void onReceiveCleanup(messageqcpp::ByteStream& Ibs); + void onReceiveRollback(messageqcpp::ByteStream& Ibs); - void onReceiveErrFileRqst(ByteStream& Ibs); - void onReceiveBadFileRqst(ByteStream& Ibs); + void onReceiveErrFileRqst(messageqcpp::ByteStream& Ibs); + void onReceiveBadFileRqst(messageqcpp::ByteStream& Ibs); void onCpimportSuccess(); void onCpimportFailure(); @@ -103,11 +104,11 @@ class WEDataLoader : public Observer { fMode = Mode; } - void updateTxBytes(unsigned int Tx) + void updateTxBytes(messageqcpp::BSSizeType Tx) { fTxBytes += Tx; } - void updateRxBytes(unsigned int Rx) + void updateRxBytes(messageqcpp::BSSizeType Rx) { fRxBytes += Rx; } @@ -132,11 +133,11 @@ class WEDataLoader : public Observer fObjId = ObjId; } - unsigned int getTxBytes() + messageqcpp::BSSizeType getTxBytes() { return fTxBytes; } - unsigned int getRxBytes() + messageqcpp::BSSizeType getRxBytes() { return fRxBytes; } @@ -172,8 +173,8 @@ class WEDataLoader : public Observer int fMode; std::ofstream fDataDumpFile; std::ofstream fJobFile; - unsigned int fTxBytes; - unsigned int fRxBytes; + messageqcpp::BSSizeType fTxBytes; + messageqcpp::BSSizeType fRxBytes; char fPmId; int fObjId; // Object Identifier for logging diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index b076c91c2..d4eb6af6e 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -2289,7 +2289,7 @@ int WESDHandler::getNextDbrPm2Send() int WESDHandler::leastDataSendPm() { - unsigned int aTx = 0; + messageqcpp::BSSizeType aTx = 0; int aPmId = 0; for (int aCnt = 1; aCnt <= fPmCount; aCnt++) diff --git a/writeengine/splitter/we_splclient.cpp b/writeengine/splitter/we_splclient.cpp index 8bcbdcd56..b4eee7e16 100644 --- a/writeengine/splitter/we_splclient.cpp +++ b/writeengine/splitter/we_splclient.cpp @@ -208,7 +208,7 @@ void WESplClient::send() messageqcpp::SBS aSbs = fSendQueue.front(); fSendQueue.pop(); aLock.unlock(); - int aLen = (*aSbs).length(); + messageqcpp::BSSizeType aLen = (*aSbs).length(); if (aLen > 0) { @@ -241,7 +241,7 @@ void WESplClient::recv() rm_ts.tv_sec = fRdSecTo; // 0 when data sending otherwise 1- second rm_ts.tv_nsec = 20000000; // 20 milliSec bool isTimeOut = false; - int aLen = 0; + messageqcpp::BSSizeType aLen = 0; try { diff --git a/writeengine/splitter/we_splclient.h b/writeengine/splitter/we_splclient.h index 51db86399..e38ee0d25 100644 --- a/writeengine/splitter/we_splclient.h +++ b/writeengine/splitter/we_splclient.h @@ -29,7 +29,6 @@ #pragma once -#include "threadsafequeue.h" #include "resourcemanager.h" #include "we_messages.h" @@ -122,11 +121,11 @@ class WESplClient { return fRowTx; } - uint32_t getBytesRcv() const + messageqcpp::BSSizeType getBytesRcv() const { return fBytesRcv; } - uint32_t getBytesTx() + messageqcpp::BSSizeType getBytesTx() { boost::mutex::scoped_lock aLock(fTxMutex); return fBytesTx; @@ -214,17 +213,17 @@ class WESplClient { return fIpAddress; } - void setBytesRcv(uint32_t BytesRcv) + void setBytesRcv(messageqcpp::BSSizeType BytesRcv) { fBytesRcv = BytesRcv; } - void setBytesTx(uint32_t BytesTx) + void setBytesTx(messageqcpp::BSSizeType BytesTx) { boost::mutex::scoped_lock aLock(fTxMutex); fBytesTx = BytesTx; aLock.unlock(); } - void updateBytesTx(uint32_t fBytes) + void updateBytesTx(messageqcpp::BSSizeType fBytes) { boost::mutex::scoped_lock aLock(fTxMutex); fBytesTx += fBytes; @@ -358,8 +357,8 @@ class WESplClient int fDataRqstCnt; // Data request count long fRdSecTo; // read timeout sec unsigned int fRowTx; // No. Of Rows Transmitted - uint32_t fBytesTx; - uint32_t fBytesRcv; + messageqcpp::BSSizeType fBytesTx; + messageqcpp::BSSizeType fBytesRcv; time_t fLastInTime; time_t fStartTime; bool fSend; From a6eb5ca689c80748819c829a41ed5855f53ad0bd Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Sat, 9 Nov 2024 22:47:04 +0300 Subject: [PATCH 14/65] MCOL-5719: Move ownership mechanism to KV storage. (#3266) * MCOL-5719 Move ownership mechanism to FDB --- CMakeLists.txt | 3 +- build/bootstrap_mcs.sh | 1 + storage-manager/CMakeLists.txt | 8 +- storage-manager/src/KVPrefixes.cpp | 24 +++ storage-manager/src/KVPrefixes.hpp | 29 +++ storage-manager/src/KVStorageInitializer.cpp | 64 +++++++ storage-manager/src/KVStorageInitializer.h | 38 ++++ storage-manager/src/Ownership.cpp | 177 ++++++++++++------- storage-manager/src/Ownership.h | 9 + storage-manager/storagemanager.cnf.in | 4 + utils/fdb_wrapper_cpp/test/test_fdb_api.cpp | 2 +- 11 files changed, 287 insertions(+), 72 deletions(-) create mode 100644 storage-manager/src/KVPrefixes.cpp create mode 100644 storage-manager/src/KVPrefixes.hpp create mode 100644 storage-manager/src/KVStorageInitializer.cpp create mode 100644 storage-manager/src/KVStorageInitializer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 70f9c0b8b..0f8cd0737 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -357,6 +357,7 @@ SET (ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchlo SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup") SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") +SET (ENGINE_UTILS_FDB_WRAPPER "${CMAKE_CURRENT_SOURCE_DIR}/utils/fdb_wrapper_cpp/include") SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") SET (ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache") @@ -374,7 +375,7 @@ SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk" SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) -SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) +SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE} ${ENGINE_UTILS_FDB_WRAPPER}) ADD_SUBDIRECTORY(dbcon/mysql) IF(NOT TARGET columnstore) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 3bae32e9d..a95901b3b 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -212,6 +212,7 @@ clean_old_installation() rm -rf /etc/mysql rm -rf /etc/my.cnf.d/columnstore.cnf rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf + fdbcli --exec "writemode on; clearrange SM_A SM_z" } build() diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 4297081c5..707595b70 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -38,6 +38,8 @@ set(storagemanager_SRCS src/SyncTask.cpp src/ListIOTask.cpp src/TerminateIOTask.cpp + src/KVStorageInitializer.cpp + src/KVPrefixes.cpp ../utils/common/crashtrace.cpp ) @@ -62,13 +64,11 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/storagemanager.cnf.in" "${CMAKE_CURR link_directories(${CMAKE_BINARY_DIR}/lib) set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) - - add_library(storagemanager SHARED ${storagemanager_SRCS}) -add_dependencies(storagemanager marias3 external_boost) +add_dependencies(storagemanager marias3 external_boost fdbcs) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) +target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread fdbcs ${S3API_DEPS}) add_executable(StorageManager src/main.cpp) target_link_libraries(StorageManager storagemanager) diff --git a/storage-manager/src/KVPrefixes.cpp b/storage-manager/src/KVPrefixes.cpp new file mode 100644 index 000000000..8183d9f69 --- /dev/null +++ b/storage-manager/src/KVPrefixes.cpp @@ -0,0 +1,24 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "KVPrefixes.hpp" + +namespace storagemanager +{ +// FDB recommends keep the key size up to 32 bytes. +const char* KVPrefixes[2] = {/*OWNERSHIP*/ "SM_O_", /*META*/ "SM_M_"}; +} // namespace storagemanager diff --git a/storage-manager/src/KVPrefixes.hpp b/storage-manager/src/KVPrefixes.hpp new file mode 100644 index 000000000..92d8b191c --- /dev/null +++ b/storage-manager/src/KVPrefixes.hpp @@ -0,0 +1,29 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +namespace storagemanager +{ +enum class KVPrefixId +{ + SM_OWNERSHIP = 0, + SM_META +}; + +extern const char* KVPrefixes[]; +} // namespace storagemanager diff --git a/storage-manager/src/KVStorageInitializer.cpp b/storage-manager/src/KVStorageInitializer.cpp new file mode 100644 index 000000000..77f9af8a3 --- /dev/null +++ b/storage-manager/src/KVStorageInitializer.cpp @@ -0,0 +1,64 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#include "Config.h" +#include "SMLogging.h" +#include "KVStorageInitializer.h" + +namespace storagemanager +{ +static std::shared_ptr fdbDataBaseInstance; +static std::unique_ptr fdbNetworkInstance; +static std::mutex kvStorageLock; + +static void logError(SMLogging* logger, const char* errorMsg) +{ + logger->log(LOG_CRIT, errorMsg); + throw std::runtime_error(errorMsg); +} + +std::shared_ptr KVStorageInitializer::getStorageInstance() +{ + if (fdbDataBaseInstance) + return fdbDataBaseInstance; + + auto* config = Config::get(); + auto* logger = SMLogging::get(); + + std::unique_lock lock(kvStorageLock); + if (fdbDataBaseInstance) + return fdbDataBaseInstance; + + if (!FDBCS::setAPIVersion()) + logError(logger, "Ownership: FDB setAPIVersion failed."); + + fdbNetworkInstance = std::make_unique(); + if (!fdbNetworkInstance->setUpAndRunNetwork()) + logError(logger, "Ownership: FDB setUpAndRunNetwork failed."); + + std::string clusterFilePath = config->getValue("ObjectStorage", "fdb_cluster_file_path"); + if (clusterFilePath.empty()) + logError(logger, + "Ownership: Need to specify `Foundation DB cluster file path` in the storagemanager.cnf file"); + + fdbDataBaseInstance = FDBCS::DataBaseCreator::createDataBase(clusterFilePath); + if (!fdbDataBaseInstance) + logError(logger, "Ownership: FDB createDataBase failed."); + + return fdbDataBaseInstance; +} + +} // namespace storagemanager diff --git a/storage-manager/src/KVStorageInitializer.h b/storage-manager/src/KVStorageInitializer.h new file mode 100644 index 000000000..cd1d829a0 --- /dev/null +++ b/storage-manager/src/KVStorageInitializer.h @@ -0,0 +1,38 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once +#include +#include "fdbcs.hpp" + +namespace storagemanager +{ +// Represnts a `Key/Value` storage initializer based on Singleton pattern. +// Initializes all needed `Key/Value` storage machinery once and return pointer +// to the `Key/Value` storage. +class KVStorageInitializer +{ + public: + static std::shared_ptr getStorageInstance(); + KVStorageInitializer() = delete; + KVStorageInitializer(const KVStorageInitializer&) = delete; + KVStorageInitializer& operator=(KVStorageInitializer&) = delete; + KVStorageInitializer(KVStorageInitializer&&) = delete; + KVStorageInitializer& operator=(KVStorageInitializer&&) = delete; +}; + +} // namespace storagemanager diff --git a/storage-manager/src/Ownership.cpp b/storage-manager/src/Ownership.cpp index cadcebc22..0fe670b76 100644 --- a/storage-manager/src/Ownership.cpp +++ b/storage-manager/src/Ownership.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2019 MariaDB Corporation +/* Copyright (C) 2024 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -19,6 +19,9 @@ #include "Config.h" #include "Cache.h" #include "Synchronizer.h" +#include "KVStorageInitializer.h" +#include "KVPrefixes.hpp" +#include "fdbcs.hpp" #include #include #include @@ -30,6 +33,31 @@ namespace bf = boost::filesystem; namespace storagemanager { +// FDB recommends keep the key size up to 32 bytes. +const char* ownerShipStates[3] = {/*OWNED*/ "_O", /*FLUSHING*/ "_F", /*REQUEST_TRANSFER*/ "_RT"}; + +inline std::string getKeyName(const std::string& fileName, OwnershipStateId state) +{ + return KVPrefixes[static_cast(KVPrefixId::SM_OWNERSHIP)] + fileName + + ownerShipStates[static_cast(state)]; +} + +inline void Ownership::removeKeys(const std::string& fileName, const std::vector& states) +{ + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + + for (const auto state : states) + tnx->remove(getKeyName(fileName, state)); + + if (!tnx->commit()) + { + const char* msg = "Ownership: commit `removeKeys` failed "; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } +} + Ownership::Ownership() { Config* config = Config::get(); @@ -61,6 +89,7 @@ Ownership::Ownership() logger->log(LOG_CRIT, msg); throw runtime_error(msg); } + monitor = new Monitor(this); } @@ -124,28 +153,22 @@ bf::path Ownership::get(const bf::path& p, bool getOwnership) return ret; } -// minor timesaver -#define TOUCH(p, f) \ - { \ - int fd = ::open((metadataPrefix / p / f).string().c_str(), O_TRUNC | O_CREAT | O_WRONLY, 0660); \ - if (fd >= 0) \ - ::close(fd); \ - else \ - { \ - char buf[80]; \ - int saved_errno = errno; \ - cerr << "failed to touch " << metadataPrefix / p / f << " got " << strerror_r(saved_errno, buf, 80) \ - << endl; \ - } \ - } - -#define DELETE(p, f) ::unlink((metadataPrefix / p / f).string().c_str()); - void Ownership::touchFlushing(const bf::path& prefix, volatile bool* doneFlushing) const { while (!*doneFlushing) { - TOUCH(prefix, "FLUSHING"); + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string flushingKey = getKeyName(prefix.string(), OwnershipStateId::FLUSHING); + tnx->set(flushingKey, ""); + if (!tnx->commit()) + { + const char* msg = "Ownership: commit `touchFlushing` failed "; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + } try { boost::this_thread::sleep_for(boost::chrono::seconds(1)); @@ -171,10 +194,7 @@ void Ownership::releaseOwnership(const bf::path& p, bool isDtor) if (isDtor) { - // This is a quick release. If this is being destroyed, then it is through the graceful - // shutdown mechanism, which will flush data separately. - DELETE(p, "OWNED"); - DELETE(p, "FLUSHING"); + removeKeys(p.string(), {OwnershipStateId::OWNED, OwnershipStateId::FLUSHING}); return; } else @@ -191,19 +211,28 @@ void Ownership::releaseOwnership(const bf::path& p, bool isDtor) done = true; xfer.interrupt(); xfer.join(); - - // update state - DELETE(p, "OWNED"); - DELETE(p, "FLUSHING"); + removeKeys(p.string(), {OwnershipStateId::OWNED, OwnershipStateId::FLUSHING}); } void Ownership::_takeOwnership(const bf::path& p) { logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str()); - DELETE(p, "FLUSHING"); - DELETE(p, "REQUEST_TRANSFER"); - // TODO: need to consider errors taking ownership - TOUCH(p, "OWNED"); + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED); + const std::string flushingKey = getKeyName(p.string(), OwnershipStateId::FLUSHING); + const std::string requestTransferKey = getKeyName(p.string(), OwnershipStateId::REQUEST_TRANSFER); + tnx->remove(flushingKey); + tnx->remove(requestTransferKey); + tnx->set(ownedKey, ""); + if (!tnx->commit()) + { + const char* msg = "Ownership: commit `_takeOwnership` transaction failed"; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + } mutex.lock(); ownedPrefixes[p] = true; mutex.unlock(); @@ -225,45 +254,62 @@ void Ownership::takeOwnership(const bf::path& p) ownedPrefixes[p] = NULL; s.unlock(); - bool okToTransfer = false; - struct stat statbuf; - int err; - char buf[80]; - bf::path ownedPath = metadataPrefix / p / "OWNED"; - bf::path flushingPath = metadataPrefix / p / "FLUSHING"; + bool ownedKeyExists; + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED); + ownedKeyExists = tnx->get(ownedKey).first; + } // if it's not already owned, then we can take possession - err = ::stat(ownedPath.string().c_str(), &statbuf); - if (err && errno == ENOENT) + if (!ownedKeyExists) { _takeOwnership(p); return; } - TOUCH(p, "REQUEST_TRANSFER"); + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string requestTransferKey = getKeyName(p.string(), OwnershipStateId::REQUEST_TRANSFER); + tnx->set(requestTransferKey, ""); + if (!tnx->commit()) + { + const char* msg = "Ownership: commit `requestTransfer` failed "; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + } + + bool okToTransfer = false; time_t lastFlushTime = time(NULL); while (!okToTransfer && time(NULL) < lastFlushTime + 10) { // if the OWNED file is deleted or if the flushing file isn't touched after 10 secs // it is ok to take possession. - err = ::stat(ownedPath.string().c_str(), &statbuf); - if (err) + bool ownedKeyExists; { - if (errno == ENOENT) - okToTransfer = true; - else - logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", - strerror_r(errno, buf, 80), ownedPath.string().c_str()); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED); + ownedKeyExists = tnx->get(ownedKey).first; } - err = ::stat(flushingPath.string().c_str(), &statbuf); - if (err && errno != ENOENT) - logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", - strerror_r(errno, buf, 80), flushingPath.string().c_str()); - else + if (!ownedKeyExists) + okToTransfer = true; + + bool flushingKeyExists; + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string flushingKey = getKeyName(p.string(), OwnershipStateId::FLUSHING); + flushingKeyExists = tnx->get(flushingKey).first; + } + if (flushingKeyExists) { logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str()); - if (!err) - lastFlushTime = statbuf.st_mtime; + // Since notice the flushing key. + lastFlushTime = time(NULL); } if (!okToTransfer) sleep(1); @@ -286,9 +332,6 @@ Ownership::Monitor::~Monitor() void Ownership::Monitor::watchForInterlopers() { // look for requests to transfer ownership - struct stat statbuf; - int err; - char buf[80]; vector releaseList; while (!stop) @@ -302,17 +345,20 @@ void Ownership::Monitor::watchForInterlopers() break; if (prefix.second == false) continue; - bf::path p(owner->metadataPrefix / (prefix.first) / "REQUEST_TRANSFER"); - const char* cp = p.string().c_str(); - err = ::stat(cp, &statbuf); + bool requestKeyExists; + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string requestTransferKey = + getKeyName(prefix.first.string(), OwnershipStateId::REQUEST_TRANSFER); + + requestKeyExists = tnx->get(requestTransferKey).first; + } // release it if there's a release request only. Log it if there's an error other than // that the file isn't there. - if (err == 0) + if (requestKeyExists) releaseList.push_back(prefix.first); - if (err < 0 && errno != ENOENT) - owner->logger->log(LOG_ERR, "Runner::watchForInterlopers(): failed to stat %s, got %s", cp, - strerror_r(errno, buf, 80)); } s.unlock(); @@ -329,5 +375,4 @@ void Ownership::Monitor::watchForInterlopers() } } } - -} // namespace storagemanager +} // namespace storagemanager \ No newline at end of file diff --git a/storage-manager/src/Ownership.h b/storage-manager/src/Ownership.h index f1e3a5c74..f3b6a2d97 100644 --- a/storage-manager/src/Ownership.h +++ b/storage-manager/src/Ownership.h @@ -22,12 +22,20 @@ #include #include #include "SMLogging.h" +#include "fdbcs.hpp" /* This class tracks the ownership of each prefix and manages ownership transfer. Could we come up with a better name btw? */ namespace storagemanager { +enum class OwnershipStateId +{ + OWNED = 0, + FLUSHING, + REQUEST_TRANSFER +}; + class Ownership : public boost::noncopyable { public: @@ -50,6 +58,7 @@ class Ownership : public boost::noncopyable void takeOwnership(const boost::filesystem::path&); void releaseOwnership(const boost::filesystem::path&, bool isDtor = false); void _takeOwnership(const boost::filesystem::path&); + inline void removeKeys(const std::string& fileName, const std::vector& states); struct Monitor { diff --git a/storage-manager/storagemanager.cnf.in b/storage-manager/storagemanager.cnf.in index 04347aced..8999c7e63 100644 --- a/storage-manager/storagemanager.cnf.in +++ b/storage-manager/storagemanager.cnf.in @@ -80,6 +80,10 @@ max_concurrent_uploads = 21 # negotiate ownership of data from a failed instance. common_prefix_depth = 3 +# This value is used to specify a cluster file path to FDB cluster. +# See more: https://apple.github.io/foundationdb/administration.html +fdb_cluster_file_path = /etc/foundationdb/fdb.cluster + [S3] # These should be self-explanatory. Region can be blank or commented # if using a private cloud storage system. Bucket has to be set to diff --git a/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp b/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp index 98ca763f4..dc9edb3e7 100644 --- a/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp +++ b/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp @@ -16,7 +16,7 @@ MA 02110-1301, USA. */ #include -#include "../include/fdbcs.hpp" +#include "fdbcs.hpp" using namespace std; using namespace FDBCS; From 6f4274760efb916377d5c218bed7275f2e2cfda8 Mon Sep 17 00:00:00 2001 From: Alexey Antipovsky Date: Sun, 10 Nov 2024 19:31:38 +0100 Subject: [PATCH 15/65] fix(plugin): MCOL-5703 fix server crash on 'UNION ALL VALUES' queries (#3335) * fix(plugin): MCOL-5703 fix server crash on 'UNION ALL VALUES' queries --- dbcon/mysql/ha_mcs_pushdown.cpp | 17 +++++++----- .../columnstore/bugfixes/mcol-5703.result | 16 ++++++++++++ .../columnstore/bugfixes/mcol-5703.test | 26 +++++++++++++++++++ 3 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 mysql-test/columnstore/bugfixes/mcol-5703.result create mode 100644 mysql-test/columnstore/bugfixes/mcol-5703.test diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index cbf4b66d6..c4300f254 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -880,7 +880,7 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex create_explain_query_if_not_exists(thd->lex, thd->mem_root); Query_arena *arena, backup; arena = thd->activate_stmt_arena_if_needed(&backup); - COND* conds = join->conds; + COND* conds = join ? join->conds : nullptr; select_lex->where = conds; if (isPS) @@ -888,7 +888,11 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex select_lex->prep_where = conds ? conds->copy_andor_structure(thd) : 0; } - select_lex->update_used_tables(); + // no join indicates that there is no tables to update + if (join) + { + select_lex->update_used_tables(); + } if (arena) thd->restore_active_arena(arena, &backup); @@ -914,7 +918,8 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex JOIN* join = select_lex->join; // This is taken from JOIN::optimize() - join->fields = &select_lex->item_list; + if (join) + join->fields = &select_lex->item_list; // Instantiate handler::table, which is the place for the result set. if ((i == 0) && handler->prepare()) @@ -957,7 +962,7 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex return handler; } - if (!join->zero_result_cause && join->exec_const_cond && !join->exec_const_cond->val_int()) + if (join && !join->zero_result_cause && join->exec_const_cond && !join->exec_const_cond->val_int()) join->zero_result_cause = "Impossible WHERE noticed after reading const tables"; // We've called exec_const_cond->val_int(). This may have caused an error. @@ -968,7 +973,7 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex return handler; } - if (join->zero_result_cause) + if (join && join->zero_result_cause) { if (join->select_lex->have_window_funcs() && join->send_row_on_empty_set()) { @@ -988,7 +993,7 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex } } - if ((join->select_lex->options & OPTION_SCHEMA_TABLE) && + if (join && (join->select_lex->options & OPTION_SCHEMA_TABLE) && get_schema_tables_result(join, PROCESSED_BY_JOIN_EXEC)) { if (!thd->is_error()) diff --git a/mysql-test/columnstore/bugfixes/mcol-5703.result b/mysql-test/columnstore/bugfixes/mcol-5703.result new file mode 100644 index 000000000..28e35bf4b --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-5703.result @@ -0,0 +1,16 @@ +DROP DATABASE IF EXISTS mcol5703; +CREATE DATABASE mcol5703; +USE mcol5703; +CREATE TABLE `doubles` ( +`a` int(11), +`b` int(11) +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci; +INSERT INTO doubles VALUES (1, 2), (2, 4); +SELECT a, b FROM doubles UNION VALUES (3, 6); +ERROR 42000: The storage engine for the table doesn't support subquery with VALUES +SELECT a, b FROM doubles UNION ALL VALUES (3, 6); +ERROR 42000: The storage engine for the table doesn't support subquery with VALUES +SELECT a, b FROM doubles UNION DISTINCT VALUES (3, 6); +ERROR 42000: The storage engine for the table doesn't support subquery with VALUES +DROP TABLE doubles; +DROP DATABASE mcol5703; diff --git a/mysql-test/columnstore/bugfixes/mcol-5703.test b/mysql-test/columnstore/bugfixes/mcol-5703.test new file mode 100644 index 000000000..279392e81 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-5703.test @@ -0,0 +1,26 @@ +--source ../include/have_columnstore.inc + +--disable_warnings +DROP DATABASE IF EXISTS mcol5703; +--enable_warnings + +CREATE DATABASE mcol5703; +USE mcol5703; + + +CREATE TABLE `doubles` ( + `a` int(11), + `b` int(11) +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci; +INSERT INTO doubles VALUES (1, 2), (2, 4); + +--ERROR 1178 +SELECT a, b FROM doubles UNION VALUES (3, 6); +--ERROR 1178 +SELECT a, b FROM doubles UNION ALL VALUES (3, 6); +--ERROR 1178 +SELECT a, b FROM doubles UNION DISTINCT VALUES (3, 6); + +DROP TABLE doubles; +DROP DATABASE mcol5703; + From c6b747b7c0d954a4c6e6776b23de8deb7cd7caa4 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Mon, 11 Nov 2024 18:29:53 +0000 Subject: [PATCH 16/65] fix(DEC):MCOL-5805,5808 to resolve UM-only node crash inside DEC when there is no local PP to send the local requests to. (#3349) --- dbcon/joblist/distributedenginecomm.cpp | 21 +++++++++++++++++---- dbcon/joblist/distributedenginecomm.h | 7 ++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/dbcon/joblist/distributedenginecomm.cpp b/dbcon/joblist/distributedenginecomm.cpp index b9a157934..c718deaa0 100644 --- a/dbcon/joblist/distributedenginecomm.cpp +++ b/dbcon/joblist/distributedenginecomm.cpp @@ -181,6 +181,17 @@ struct QueueShutdown namespace joblist { + +bool isDefaultLocalConnectionId(const uint32_t connectionId) +{ + return defaultLocalConnectionId() == connectionId; +} + +bool needToSendToLocalPM(const bool isExeMgr, const uint32_t connectionId) +{ + return isExeMgr && !isDefaultLocalConnectionId(connectionId); +} + DistributedEngineComm* DistributedEngineComm::fInstance = 0; /*static*/ @@ -725,7 +736,7 @@ void DistributedEngineComm::sendAcks(uint32_t uniqueID, const vector& msgs, nextPMToACK(mqe, l_msgCount, &sockIndex, toAck); idbassert(*toAck <= l_msgCount); l_msgCount -= *toAck; - if (sockIndex == localConnectionId_ && fIsExeMgr) + if (sockIndex == localConnectionId_ && needToSendToLocalPM(fIsExeMgr, sockIndex)) { sendToLocal = true; continue; @@ -766,7 +777,7 @@ void DistributedEngineComm::sendAcks(uint32_t uniqueID, const vector& msgs, writeToClient(i, ackCommand); } } - if (!pmAcked[localConnectionId_] && fIsExeMgr) + if (needToSendToLocalPM(fIsExeMgr, localConnectionId_) && !pmAcked[localConnectionId_]) { writeToClient(localConnectionId_, msg); } @@ -857,8 +868,10 @@ void DistributedEngineComm::setFlowControl(bool enabled, uint32_t uniqueID, boos } writeToClient(i, msg); } - if (fIsExeMgr) + if (needToSendToLocalPM(fIsExeMgr, localConnectionId_)) + { writeToClient(localConnectionId_, msg); + } } int32_t DistributedEngineComm::write(uint32_t senderID, const SBS& msg) @@ -898,7 +911,7 @@ int32_t DistributedEngineComm::write(uint32_t senderID, const SBS& msg) return rc; } } - if (fIsExeMgr) + if (needToSendToLocalPM(fIsExeMgr, localConnectionId_)) { return writeToClient(localConnectionId_, msg); } diff --git a/dbcon/joblist/distributedenginecomm.h b/dbcon/joblist/distributedenginecomm.h index f48295bea..d5c687b87 100644 --- a/dbcon/joblist/distributedenginecomm.h +++ b/dbcon/joblist/distributedenginecomm.h @@ -69,6 +69,11 @@ class Config; */ namespace joblist { +constexpr uint32_t defaultLocalConnectionId() +{ + return std::numeric_limits::max(); +} + class DECEventListener { public: @@ -315,7 +320,7 @@ class DistributedEngineComm boost::mutex ackLock; // localConnectionId_ is set running Setup() method - uint32_t localConnectionId_ = std::numeric_limits::max(); + uint32_t localConnectionId_ = defaultLocalConnectionId(); std::vector localNetIfaceSins_; std::mutex inMemoryEM2PPExchMutex_; std::condition_variable inMemoryEM2PPExchCV_; From 932546f47cb5f3fe4579fbd4dd38292220ed72c7 Mon Sep 17 00:00:00 2001 From: Alan Mologorsky <89034356+mariadb-AlanMologorsky@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:56:16 +0300 Subject: [PATCH 17/65] fix(CMAPI): reduce timeout value for couple CMAPI endpoints (#3153) * Minor fix in timeout for /notfound "endpoint" call. - [fix] timeout to urlopen /notfound "endpoint" in mcs-savebrm and mcs-loadbrm * chore(CMAPI): reduced a second Time Out asking MDB for a async replication status. The latency must be small b/c MDB Server to be requested is local. --------- Co-authored-by: drrtuy --- oam/install_scripts/mcs-loadbrm.py.in | 6 +++--- oam/install_scripts/mcs-savebrm.py.in | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/oam/install_scripts/mcs-loadbrm.py.in b/oam/install_scripts/mcs-loadbrm.py.in index 46bb6f5f8..da01da7d7 100755 --- a/oam/install_scripts/mcs-loadbrm.py.in +++ b/oam/install_scripts/mcs-loadbrm.py.in @@ -37,7 +37,7 @@ S3_DBROOT1_BRM_PATH = 'data1/systemFiles/dbrm/BRM_saves_current' USER = '@DEFAULT_USER@' GROUP = '@DEFAULT_GROUP@' MINUTE = 60 -HALF_A_MINUTE = 30 +FIVE_SECS = 5 UNREASONABLE_DELAY = 600 LOCALHOST = '127.0.0.1' API_VERSION = '0.4.0' @@ -76,7 +76,7 @@ def cmapi_available(host=LOCALHOST): request = Request(method='POST', url=url) ctx = get_unverified_context() try: - with urlopen(request, context=ctx, timeout=HALF_A_MINUTE) as req: + with urlopen(request, context=ctx, timeout=FIVE_SECS) as req: _ = req.read().decode('utf-8') except HTTPError as exc: if exc.code == 404: @@ -165,7 +165,7 @@ def is_node_primary(conf_root): success = False try: - with urlopen(request, context=ctx, timeout=HALF_A_MINUTE) as req: + with urlopen(request, context=ctx, timeout=FIVE_SECS) as req: response = req.read() success = True except HTTPError as exc: diff --git a/oam/install_scripts/mcs-savebrm.py.in b/oam/install_scripts/mcs-savebrm.py.in index 28a2e46b5..a3a15b6df 100755 --- a/oam/install_scripts/mcs-savebrm.py.in +++ b/oam/install_scripts/mcs-savebrm.py.in @@ -25,7 +25,7 @@ MCS_BIN_DIR = '@ENGINE_BINDIR@' SAVEBRM = os.path.join(MCS_BIN_DIR, 'save_brm') EM_FILE_SUFFIX = '_em' EM_FILE_SIZE_THRESHOLD = 1000 -HALF_A_MINUTE = 30 +FIVE_SECS = 5 NUMBER_OF_FILES_TO_KEEP = 40 DEFAULT_EM_LOCAL_PATH_PREFIX = '' LOCALHOST = '127.0.0.1' @@ -73,7 +73,7 @@ def cmapi_available(): request = Request(method='POST', url=url) ctx = get_unverified_context() try: - with urlopen(request, context=ctx, timeout=HALF_A_MINUTE) as req: + with urlopen(request, context=ctx, timeout=FIVE_SECS) as req: _ = req.read().decode('utf-8') except HTTPError as exc: if exc.code == 404: @@ -164,7 +164,7 @@ def is_node_primary(conf_root): success = False try: - with urlopen(request, context=ctx, timeout=HALF_A_MINUTE) as req: + with urlopen(request, context=ctx, timeout=FIVE_SECS) as req: response = req.read() success = True except HTTPError as exc: From 4f71071766499e687149cf4c4c55ba446a10af55 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Sun, 10 Nov 2024 13:51:17 +0000 Subject: [PATCH 18/65] feat(bootstrap): add flags to bypass costly operations to reduce recompilation time --- build/bootstrap_mcs.sh | 141 ++++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 66 deletions(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index a95901b3b..3821eef46 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -49,6 +49,9 @@ optparse.define short=F long=show-build-flags desc="Print CMake flags, while bui optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true optparse.define short=f long=do-not-freeze-revision desc="Disable revision freezing, or do not set 'update none' for columnstore submodule in MDB repository" variable=DO_NOT_FREEZE_REVISION default=false value=true optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$MDB_SOURCE_PATH/../MariaDBBuild +optparse.define short=o long=recompile-only variable=RECOMPILE_ONLY default=false value=true +optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false + source $( optparse.build ) @@ -212,7 +215,6 @@ clean_old_installation() rm -rf /etc/mysql rm -rf /etc/my.cnf.d/columnstore.cnf rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf - fdbcli --exec "writemode on; clearrange SM_A SM_z" } build() @@ -450,9 +452,10 @@ fix_config_files() message "UBSAN options were added to $MDB_SERVICE_FILE" fi fi - - message Reloading systemd - systemctl daemon-reload + if [[ $RECOMPILE_ONLY = false ]] ; then + message Reloading systemd + systemctl daemon-reload + fi } make_dir() @@ -463,78 +466,80 @@ make_dir() install() { - message_split - message "Installing MariaDB" - disable_plugins_for_bootstrap + if [[ $RECOMPILE_ONLY = false ]] ; then + message_split + message "Installing MariaDB" + disable_plugins_for_bootstrap - make_dir $REPORT_PATH - chmod 777 $REPORT_PATH + make_dir $REPORT_PATH + chmod 777 $REPORT_PATH - check_user_and_group mysql - check_user_and_group syslog + check_user_and_group mysql + check_user_and_group syslog - make_dir $CONFIG_DIR + make_dir $CONFIG_DIR - echo "[client-server] -socket=/run/mysqld/mysqld.sock" > $CONFIG_DIR/socket.cnf + echo "[client-server] + socket=/run/mysqld/mysqld.sock" > $CONFIG_DIR/socket.cnf - mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so - make_dir /var/lib/mysql + mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so + make_dir /var/lib/mysql - message "Running mysql_install_db" - sudo -u mysql mysql_install_db --rpm --user=mysql > /dev/null - mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so + message "Running mysql_install_db" + sudo -u mysql mysql_install_db --rpm --user=mysql > /dev/null + mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so - enable_columnstore_back + enable_columnstore_back - make_dir /etc/columnstore + make_dir /etc/columnstore - cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml - cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf - cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/ - cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/ + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then - make_dir /usr/share/mysql - make_dir /etc/mysql/ - cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh - cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start - > /etc/mysql/debian.cnf + if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then + make_dir /usr/share/mysql + make_dir /etc/mysql/ + cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh + cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start + > /etc/mysql/debian.cnf + fi + + fix_config_files + + make_dir /etc/my.cnf.d + if [ -d "/etc/mysql/mariadb.conf.d/" ]; then + message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d" + cp -rp /etc/mysql/mariadb.conf.d/* /etc/my.cnf.d + fi + + if [ -d "/etc/mysql/conf.d/" ]; then + message "Copying configs from /etc/mysql/conf.d/ to /etc/my.cnf.d" + cp -rp /etc/mysql/conf.d/* /etc/my.cnf.d + fi + + make_dir /var/lib/columnstore/data1 + make_dir /var/lib/columnstore/data1/systemFiles + make_dir /var/lib/columnstore/data1/systemFiles/dbrm + make_dir /run/mysqld/ + make_dir $DATA_DIR + + chmod +x $INSTALL_PREFIX/bin/mariadb* + + ldconfig + + start_storage_manager_if_needed + + message "Running columnstore-post-install" + make_dir /var/lib/columnstore/local + columnstore-post-install --rpmmode=install + message "Running install_mcs_mysql" + install_mcs_mysql.sh fi - fix_config_files - - make_dir /etc/my.cnf.d - if [ -d "/etc/mysql/mariadb.conf.d/" ]; then - message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d" - cp -rp /etc/mysql/mariadb.conf.d/* /etc/my.cnf.d - fi - - if [ -d "/etc/mysql/conf.d/" ]; then - message "Copying configs from /etc/mysql/conf.d/ to /etc/my.cnf.d" - cp -rp /etc/mysql/conf.d/* /etc/my.cnf.d - fi - - make_dir /var/lib/columnstore/data1 - make_dir /var/lib/columnstore/data1/systemFiles - make_dir /var/lib/columnstore/data1/systemFiles/dbrm - make_dir /run/mysqld/ - make_dir $DATA_DIR - - chmod +x $INSTALL_PREFIX/bin/mariadb* - - ldconfig - - start_storage_manager_if_needed - - message "Running columnstore-post-install" - make_dir /var/lib/columnstore/local - columnstore-post-install --rpmmode=install - message "Running install_mcs_mysql" - install_mcs_mysql.sh - chown -R syslog:syslog /var/log/mariadb/ chmod 777 /var/log/mariadb/ chmod 777 /var/log/mariadb/columnstore @@ -579,7 +584,9 @@ if [[ $INSTALL_DEPS = true ]] ; then install_deps fi -stop_service +if [[ $RESTART_SERVICES = true ]] ; then + stop_service +fi if [[ $NO_CLEAN = false ]] ; then clean_old_installation @@ -589,8 +596,10 @@ build run_unit_tests run_microbenchmarks_tests install -start_service -smoke -generate_svgs +if [[ $RESTART_SERVICES = true ]] ; then + start_service + smoke + generate_svgs +fi message_splitted "FINISHED" From 9d595910668f9a8847b19073c6bf93854bc37a8b Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Fri, 29 Nov 2024 13:16:52 +0300 Subject: [PATCH 19/65] Fix tests' results to match new diagnostics --- .../basic/r/mcs17_procedures_define_call_drop.result | 2 +- .../columnstore/basic/r/mcs31_update_table_negative.result | 4 ++-- mysql-test/columnstore/basic/r/mcs32_delete_table.result | 2 +- mysql-test/columnstore/basic/r/mcs33_select_basic.result | 2 +- mysql-test/columnstore/basic/r/mcs35_select_group_by.result | 2 +- .../basic/r/mcs36_select_order_by_group_by.result | 4 ++-- .../columnstore/basic/r/mcs65_crossengine_order_by.result | 6 +++--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mysql-test/columnstore/basic/r/mcs17_procedures_define_call_drop.result b/mysql-test/columnstore/basic/r/mcs17_procedures_define_call_drop.result index e0a97b55a..d2a8cced1 100644 --- a/mysql-test/columnstore/basic/r/mcs17_procedures_define_call_drop.result +++ b/mysql-test/columnstore/basic/r/mcs17_procedures_define_call_drop.result @@ -19,7 +19,7 @@ ERROR 42000: OUT or INOUT argument 1 for routine mcs17_db.proc1 is not a variabl CALL proc1(); ERROR 42000: Incorrect number of arguments for PROCEDURE mcs17_db.proc1; expected 1, got 0 CALL proc1(cnt); -ERROR 42S22: Unknown column 'cnt' in 'field list' +ERROR 42S22: Unknown column 'cnt' in 'CALL' CREATE PROCEDURE proc1 (OUT cnt INT) COMMENT 'comment2' BEGIN SELECT COUNT(*) INTO cnt FROM t1; diff --git a/mysql-test/columnstore/basic/r/mcs31_update_table_negative.result b/mysql-test/columnstore/basic/r/mcs31_update_table_negative.result index 4555d9a57..e298e4e02 100644 --- a/mysql-test/columnstore/basic/r/mcs31_update_table_negative.result +++ b/mysql-test/columnstore/basic/r/mcs31_update_table_negative.result @@ -8,9 +8,9 @@ CREATE TABLE t1 (a INT NOT NULL, b INT) ENGINE=Columnstore; INSERT INTO t1 VALUES (1,1),(1,2),(1,3),(3,1),(3,2); CREATE table t2 LIKE t1; UPDATE t1 SET nonexistingColumn = 1; -ERROR 42S22: Unknown column 'nonexistingColumn' in 'field list' +ERROR 42S22: Unknown column 'nonexistingColumn' in 'SET' UPDATE t2 SET a = (SELECT nonexistingColumn FROM t1); -ERROR 42S22: Unknown column 'nonexistingColumn' in 'field list' +ERROR 42S22: Unknown column 'nonexistingColumn' in 'SELECT' UPDATE t1 SET a = NULL; ERROR HY000: Internal error: CAL0002: Update Failed: MCS-4015: Column 'a' cannot be null. UPDATE t1 SET a = NULL, b =1; diff --git a/mysql-test/columnstore/basic/r/mcs32_delete_table.result b/mysql-test/columnstore/basic/r/mcs32_delete_table.result index d19a02461..5f22a4ea0 100644 --- a/mysql-test/columnstore/basic/r/mcs32_delete_table.result +++ b/mysql-test/columnstore/basic/r/mcs32_delete_table.result @@ -13,7 +13,7 @@ SELECT COUNT(*) FROM t1; COUNT(*) 2 DELETE FROM t1 WHERE unknown = 2; -ERROR 42S22: Unknown column 'unknown' in 'where clause' +ERROR 42S22: Unknown column 'unknown' in 'WHERE' INSERT INTO t1 VALUES (1, NULL); DELETE FROM t1 WHERE b IS NULL; SELECT COUNT(*) FROM t1; diff --git a/mysql-test/columnstore/basic/r/mcs33_select_basic.result b/mysql-test/columnstore/basic/r/mcs33_select_basic.result index 423bb311e..eb2b645fc 100644 --- a/mysql-test/columnstore/basic/r/mcs33_select_basic.result +++ b/mysql-test/columnstore/basic/r/mcs33_select_basic.result @@ -86,5 +86,5 @@ SELECT @x / @y; @x / @y NULL SELECT unknown; -ERROR 42S22: Unknown column 'unknown' in 'field list' +ERROR 42S22: Unknown column 'unknown' in 'SELECT' DROP DATABASE IF EXISTS mcs33_db1; diff --git a/mysql-test/columnstore/basic/r/mcs35_select_group_by.result b/mysql-test/columnstore/basic/r/mcs35_select_group_by.result index 7cf594ab5..7de2749c0 100644 --- a/mysql-test/columnstore/basic/r/mcs35_select_group_by.result +++ b/mysql-test/columnstore/basic/r/mcs35_select_group_by.result @@ -171,5 +171,5 @@ COUNT(*) SELECT COUNT(*) FROM t1 WHERE t1_blob IS NOT NULL OR t1_tinyint IS NULL GROUP BY t1_varchar; ERROR HY000: Internal error: VARBINARY/BLOB in filter or function is not supported. SELECT COUNT(*) FROM t1 GROUP BY t1_tinyint, t1_int, t1_bigint HAVING t1_char = 'b'; -ERROR 42S22: Unknown column 't1_char' in 'having clause' +ERROR 42S22: Unknown column 't1_char' in 'HAVING' DROP DATABASE IF EXISTS mcs35_db1; diff --git a/mysql-test/columnstore/basic/r/mcs36_select_order_by_group_by.result b/mysql-test/columnstore/basic/r/mcs36_select_order_by_group_by.result index a0f55dff5..62b159233 100644 --- a/mysql-test/columnstore/basic/r/mcs36_select_order_by_group_by.result +++ b/mysql-test/columnstore/basic/r/mcs36_select_order_by_group_by.result @@ -76,10 +76,10 @@ spID userid MIN(t1.score) 3 3 3 SELECT t2.userid, MIN(t1.score) FROM t1, t2 WHERE t1.userID=t2.userID GROUP BY unknown ORDER BY t2.userid; -ERROR 42S22: Unknown column 'unknown' in 'group statement' +ERROR 42S22: Unknown column 'unknown' in 'GROUP BY' SELECT t2.userid, MIN(t1.score) FROM t1, t2 WHERE t1.userID=t2.userID GROUP BY t2.userid ORDER BY unknown; -ERROR 42S22: Unknown column 'unknown' in 'order clause' +ERROR 42S22: Unknown column 'unknown' in 'ORDER BY' SELECT t2.userid, MIN(t1.score) FROM t1, t2 WHERE t1.userID=t2.userID GROUP BY t2.userid ORDER BY NULL; userid MIN(t1.score) diff --git a/mysql-test/columnstore/basic/r/mcs65_crossengine_order_by.result b/mysql-test/columnstore/basic/r/mcs65_crossengine_order_by.result index dca8a9ec3..2fdce2a2b 100644 --- a/mysql-test/columnstore/basic/r/mcs65_crossengine_order_by.result +++ b/mysql-test/columnstore/basic/r/mcs65_crossengine_order_by.result @@ -63,11 +63,11 @@ t1_int t1_char t2_int t2_char 3 fffff 3 iii 5 a 5 hhh SELECT * FROM t1, t2 WHERE t1.t1_int = t2.t2_int ORDER BY -1; -ERROR 42S22: Unknown column '???' in 'order clause' +ERROR 42S22: Unknown column '???' in 'ORDER BY' SELECT * FROM t1, t2 WHERE t1.t1_int = t2.t2_int ORDER BY 0; -ERROR 42S22: Unknown column '0' in 'order clause' +ERROR 42S22: Unknown column '0' in 'ORDER BY' SELECT * FROM t1, t2 WHERE t1.t1_int = t2.t2_int ORDER BY 5; -ERROR 42S22: Unknown column '5' in 'order clause' +ERROR 42S22: Unknown column '5' in 'ORDER BY' SELECT * FROM t1 LEFT JOIN t2 ON t1.t1_int = t2.t2_int ORDER BY t1.t1_int DESC; t1_int t1_char t2_int t2_char 7 ggg 7 nnnnn From 63abfd92fd47f2e49b087c433ca76b5aaca82a5c Mon Sep 17 00:00:00 2001 From: Alan Mologorsky <89034356+mariadb-AlanMologorsky@users.noreply.github.com> Date: Fri, 29 Nov 2024 20:19:46 +0300 Subject: [PATCH 20/65] fix(CI,mtr): temporary disable fdb_api and parquet-related tests for multinode mtr --- .drone.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 9c88e151c..f23b16f65 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -724,7 +724,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'docker exec -t mcs1 mariadb -e "create database if not exists test;"', // delay for manual debugging on live instance 'sleep $${COMPOSE_DELAY_SECONDS:-1s}', - 'docker exec -t mcs1 bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"', + 'docker exec -t mcs1 bash -c "cd ' + mtr_path + ' && ./mtr --skip-test=' + "'.*parquet.*|.*fdb_api.*'" + ' --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"', ], }, From 0bc384d5f0d8c1583be9cc066a8689f1281a3ba2 Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Thu, 21 Nov 2024 14:52:01 +0300 Subject: [PATCH 21/65] fix(ubsan): MCOL-5844 - iron out UBSAN reports The most important fix here is the fix of possible buffer overrun in DATEFORMAT() function. A "%W" format, repeated enough times, would overflow the 256-bytes buffer for result. Now we use ostringstream to construct result and we are safe. Changes in date/time projection functions made me fix difference between us and server behavior. The new, better behavior is reflected in changes in tests' results. Also, there was incorrect logic in TRUNCATE() and ROUND() functions in computing the decimal "shift." --- dbcon/execplan/arithmeticoperator.h | 13 ++ dbcon/joblist/jlf_common.h | 1 + dbcon/mysql/ha_mcs_execplan.cpp | 2 +- dmlproc/dmlprocessor.cpp | 5 +- .../basic/r/mcs119_date_formats.result | 4 +- .../basic/r/mcs185_dayname_function.result | 4 +- .../basic/r/mcs186_dayofyear_function.result | 4 +- .../basic/r/mcs236_extract_function.result | 12 +- .../basic/r/mcs275_timediff_function.result | 6 +- .../basic/r/mcs92_date_functions.result | 20 +-- .../basic/t/mcs92_date_functions.test | 4 + utils/funcexp/func_date_format.cpp | 133 +++++++++--------- utils/funcexp/func_dayname.cpp | 2 +- utils/funcexp/func_dayofweek.cpp | 2 +- utils/funcexp/func_dayofyear.cpp | 5 + utils/funcexp/func_extract.cpp | 8 +- utils/funcexp/func_from_unixtime.cpp | 2 +- utils/funcexp/func_round.cpp | 25 +++- utils/funcexp/func_timediff.cpp | 6 +- utils/funcexp/func_truncate.cpp | 43 ++++-- utils/funcexp/func_weekday.cpp | 2 +- utils/funcexp/funchelpers.h | 16 ++- utils/funcexp/tdriver.cpp | 3 +- utils/funcexp/timeextract.h | 3 +- utils/loggingcpp/message.cpp | 24 ++++ utils/loggingcpp/messageobj.h | 8 ++ 26 files changed, 224 insertions(+), 133 deletions(-) diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 399c7950f..6c66e9253 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -313,6 +313,19 @@ inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, Parse template inline T ArithmeticOperator::execute(T op1, T op2, bool& isNull) { + if (isNull) + { + // at least one operand is NULL. + // do nothing, return 0. + if constexpr (std::is_same::value) + { + return datatypes::TSInt128(); // returns 0 + } + else + { + return T{0}; + } + } switch (fOp) { case OP_ADD: return op1 + op2; diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index 427b5bfed..a0c03548a 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -225,6 +225,7 @@ struct JobInfo , stringScanThreshold(1) , wfqLimitStart(0) , wfqLimitCount(-1) + , djsForceRun(false) , timeZone(0) , maxPmJoinResultCount(1048576) { diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 6ac70dc77..e8b7b9a83 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -395,7 +395,7 @@ bool sortItemIsInGroupRec(Item* sort_item, Item* group_item) Item_ref* ifp_sort_ref = static_cast(sort_item); found = sortItemIsInGroupRec(*ifp_sort_ref->ref, group_item); } - else if (!found && sort_item->type() == Item::FIELD_ITEM) + else if (sort_item->type() == Item::FIELD_ITEM) { return found; } diff --git a/dmlproc/dmlprocessor.cpp b/dmlproc/dmlprocessor.cpp index 34529212f..833705b68 100644 --- a/dmlproc/dmlprocessor.cpp +++ b/dmlproc/dmlprocessor.cpp @@ -1316,7 +1316,10 @@ int DMLServer::start() } } -DMLProcessor::DMLProcessor(messageqcpp::IOSocket ios, BRM::DBRM* aDbrm) : fIos(ios), fDbrm(aDbrm) +DMLProcessor::DMLProcessor(messageqcpp::IOSocket ios, BRM::DBRM* aDbrm) + : fIos(ios) + , fDbrm(aDbrm) + , fConcurrentSupport(false) { csc = CalpontSystemCatalog::makeCalpontSystemCatalog(); csc->identity(CalpontSystemCatalog::EC); diff --git a/mysql-test/columnstore/basic/r/mcs119_date_formats.result b/mysql-test/columnstore/basic/r/mcs119_date_formats.result index 6a956fcae..b6ef46fd2 100644 --- a/mysql-test/columnstore/basic/r/mcs119_date_formats.result +++ b/mysql-test/columnstore/basic/r/mcs119_date_formats.result @@ -71,14 +71,14 @@ INSERT INTO t1 VALUES('2016-02-01', '11:11:11', '2017-02-01 12:12:12'); SELECT t1_dt, DATE_FORMAT(t1_dt, '%Y-%m-%d') a, DATE_FORMAT(t1_d, '%a %D %b %Y') b, DATE_FORMAT(t1_d, '%W %D %M %Y') c FROM t1; t1_dt a b c NULL NULL NULL NULL -0000-00-00 00:00:00 0000-00-00 Sun 0th NON_VALID 0000 Sunday 0th NON_VALID 0000 +0000-00-00 00:00:00 0000-00-00 NULL NULL 2020-11-11 12:12:12 2020-11-11 Mon 11th Nov 2222 Monday 11th November 2222 2020-12-31 12:34:56 2020-12-31 Wed 1st Jan 2020 Wednesday 1st January 2020 2017-02-01 12:12:12 2017-02-01 Mon 1st Feb 2016 Monday 1st February 2016 SELECT t1_dt, DATE_FORMAT(t1_dt, '%Y/%m/%d %T') a, DATE_FORMAT(t1_dt, '%a %D %b %Y %H:%i') b, DATE_FORMAT(t1_dt, '%W %D %M %Y %T') c FROM t1; t1_dt a b c NULL NULL NULL NULL -0000-00-00 00:00:00 0000/00/00 00:00:00 Sun 0th NON_VALID 0000 00:00 Sunday 0th NON_VALID 0000 00:00:00 +0000-00-00 00:00:00 0000/00/00 00:00:00 NULL NULL 2020-11-11 12:12:12 2020/11/11 12:12:12 Wed 11th Nov 2020 12:12 Wednesday 11th November 2020 12:12:12 2020-12-31 12:34:56 2020/12/31 12:34:56 Thu 31st Dec 2020 12:34 Thursday 31st December 2020 12:34:56 2017-02-01 12:12:12 2017/02/01 12:12:12 Wed 1st Feb 2017 12:12 Wednesday 1st February 2017 12:12:12 diff --git a/mysql-test/columnstore/basic/r/mcs185_dayname_function.result b/mysql-test/columnstore/basic/r/mcs185_dayname_function.result index f5534e802..99e98e92e 100644 --- a/mysql-test/columnstore/basic/r/mcs185_dayname_function.result +++ b/mysql-test/columnstore/basic/r/mcs185_dayname_function.result @@ -18,12 +18,12 @@ DAYNAME('2020-12-22') Tuesday SELECT a, DAYNAME(a) FROM t1 ORDER BY 1; a DAYNAME(a) -0000-00-00 Sunday +0000-00-00 NULL 1212-12-12 Wednesday 3333-03-03 Tuesday SELECT b, DAYNAME(b) FROM t1 ORDER BY 1; b DAYNAME(b) -0000-00-00 00:00:00 Sunday +0000-00-00 00:00:00 NULL 1212-12-11 11:11:11 Tuesday 3333-03-04 03:33:33 Wednesday DROP DATABASE mcs185_db; diff --git a/mysql-test/columnstore/basic/r/mcs186_dayofyear_function.result b/mysql-test/columnstore/basic/r/mcs186_dayofyear_function.result index 43b19bee8..468635df6 100644 --- a/mysql-test/columnstore/basic/r/mcs186_dayofyear_function.result +++ b/mysql-test/columnstore/basic/r/mcs186_dayofyear_function.result @@ -18,12 +18,12 @@ DAYOFYEAR('2020-12-22') 357 SELECT a, DAYOFYEAR(a) FROM t1 ORDER BY 1; a DAYOFYEAR(a) -0000-00-00 2147483647 +0000-00-00 NULL 1212-12-12 347 3333-03-03 62 SELECT b, DAYOFYEAR(b) FROM t1 ORDER BY 1; b DAYOFYEAR(b) -0000-00-00 00:00:00 2147483647 +0000-00-00 00:00:00 NULL 1212-12-11 11:11:11 346 3333-03-04 03:33:33 63 DROP DATABASE mcs186_db; diff --git a/mysql-test/columnstore/basic/r/mcs236_extract_function.result b/mysql-test/columnstore/basic/r/mcs236_extract_function.result index f8f8fe75b..1fa1097ee 100644 --- a/mysql-test/columnstore/basic/r/mcs236_extract_function.result +++ b/mysql-test/columnstore/basic/r/mcs236_extract_function.result @@ -321,9 +321,9 @@ NULL NULL SELECT t1_t, EXTRACT(HOUR_MICROSECOND FROM t1_t) FROM t1 ORDER BY 1; t1_t EXTRACT(HOUR_MICROSECOND FROM t1_t) NULL NULL -11:11:11 -558149696 -11:11:11 -558149696 -12:12:12 952915712 +11:11:11 111111000000 +11:11:11 111111000000 +12:12:12 121212000000 SELECT t1_t, EXTRACT(HOUR_SECOND FROM t1_t) FROM t1 ORDER BY 1; t1_t EXTRACT(HOUR_SECOND FROM t1_t) NULL NULL @@ -339,9 +339,9 @@ NULL NULL SELECT t1_t, EXTRACT(DAY_MICROSECOND FROM t1_t) FROM t1 ORDER BY 1; t1_t EXTRACT(DAY_MICROSECOND FROM t1_t) NULL NULL -11:11:11 -558149696 -11:11:11 -558149696 -12:12:12 952915712 +11:11:11 111111000000 +11:11:11 111111000000 +12:12:12 121212000000 SELECT t1_t, EXTRACT(DAY_SECOND FROM t1_t) FROM t1 ORDER BY 1; t1_t EXTRACT(DAY_SECOND FROM t1_t) NULL NULL diff --git a/mysql-test/columnstore/basic/r/mcs275_timediff_function.result b/mysql-test/columnstore/basic/r/mcs275_timediff_function.result index 36abb50fd..75f61850f 100644 --- a/mysql-test/columnstore/basic/r/mcs275_timediff_function.result +++ b/mysql-test/columnstore/basic/r/mcs275_timediff_function.result @@ -33,9 +33,9 @@ t1_TIME TIMEDIFF(t1_TIME, '2008-02-19 22:22:22') 23:59:59 NULL SELECT t1_TIME, TIMEDIFF(t1_TIME, '23:59:59') FROM t1 ORDER BY 1; t1_TIME TIMEDIFF(t1_TIME, '23:59:59') -01:37:50 -838:59:59 -22:12:02 -838:59:59 -23:59:59 -838:59:59 +01:37:50 -22:22:09 +22:12:02 -01:47:57 +23:59:59 00:00:00 SELECT t1_DATETIME, TIMEDIFF(t1_DATETIME, '2001-02-19 22:22:22') FROM t1 ORDER BY 1; t1_DATETIME TIMEDIFF(t1_DATETIME, '2001-02-19 22:22:22') 1997-12-12 22:12:02 -838:59:59 diff --git a/mysql-test/columnstore/basic/r/mcs92_date_functions.result b/mysql-test/columnstore/basic/r/mcs92_date_functions.result index e7936dcd3..bc563807a 100644 --- a/mysql-test/columnstore/basic/r/mcs92_date_functions.result +++ b/mysql-test/columnstore/basic/r/mcs92_date_functions.result @@ -295,14 +295,14 @@ NULL NULL SELECT t1_dt, DATE_FORMAT(t1_dt, '%Y-%m-%d') a, DATE_FORMAT(t1_d, '%a %D %b %Y') b, DATE_FORMAT(t1_d, '%W %D %M %Y') c FROM t1; t1_dt a b c NULL NULL NULL NULL -0000-00-00 00:00:00 0000-00-00 Sun 0th NON_VALID 0000 Sunday 0th NON_VALID 0000 +0000-00-00 00:00:00 0000-00-00 NULL NULL 2020-11-11 12:12:12 2020-11-11 Mon 11th Nov 2222 Monday 11th November 2222 2020-12-31 12:34:56 2020-12-31 Wed 1st Jan 2020 Wednesday 1st January 2020 2017-02-01 12:12:12 2017-02-01 Mon 1st Feb 2016 Monday 1st February 2016 SELECT t1_dt, DATE_FORMAT(t1_dt, '%Y/%m/%d %T') a, DATE_FORMAT(t1_dt, '%a %D %b %Y %H:%i') b, DATE_FORMAT(t1_dt, '%W %D %M %Y %T') c FROM t1; t1_dt a b c NULL NULL NULL NULL -0000-00-00 00:00:00 0000/00/00 00:00:00 Sun 0th NON_VALID 0000 00:00 Sunday 0th NON_VALID 0000 00:00:00 +0000-00-00 00:00:00 0000/00/00 00:00:00 NULL NULL 2020-11-11 12:12:12 2020/11/11 12:12:12 Wed 11th Nov 2020 12:12 Wednesday 11th November 2020 12:12:12 2020-12-31 12:34:56 2020/12/31 12:34:56 Thu 31st Dec 2020 12:34 Thursday 31st December 2020 12:34:56 2017-02-01 12:12:12 2017/02/01 12:12:12 Wed 1st Feb 2017 12:12 Wednesday 1st February 2017 12:12:12 @@ -312,14 +312,14 @@ en_US SELECT t1_dt, DAYNAME(t1_dt), DAYOFWEEK(t1_dt), WEEKDAY(t1_dt) FROM t1; t1_dt DAYNAME(t1_dt) DAYOFWEEK(t1_dt) WEEKDAY(t1_dt) NULL NULL NULL NULL -0000-00-00 00:00:00 Sunday 1 6 +0000-00-00 00:00:00 NULL NULL NULL 2020-11-11 12:12:12 Wednesday 4 2 2020-12-31 12:34:56 Thursday 5 3 2017-02-01 12:12:12 Wednesday 4 2 SELECT t1_d, DAYNAME(t1_d), DAYOFWEEK(t1_d), WEEKDAY(t1_d) FROM t1; t1_d DAYNAME(t1_d) DAYOFWEEK(t1_d) WEEKDAY(t1_d) NULL NULL NULL NULL -0000-00-00 Sunday 1 6 +0000-00-00 NULL NULL NULL 2222-11-11 Monday 2 0 2020-01-01 Wednesday 4 2 2016-02-01 Monday 2 0 @@ -333,7 +333,7 @@ NULL NULL NULL NULL NULL SELECT t1_dt, EXTRACT(DAY FROM t1_dt) 'DAY', EXTRACT(YEAR FROM t1_dt) 'YEAR', EXTRACT(MONTH FROM t1_dt) 'MONTH', EXTRACT(WEEK FROM t1_dt) 'WEEK' FROM t1; t1_dt DAY YEAR MONTH WEEK NULL NULL NULL NULL NULL -0000-00-00 00:00:00 0 0 0 613566753 +0000-00-00 00:00:00 0 0 0 1 2020-11-11 12:12:12 11 2020 11 46 2020-12-31 12:34:56 31 2020 12 53 2017-02-01 12:12:12 1 2017 2 5 @@ -361,10 +361,10 @@ NULL NULL SELECT t1_t, TIMEDIFF('12:58:11', t1_t) FROM t1; t1_t TIMEDIFF('12:58:11', t1_t) NULL NULL -00:00:00 838:59:59 -12:12:12 838:59:59 -11:11:11 838:59:59 -11:11:11 838:59:59 +00:00:00 12:58:11 +12:12:12 00:45:59 +11:11:11 01:47:00 +11:11:11 01:47:00 SELECT t1_d, t1_dt, TIMESTAMPDIFF(MONTH, t1_d, t1_dt) FROM t1; t1_d t1_dt TIMESTAMPDIFF(MONTH, t1_d, t1_dt) NULL NULL NULL @@ -396,7 +396,7 @@ NULL NULL NULL SELECT t1_d, WEEK(t1_d), t1_dt, WEEK(t1_dt) FROM t1; t1_d WEEK(t1_d) t1_dt WEEK(t1_dt) NULL NULL NULL NULL -0000-00-00 613566753 0000-00-00 00:00:00 613566753 +0000-00-00 1 0000-00-00 00:00:00 1 2222-11-11 45 2020-11-11 12:12:12 45 2020-01-01 0 2020-12-31 12:34:56 52 2016-02-01 5 2017-02-01 12:12:12 5 diff --git a/mysql-test/columnstore/basic/t/mcs92_date_functions.test b/mysql-test/columnstore/basic/t/mcs92_date_functions.test index 8301c7f85..012dec94c 100644 --- a/mysql-test/columnstore/basic/t/mcs92_date_functions.test +++ b/mysql-test/columnstore/basic/t/mcs92_date_functions.test @@ -70,6 +70,8 @@ SELECT t1_dt, DAYNAME(t1_dt), DAYOFWEEK(t1_dt), WEEKDAY(t1_dt) FROM t1; SELECT t1_d, DAYNAME(t1_d), DAYOFWEEK(t1_d), WEEKDAY(t1_d) FROM t1; SELECT t1_dt, EXTRACT(SECOND FROM t1_dt) 'SECOND', EXTRACT(DAY_HOUR FROM t1_dt) 'DAY_HOUR', EXTRACT(HOUR FROM t1_dt) 'HOUR', EXTRACT(MINUTE FROM t1_dt) 'MINUTE' FROM t1; +# please note that server's 613566753 is much more nonsensical than our new 1. +# returning NULL would be even more sensible. SELECT t1_dt, EXTRACT(DAY FROM t1_dt) 'DAY', EXTRACT(YEAR FROM t1_dt) 'YEAR', EXTRACT(MONTH FROM t1_dt) 'MONTH', EXTRACT(WEEK FROM t1_dt) 'WEEK' FROM t1; SELECT t1_dt, MONTH(t1_dt) FROM t1; @@ -82,6 +84,8 @@ SELECT t1_d, t1_dt, TIMESTAMPDIFF(DAY, t1_d, t1_dt) FROM t1; SELECT t1_d, t1_dt, TIMESTAMPDIFF(MINUTE, t1_d, t1_dt) FROM t1; SELECT t1_d, t1_dt, TIMESTAMPDIFF(SECOND, t1_d, t1_dt) FROM t1; +# please note that server's 613566753 is much more nonsensical than our new 1. +# returning NULL would be even more sensible. SELECT t1_d, WEEK(t1_d), t1_dt, WEEK(t1_dt) FROM t1; SELECT t1_d, YEAR(t1_d), t1_dt, YEAR(t1_dt) FROM t1; diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index a2e996e18..bb595aa5d 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -38,20 +38,21 @@ namespace funcexp { namespace helpers { -const string IDB_date_format(const DateTime& dt, const string& format) +const string IDB_date_format(const DateTime& dt, const string& format, bool& isNull) { // assume 256 is enough. assume not allowing incomplete date + // XXX: imagine %W gets repeated 60 times and day of week is "wednesday".. + std::ostringstream oss; char buf[256]; - char* ptr = buf; uint32_t weekday = 0; uint32_t dayval = 0; uint32_t weekval = 0; uint32_t weekyear = 0; - for (uint32_t i = 0; i < format.length(); i++) + for (uint32_t i = 0; !isNull && i < format.length(); i++) { if (format[i] != '%') - *ptr++ = format[i]; + oss << format[i]; else { i++; @@ -59,176 +60,170 @@ const string IDB_date_format(const DateTime& dt, const string& format) switch (format[i]) { case 'M': - sprintf(ptr, "%s", helpers::monthFullNames[dt.month].c_str()); - ptr += helpers::monthFullNames[dt.month].length(); + oss << helpers::monthFullNames[dt.month]; break; case 'b': - sprintf(ptr, "%s", helpers::monthAbNames[dt.month].c_str()); - ptr += helpers::monthAbNames[dt.month].length(); + oss << helpers::monthAbNames[dt.month].c_str(); break; case 'W': - weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false); - sprintf(ptr, "%s", helpers::weekdayFullNames[weekday].c_str()); - ptr += helpers::weekdayFullNames[weekday].length(); + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false, isNull); + oss << helpers::weekdayFullNames[weekday]; break; case 'w': - weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, true); - sprintf(ptr, "%01d", weekday); - ptr += 1; + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, true, isNull); + sprintf(buf, "%01d", weekday); + oss << buf; break; case 'a': - weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false); - sprintf(ptr, "%s", helpers::weekdayAbNames[weekday].c_str()); - ptr += helpers::weekdayAbNames[weekday].length(); + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false, isNull); + oss << helpers::weekdayAbNames[weekday]; break; case 'D': - sprintf(ptr, "%s", helpers::dayOfMonth[dt.day].c_str()); - ptr += helpers::dayOfMonth[dt.day].length(); + oss << helpers::dayOfMonth[dt.day].c_str(); break; case 'Y': - sprintf(ptr, "%04d", dt.year); - ptr += 4; + sprintf(buf, "%04d", dt.year); + oss << buf; break; case 'y': - sprintf(ptr, "%02d", dt.year % 100); - ptr += 2; + sprintf(buf, "%02d", dt.year % 100); + oss << buf; break; case 'm': - sprintf(ptr, "%02d", dt.month); - ptr += 2; + sprintf(buf, "%02d", dt.month); + oss << buf; break; case 'c': - sprintf(ptr, "%d", dt.month); - ptr = ptr + (dt.month >= 10 ? 2 : 1); + sprintf(buf, "%d", dt.month); + oss << buf; break; case 'd': - sprintf(ptr, "%02d", dt.day); - ptr += 2; + sprintf(buf, "%02d", dt.day); + oss << buf; break; case 'e': - sprintf(ptr, "%d", dt.day); - ptr = ptr + (dt.day >= 10 ? 2 : 1); + sprintf(buf, "%d", dt.day); + oss << buf; break; case 'f': - sprintf(ptr, "%06d", dt.msecond); - ptr += 6; + sprintf(buf, "%06d", dt.msecond); + oss << buf; break; case 'H': - sprintf(ptr, "%02d", dt.hour); - ptr += 2; + sprintf(buf, "%02d", dt.hour); + oss << buf; break; case 'h': case 'I': - sprintf(ptr, "%02d", (dt.hour % 24 + 11) % 12 + 1); - ptr += 2; + sprintf(buf, "%02d", (dt.hour % 24 + 11) % 12 + 1); + oss << buf; break; case 'i': /* minutes */ - sprintf(ptr, "%02d", dt.minute); - ptr += 2; + sprintf(buf, "%02d", dt.minute); + oss << buf; break; case 'j': dayval = helpers::calc_mysql_daynr(dt.year, dt.month, dt.day) - helpers::calc_mysql_daynr(dt.year, 1, 1) + 1; - sprintf(ptr, "%03d", dayval); - ptr += 3; + sprintf(buf, "%03d", dayval); + oss << buf; break; case 'k': - sprintf(ptr, "%d", dt.hour); - ptr += (dt.hour >= 10 ? 2 : 1); + sprintf(buf, "%d", dt.hour); + oss << buf; break; case 'l': - sprintf(ptr, "%d", (dt.hour % 24 + 11) % 12 + 1); - ptr += ((dt.hour % 24 + 11) % 12 + 1 >= 10 ? 2 : 1); + sprintf(buf, "%d", (dt.hour % 24 + 11) % 12 + 1); + oss << buf; break; case 'p': - sprintf(ptr, "%s", (dt.hour % 24 < 12 ? "AM" : "PM")); - ptr += 2; + sprintf(buf, "%s", (dt.hour % 24 < 12 ? "AM" : "PM")); + oss << buf; break; case 'r': - sprintf(ptr, (dt.hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"), + sprintf(buf, (dt.hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"), (dt.hour + 11) % 12 + 1, dt.minute, dt.second); - ptr += 11; + oss << buf; break; case 'S': case 's': - sprintf(ptr, "%02d", dt.second); - ptr += 2; + sprintf(buf, "%02d", dt.second); + oss << buf; break; case 'T': - sprintf(ptr, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); - ptr += 8; + sprintf(buf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + oss << buf; break; case 'U': weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, 0); - sprintf(ptr, "%02d", weekval); - ptr += 2; + sprintf(buf, "%02d", weekval); + oss << buf; break; case 'V': weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO); - sprintf(ptr, "%02d", weekval); - ptr += 2; + sprintf(buf, "%02d", weekval); + oss << buf; break; case 'u': weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS); - sprintf(ptr, "%02d", weekval); - ptr += 2; + sprintf(buf, "%02d", weekval); + oss << buf; break; case 'v': weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS); - sprintf(ptr, "%02d", weekval); - ptr += 2; + sprintf(buf, "%02d", weekval); + oss << buf; break; case 'x': helpers::calc_mysql_week( dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS, &weekyear); - sprintf(ptr, "%04d", weekyear); - ptr += 4; + sprintf(buf, "%04d", weekyear); + oss << buf; break; case 'X': helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO, &weekyear); - sprintf(ptr, "%04d", weekyear); - ptr += 4; + sprintf(buf, "%04d", weekyear); + oss << buf; break; - default: *ptr++ = format[i]; + default: oss << format[i]; } } } - *ptr = 0; - return string(buf); + return oss.str(); } } // namespace helpers @@ -394,7 +389,7 @@ string Func_date_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& const string& format = parm[1]->data()->getStrVal(row, isNull).safeString(""); - return helpers::IDB_date_format(dt, format); + return helpers::IDB_date_format(dt, format, isNull); } int32_t Func_date_format::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index e418bc67d..a5c870207 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -163,7 +163,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is default: isNull = true; return -1; } - dayofweek = helpers::calc_mysql_weekday(year, month, day, false); + dayofweek = helpers::calc_mysql_weekday(year, month, day, false, isNull); return dayofweek; } diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index b9700bdab..0f8b07253 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -170,7 +170,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& default: isNull = true; return -1; } - return helpers::calc_mysql_weekday(year, month, day, true) + 1; + return helpers::calc_mysql_weekday(year, month, day, true, isNull) + 1; } } // namespace funcexp diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index 215ec2150..5cd0dd47a 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -156,6 +156,11 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& default: isNull = true; return -1; } + if (year == 0 && month == 0 && day == 0) + { + isNull = true; + return 0; + } return helpers::calc_mysql_daynr(year, month, day) - helpers::calc_mysql_daynr(year, 1, 1) + 1; } diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 14cb62051..4f04bb220 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -109,11 +109,11 @@ long long dateGet(uint64_t time, IntervalColumn::interval_type unit, bool dateTy long long timeGet(uint64_t time, IntervalColumn::interval_type unit) { - int32_t hour = 0, min = 0, sec = 0, msec = 0, day = 0; + int64_t hour = 0, min = 0, sec = 0, msec = 0, day = 0; - min = (int32_t)((time >> 32) & 0xff); - sec = (int32_t)((time >> 24) & 0xff); - msec = (int32_t)((time & 0xfffff)); + min = (int64_t)((time >> 32) & 0xff); + sec = (int64_t)((time >> 24) & 0xff); + msec = (int64_t)((time & 0xfffff)); // If negative, mask so it doesn't turn positive int64_t mask = 0; diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index 73103ad59..6f9279661 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -125,7 +125,7 @@ string Func_from_unixtime::getStrVal(rowgroup::Row& row, FunctionParm& parm, boo if (parm.size() == 2) { const auto& format = parm[1]->data()->getStrVal(row, isNull); - return helpers::IDB_date_format(dt, format.safeString("")); + return helpers::IDB_date_format(dt, format.safeString(""), isNull); } char buf[256] = {0}; diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 3b765a916..2b7d68b72 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -389,19 +389,36 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. volatile int128_t p = 1; + if (isNull) + break; + if (!isNull && parm.size() > 1) // round(X, D) { int128_t nvp = p; d = parm[1]->data()->getIntVal(row, isNull); - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); + if (isNull) + break; + + int64_t expectedScale = decimal.scale - d; + + // prevent overflow. + if (expectedScale > datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + // also do not allow for incorrect behavior due to underflow. + if (expectedScale < 0) + { + d += expectedScale; + } + helpers::decimalPlaceDec(d, nvp, decimal.scale); p = nvp; } - if (isNull) - break; if (d < -datatypes::INT128MAXPRECISION) { diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index bed01600a..68d4000bb 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -137,7 +137,8 @@ string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is isNull = true; break; } - val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + val1 = isTime1 ? parm[0]->data()->getTimeIntVal(row, isNull) + : parm[0]->data()->getDatetimeIntVal(row, isNull); break; case execplan::CalpontSystemCatalog::TIMESTAMP: @@ -225,7 +226,8 @@ string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is isTime2 = true; /* fall through */ case execplan::CalpontSystemCatalog::DATETIME: - val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + val2 = isTime2 ? parm[1]->data()->getTimeIntVal(row, isNull) + : parm[1]->data()->getDatetimeIntVal(row, isNull); break; case execplan::CalpontSystemCatalog::TIMESTAMP: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index c8bafca5c..cccaa3666 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -315,25 +315,25 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN int64_t d = 0; decimal = parm[0]->data()->getDecimalVal(row, isNull); + if (isNull) + { + break; + } if (!op_ct.isWideDecimalType()) { //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. volatile int64_t p = 1; - if (!isNull) - { - int64_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } + int64_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); if (isNull) break; + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + int64_t x = decimal.value; if (d > 0) @@ -371,20 +371,33 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. volatile int128_t p = 1; + if (isNull) + break; + if (!isNull) { int128_t nvp = p; d = parm[1]->data()->getIntVal(row, isNull); - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); + int64_t expectedScale = decimal.scale - d; + + // prevent overflow. + if (expectedScale > datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + // also do not allow for incorrect behavior due to underflow. + if (expectedScale < 0) + { + d += expectedScale; + } + helpers::decimalPlaceDec(d, nvp, decimal.scale); p = nvp; } - if (isNull) - break; - if (d < -datatypes::INT128MAXPRECISION) { decimal.s128Value = 0; diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index e78d12c66..7b7fb9377 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -160,7 +160,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is default: isNull = true; return -1; } - return helpers::calc_mysql_weekday(year, month, day, false); + return helpers::calc_mysql_weekday(year, month, day, false, isNull); } } // namespace funcexp diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index 2738b674d..be65235d0 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -125,7 +125,7 @@ inline uint32_t calc_mysql_daynr(uint32_t year, uint32_t month, uint32_t day) int y = year; long delsum; - if (!dataconvert::isDateValid(day, month, year)) + if (!dataconvert::isDateValid(day, month, year) || (day == 0 && month == 0 && year == 0)) return 0; delsum = (long)(365 * y + 31 * ((int)month - 1) + (int)day); @@ -204,10 +204,13 @@ inline void get_date_from_mysql_daynr(long daynr, dataconvert::DateTime& dateTim // else: // 0 = Monday, 1 = Tuesday, ..., 6 = Sunday // This is a mirror of calc_weekday, at a later date we should use sql_time.h -inline uint32_t calc_mysql_weekday(uint32_t year, uint32_t month, uint32_t day, bool sundayFirst) +inline uint32_t calc_mysql_weekday(uint32_t year, uint32_t month, uint32_t day, bool sundayFirst, bool& isNull) { - if (!dataconvert::isDateValid(day, month, year)) + if (!dataconvert::isDateValid(day, month, year) || (day == 0 && month == 0 && year == 0)) + { + isNull = true; return 0; + } uint32_t daynr = calc_mysql_daynr(year, month, day); return ((int)((daynr + 5L + (sundayFirst ? 1L : 0L)) % 7)); @@ -252,7 +255,8 @@ inline uint32_t calc_mysql_week(uint32_t year, uint32_t month, uint32_t day, int bool week_year = modeflags & WEEK_NO_ZERO; bool first_weekday = modeflags & WEEK_GT_THREE_DAYS; - uint32_t weekday = calc_mysql_weekday(year, 1, 1, !monday_first); + bool isNullDummy = false; + uint32_t weekday = calc_mysql_weekday(year, 1, 1, !monday_first, isNullDummy); if (weekyear) { @@ -351,7 +355,7 @@ inline bool calc_time_diff(int64_t time1, int64_t time2, int l_sign, long long* days -= l_sign * calc_mysql_daynr(year2, month2, day2); - microseconds = ((long long)days * (long)(86400) + (long long)(hour1 * 3600L + min1 * 60L + sec1) - + microseconds = (int128_t(days) * (86400) + (long long)(hour1 * 3600L + min1 * 60L + sec1) - l_sign * (long long)(hour2 * 3600L + min2 * 60L + sec2)) * (long long)(1000000) + (long long)msec1 - l_sign * (long long)msec2; @@ -683,7 +687,7 @@ inline string longDoubleToString(long double ld) uint64_t dateAdd(uint64_t time, const std::string& expr, execplan::IntervalColumn::interval_type unit, bool dateType, execplan::OpType funcType); -const std::string IDB_date_format(const dataconvert::DateTime&, const std::string&); +const std::string IDB_date_format(const dataconvert::DateTime&, const std::string&, bool& isNull); const std::string timediff(int64_t, int64_t, bool isDateTime = true); const char* convNumToStr(int64_t, char*, int); diff --git a/utils/funcexp/tdriver.cpp b/utils/funcexp/tdriver.cpp index 8ab2fe614..ebcbfc9ea 100644 --- a/utils/funcexp/tdriver.cpp +++ b/utils/funcexp/tdriver.cpp @@ -275,8 +275,9 @@ class FuncExpTest : public CppUnit::TestFixture for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) { boost::gregorian::date d(date_tests[i].date.year, date_tests[i].date.month, date_tests[i].date.day); + bool isNullDummy = false; uint32_t dayofweek = helpers::calc_mysql_weekday(date_tests[i].date.year, date_tests[i].date.month, - date_tests[i].date.day, false); + date_tests[i].date.day, false, isNullDummy); bool check = (strcmp(helpers::weekdayFullNames[dayofweek].c_str(), date_tests[i].dayname) == 0); diff --git a/utils/funcexp/timeextract.h b/utils/funcexp/timeextract.h index 5e0fbc792..181ae5021 100644 --- a/utils/funcexp/timeextract.h +++ b/utils/funcexp/timeextract.h @@ -69,7 +69,8 @@ class TimeExtractor uint32_t yearfirst = helpers::calc_mysql_daynr(dateTime.year, 1, 1); // figure out which day of week Jan-01 is - uint32_t firstweekday = helpers::calc_mysql_weekday(dateTime.year, 1, 1, sundayFirst); + bool isNullDummy = false; + uint32_t firstweekday = helpers::calc_mysql_weekday(dateTime.year, 1, 1, sundayFirst, isNullDummy); // calculate the offset to the first week starting day uint32_t firstoffset = firstweekday ? (7 - firstweekday) : 0; diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index f8edf9703..321f44bd4 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -127,6 +127,30 @@ void Message::Args::add(uint64_t u64) fArgs.push_back(u64); } +void Message::Args::add(int128_t i128) +{ + uint128_t x = i128 < 0 ? -i128 : i128; + std::vector digits; + std::ostringstream oss; + while (x > 0) { + char c = (x % 10) + '0'; + digits.push_back(c); + x /= 10; + } + if (digits.size() < 1) { + digits.push_back('0'); + } + if (i128 < 0) + { + oss << '-'; + } + for(int32_t i=int(digits.size() - 1); i >= 0; i--) + { + oss << digits[i]; + } + fArgs.push_back(oss.str()); +} + void Message::Args::add(const string& s) { fArgs.push_back(s); diff --git a/utils/loggingcpp/messageobj.h b/utils/loggingcpp/messageobj.h index 2d67d0a2d..3d73ea1ab 100644 --- a/utils/loggingcpp/messageobj.h +++ b/utils/loggingcpp/messageobj.h @@ -65,6 +65,14 @@ class Message */ void add(uint64_t i); + /* define types to not to include mcs_numeric_limits.h */ + using int128_t = __int128; + using uint128_t = unsigned __int128; + + /** @brief add an 128 bit int arg to the message + */ + void add(int128_t i128); + /** @brief add a float arg to the message */ void add(double d); From 4878caee4e5a0a8bb2b232ffab1b1090ebcef870 Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Mon, 2 Dec 2024 12:53:08 +0300 Subject: [PATCH 22/65] Shorter code --- utils/loggingcpp/message.cpp | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index 321f44bd4..3debd218b 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -42,6 +42,8 @@ using namespace config; #include "installdir.h" #include "format.h" +#include "mcs_int128.h" + namespace { boost::mutex mx; @@ -129,26 +131,7 @@ void Message::Args::add(uint64_t u64) void Message::Args::add(int128_t i128) { - uint128_t x = i128 < 0 ? -i128 : i128; - std::vector digits; - std::ostringstream oss; - while (x > 0) { - char c = (x % 10) + '0'; - digits.push_back(c); - x /= 10; - } - if (digits.size() < 1) { - digits.push_back('0'); - } - if (i128 < 0) - { - oss << '-'; - } - for(int32_t i=int(digits.size() - 1); i >= 0; i--) - { - oss << digits[i]; - } - fArgs.push_back(oss.str()); + fArgs.push_back(TSInt128(i128).toString()); } void Message::Args::add(const string& s) From a2eafa492a20a18cc297ed7c74132c11cafd7319 Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Mon, 2 Dec 2024 17:51:56 +0300 Subject: [PATCH 23/65] Shorter code --- utils/loggingcpp/message.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index 3debd218b..765c1bd47 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -131,7 +131,7 @@ void Message::Args::add(uint64_t u64) void Message::Args::add(int128_t i128) { - fArgs.push_back(TSInt128(i128).toString()); + fArgs.push_back(datatypes::TSInt128(i128).toString()); } void Message::Args::add(const string& s) From bba2133cd00ce42855ecb3426ed1ff3e3aa8c8e5 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Tue, 3 Dec 2024 12:17:34 +0300 Subject: [PATCH 24/65] fix(fdb,regression): Add cluster path for test config (#3359) --- storage-manager/test_data/storagemanager.cnf | 1 + 1 file changed, 1 insertion(+) diff --git a/storage-manager/test_data/storagemanager.cnf b/storage-manager/test_data/storagemanager.cnf index 8dc742b06..eb490763c 100644 --- a/storage-manager/test_data/storagemanager.cnf +++ b/storage-manager/test_data/storagemanager.cnf @@ -25,6 +25,7 @@ metadata_path = ${HOME}/sm-unittest/metadata journal_path = ${HOME}/sm-unittest/journal max_concurrent_downloads = 20 max_concurrent_uploads = 20 +fdb_cluster_file_path = /etc/foundationdb/fdb.cluster # This is the depth of the common prefix that all files managed by SM have # Ex: /usr/local/mariadb/columnstore/data1, and From 6445f4dff3cff1327eef5030d060baf48e715433 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Tue, 3 Dec 2024 22:17:49 +0000 Subject: [PATCH 25/65] feat(joblist,runtime): this is the first part of the execution model that produces a workload that can be predicted for a given query. * feat(joblist,runtime): this is the first part of the execution model that produces a workload that can be predicted for a given query. - forces to UM join converter to use a value from a configuration - replaces a constant used to control a number of outstanding requests with a value depends on column width - modifies related Columnstore.xml values --- dbcon/joblist/distributedenginecomm.cpp | 2 + dbcon/joblist/primitivestep.h | 6 ++- dbcon/joblist/tuple-bps.cpp | 38 +++++++++---------- dbcon/joblist/tupleannexstep.cpp | 31 +++++++-------- dbcon/joblist/tuplehashjoin.cpp | 12 ++---- oam/etc/Columnstore.xml | 11 +++--- .../primproc/batchprimitiveprocessor.cpp | 35 +++++++++++------ utils/joiner/tuplejoiner.cpp | 11 ++---- utils/joiner/tuplejoiner.h | 4 +- 9 files changed, 76 insertions(+), 74 deletions(-) diff --git a/dbcon/joblist/distributedenginecomm.cpp b/dbcon/joblist/distributedenginecomm.cpp index c718deaa0..1bb79bfc0 100644 --- a/dbcon/joblist/distributedenginecomm.cpp +++ b/dbcon/joblist/distributedenginecomm.cpp @@ -68,6 +68,7 @@ using namespace oam; using namespace joblist; #include "atomicops.h" +#include "threadnaming.h" namespace { @@ -131,6 +132,7 @@ struct EngineCommRunner uint32_t connIndex; void operator()() { + utils::setThreadName("DECRunner"); // cout << "Listening on client at 0x" << hex << (ptrdiff_t)client << dec << endl; try { diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index 13a44516e..4b94dd453 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -112,12 +112,12 @@ class pColStep : public JobStep * * Starts processing. Set at least the RID list before calling this. */ - virtual void run(){}; + virtual void run() {}; /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(){}; + virtual void join() {}; virtual const std::string toString() const; @@ -1459,6 +1459,8 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep void interleaveJobs(std::vector* jobs) const; void sendJobs(const std::vector& jobs); uint32_t numDBRoots; + // presumably there must be not more than 2^32 blocks per job as of 23.02. + uint32_t blocksPerJob; /* Pseudo column filter processing. Think about refactoring into a separate class. */ bool processPseudoColFilters(uint32_t extentIndex, boost::shared_ptr> dbRootPMMap) const; diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index 6b1efc6c0..5696a1376 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -19,7 +19,7 @@ // $Id: tuple-bps.cpp 9705 2013-07-17 20:06:07Z pleblanc $ #include -//#define NDEBUG +// #define NDEBUG #include #include #include @@ -77,7 +77,9 @@ using namespace querytele; #include "columnwidth.h" #include "pseudocolumn.h" -//#define DEBUG 1 +// #define DEBUG 1 + +// #include "poormanprofiler.inc" extern boost::mutex fileLock_g; @@ -396,15 +398,6 @@ void TupleBPS::initializeConfigParms() { string strVal; - //...Get the tuning parameters that throttle msgs sent to primproc - //...fFilterRowReqLimit puts a cap on how many rids we will request from - //... primproc, before pausing to let the consumer thread catch up. - //... Without this limit, there is a chance that PrimProc could flood - //... ExeMgr with thousands of messages that will consume massive - //... amounts of memory for a 100 gigabyte database. - //...fFilterRowReqThreshold is the level at which the number of outstanding - //... rids must fall below, before the producer can send more rids. - // These could go in constructor fRequestSize = fRm->getJlRequestSize(); fMaxOutstandingRequests = fRm->getJlMaxOutstandingRequests(); @@ -556,14 +549,14 @@ TupleBPS::TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo) : BatchPrimi throw runtime_error(oss.str()); } - catch(std::exception& ex) + catch (std::exception& ex) { std::ostringstream oss; oss << "Error getting AUX column OID for table " << tableName.toString(); oss << " due to: " << ex.what(); throw runtime_error(oss.str()); } - catch(...) + catch (...) { std::ostringstream oss; oss << "Error getting AUX column OID for table " << tableName.toString(); @@ -1684,7 +1677,8 @@ void TupleBPS::sendJobs(const vector& jobs) if (recvWaiting) condvar.notify_all(); - while ((msgsSent - msgsRecvd > fMaxOutstandingRequests << LOGICAL_EXTENT_CONVERTER) && !fDie) + // Send not more than fMaxOutstandingRequests jobs out. min(blocksPerJob) = 16 + while ((msgsSent - msgsRecvd > fMaxOutstandingRequests * (blocksPerJob >> 1)) && !fDie) { sendWaiting = true; condvarWakeupProducer.wait(tplLock); @@ -2007,7 +2001,6 @@ void TupleBPS::makeJobs(vector* jobs) uint32_t i; uint32_t lbidsToScan; uint32_t blocksToScan; - uint32_t blocksPerJob; LBID_t startingLBID; oam::OamCache* oamCache = oam::OamCache::makeOamCache(); boost::shared_ptr> dbRootConnectionMap = oamCache->getDBRootToConnectionMap(); @@ -2227,6 +2220,8 @@ void TupleBPS::processByteStreamVector(vectorpmSendsFinalResult()) { + utils::setThreadName("BSPJoin"); + data->joinedData = RGData(data->local_outputRG); data->local_outputRG.setData(&data->joinedData); data->local_outputRG.resetRowGroup(data->local_primRG.getBaseRid()); @@ -2340,6 +2335,8 @@ void TupleBPS::processByteStreamVector(vectorjoinedData); } + + utils::setThreadName("ByteStreamProcessor"); } else { @@ -2351,6 +2348,7 @@ void TupleBPS::processByteStreamVector(vector 0 && !cancelled()) { @@ -2358,6 +2356,8 @@ void TupleBPS::processByteStreamVector(vectorlocal_fe2Output, dlp); } + utils::setThreadName("ByteStreamProcessor"); + data->cachedIO_Thread += cachedIO; data->physIO_Thread += physIO; data->touchedBlocks_Thread += touchedBlocks; @@ -2777,8 +2777,7 @@ void TupleBPS::receiveMultiPrimitiveMessages() << totalBlockedReadCount << "/" << totalBlockedWriteCount << "; output size-" << ridsReturned << endl << "\tPartitionBlocksEliminated-" << fNumBlksSkipped << "; MsgBytesIn-" << msgBytesInKB << "KB" - << "; MsgBytesOut-" << msgBytesOutKB << "KB" - << "; TotalMsgs-" << totalMsgs << endl + << "; MsgBytesOut-" << msgBytesOutKB << "KB" << "; TotalMsgs-" << totalMsgs << endl << "\t1st read " << dlTimes.FirstReadTimeString() << "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << "s\n\tUUID " << uuids::to_string(fStepUuid) << "\n\tQuery UUID " @@ -3179,9 +3178,8 @@ bool TupleBPS::deliverStringTableRowGroup() const void TupleBPS::formatMiniStats() { ostringstream oss; - oss << "BPS " - << "PM " << alias() << " " << fTableOid << " " << fBPP->toMiniString() << " " << fPhysicalIO << " " - << fCacheIO << " " << fNumBlksSkipped << " " + oss << "BPS " << "PM " << alias() << " " << fTableOid << " " << fBPP->toMiniString() << " " << fPhysicalIO + << " " << fCacheIO << " " << fNumBlksSkipped << " " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " << ridsReturned << " "; diff --git a/dbcon/joblist/tupleannexstep.cpp b/dbcon/joblist/tupleannexstep.cpp index a62bac653..f0c9f3225 100644 --- a/dbcon/joblist/tupleannexstep.cpp +++ b/dbcon/joblist/tupleannexstep.cpp @@ -18,7 +18,7 @@ // $Id: tupleannexstep.cpp 9661 2013-07-01 20:33:05Z pleblanc $ -//#define NDEBUG +// #define NDEBUG #include #include #include @@ -251,10 +251,6 @@ void TupleAnnexStep::run() fRunnersList.resize(fMaxThreads); fInputIteratorsList.resize(fMaxThreads + 1); - // Activate stats collecting before CS spawns threads. - if (traceOn()) - dlTimes.setFirstReadTime(); - // *DRRTUY Make this block conditional StepTeleStats sts; sts.query_uuid = fQueryUuid; @@ -858,7 +854,7 @@ void TupleAnnexStep::finalizeParallelOrderByDistinct() break; } } // end of limit bound while loop - } // end of if-else + } // end of if-else if (fRowGroupOut.getRowCount() > 0) { @@ -1045,7 +1041,7 @@ void TupleAnnexStep::finalizeParallelOrderBy() break; } } // end of limit bound while loop - } // end of if-else + } // end of if-else if (fRowGroupOut.getRowCount() > 0) { @@ -1065,9 +1061,6 @@ void TupleAnnexStep::finalizeParallelOrderBy() if (traceOn()) { - if (dlTimes.FirstReadTime().tv_sec == 0) - dlTimes.setFirstReadTime(); - dlTimes.setLastReadTime(); dlTimes.setEndOfInputTime(); printCalTrace(); @@ -1102,6 +1095,13 @@ void TupleAnnexStep::executeParallelOrderBy(uint64_t id) try { more = fInputDL->next(fInputIteratorsList[id], &rgDataIn); + + // Stats collecting. + if (more && (id == 1) && traceOn()) + { + dlTimes.setFirstReadTime(); + } + if (more) dlOffset++; @@ -1241,14 +1241,9 @@ void TupleAnnexStep::formatMiniStats() { ostringstream oss; oss << "TNS "; - oss << "UM " - << "- " - << "- " - << "- " - << "- " - << "- " - << "- " << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " - << fRowsReturned << " "; + oss << "UM " << "- " << "- " << "- " << "- " << "- " << "- " + << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " << fRowsReturned + << " "; fMiniInfo += oss.str(); } diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 6e6a60c3d..d935a67dc 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -278,12 +278,12 @@ void TupleHashJoinStep::startSmallRunners(uint index) if (typelessJoin[index]) { joiner.reset(new TupleJoiner(smallRGs[index], largeRG, smallSideKeys[index], largeSideKeys[index], jt, - &jobstepThreadPool)); + &jobstepThreadPool, numCores)); } else { joiner.reset(new TupleJoiner(smallRGs[index], largeRG, smallSideKeys[index][0], largeSideKeys[index][0], - jt, &jobstepThreadPool)); + jt, &jobstepThreadPool, numCores)); } joiner->setUniqueLimit(uniqueLimit); @@ -1297,15 +1297,11 @@ void TupleHashJoinStep::formatMiniStats(uint32_t index) else oss << "- "; - oss << " " - << "- " - << "- " - << "- " + oss << " " << "- " << "- " << "- " << "- " // << JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " " // dlTimes are not timed in this step, using '--------' instead. - << "-------- " - << "-\n"; + << "-------- " << "-\n"; fMiniInfo += oss.str(); } diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 2901a87a6..0de06e10d 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -203,12 +203,11 @@ is 20 extents worth of work for the PMs to process at any given time. ProcessorThreadsPerScan * MaxOutstandingRequests should be at least as many threads are available across all PMs. --> - - - - 100 + + + 20000 + 1000 diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 57b8111c9..dfe1527a7 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -1387,6 +1387,7 @@ void BatchPrimitiveProcessor::execute() #ifdef PRIMPROC_STOPWATCH stopwatch->start("BatchPrimitiveProcessor::execute first part"); #endif + utils::setThreadName("BPPFilt&Pr"); // if only one scan step which has no predicate, async load all columns if (filterCount == 1 && hasScan) @@ -1550,6 +1551,8 @@ void BatchPrimitiveProcessor::execute() #endif outputRG.resetRowGroup(baseRid); + utils::setThreadName("BPPFE1_1"); + if (fe1) { uint32_t newRidCount = 0; @@ -1616,6 +1619,8 @@ void BatchPrimitiveProcessor::execute() } if (fe2) { + utils::setThreadName("BPPFE2_1"); + /* functionize this -> processFE2() */ fe2Output.resetRowGroup(baseRid); fe2Output.getRow(0, &fe2Out); @@ -1646,6 +1651,8 @@ void BatchPrimitiveProcessor::execute() if (fAggregator) { + utils::setThreadName("BPPAgg_1"); + *serialized << (uint8_t)1; // the "count this msg" var // see TupleBPS::setFcnExpGroup2() and where it gets called. @@ -1662,17 +1669,17 @@ void BatchPrimitiveProcessor::execute() if ((currentBlockOffset + 1) == count) // @bug4507, 8k { - fAggregator->loadResult(*serialized); // @bug4507, 8k - } // @bug4507, 8k + fAggregator->loadResult(*serialized); // @bug4507, 8k + } // @bug4507, 8k else if (utils::MonitorProcMem::isMemAvailable()) // @bug4507, 8k { fAggregator->loadEmptySet(*serialized); // @bug4507, 8k - } // @bug4507, 8k - else // @bug4507, 8k + } // @bug4507, 8k + else // @bug4507, 8k { fAggregator->loadResult(*serialized); // @bug4507, 8k fAggregator->aggReset(); // @bug4507, 8k - } // @bug4507, 8k + } // @bug4507, 8k } if (!fAggregator && !fe2) @@ -1726,6 +1733,8 @@ void BatchPrimitiveProcessor::execute() do // while (startRid > 0) { + utils::setThreadName("BPPJoin_1"); + #ifdef PRIMPROC_STOPWATCH stopwatch->start("-- executeTupleJoin()"); startRid = executeTupleJoin(startRid, largeSideRowGroup); @@ -1777,6 +1786,8 @@ void BatchPrimitiveProcessor::execute() *serialized << sendCount; if (fe2) { + utils::setThreadName("BPPFE2_2"); + /* functionize this -> processFE2()*/ fe2Output.resetRowGroup(baseRid); fe2Output.setDBRoot(dbRoot); @@ -1800,21 +1811,23 @@ void BatchPrimitiveProcessor::execute() if (fAggregator) { + utils::setThreadName("BPPAgg_2"); + fAggregator->addRowGroup(&nextRG); if ((currentBlockOffset + 1) == count && moreRGs == false && startRid == 0) // @bug4507, 8k { - fAggregator->loadResult(*serialized); // @bug4507, 8k - } // @bug4507, 8k + fAggregator->loadResult(*serialized); // @bug4507, 8k + } // @bug4507, 8k else if (utils::MonitorProcMem::isMemAvailable()) // @bug4507, 8k { fAggregator->loadEmptySet(*serialized); // @bug4507, 8k - } // @bug4507, 8k - else // @bug4507, 8k + } // @bug4507, 8k + else // @bug4507, 8k { fAggregator->loadResult(*serialized); // @bug4507, 8k fAggregator->aggReset(); // @bug4507, 8k - } // @bug4507, 8k + } // @bug4507, 8k } else { @@ -1901,6 +1914,7 @@ void BatchPrimitiveProcessor::execute() // cout << "sent physIO=" << physIO << " cachedIO=" << cachedIO << // " touchedBlocks=" << touchedBlocks << endl; } + utils::setThreadName("BPPExecuteEnd"); #ifdef PRIMPROC_STOPWATCH stopwatch->stop("BatchPrimitiveProcessor::execute fourth part"); @@ -2751,7 +2765,6 @@ void BatchPrimitiveProcessor::buildVSSCache(uint32_t loopCount) if (rc == 0) for (i = 0; i < vssData.size(); i++) vssCache.insert(make_pair(lbidList[i], vssData[i])); - } } // namespace primitiveprocessor diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index bd7046efa..d7cea726f 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -39,7 +39,7 @@ namespace joiner // Typed joiner ctor TupleJoiner::TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::RowGroup& largeInput, uint32_t smallJoinColumn, uint32_t largeJoinColumn, JoinType jt, - threadpool::ThreadPool* jsThreadPool) + threadpool::ThreadPool* jsThreadPool, const uint64_t numCores) : smallRG(smallInput) , largeRG(largeInput) , joinAlg(INSERTING) @@ -49,6 +49,7 @@ TupleJoiner::TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::R , bSignedUnsignedJoin(false) , uniqueLimit(100) , finished(false) + , numCores(numCores) , jobstepThreadPool(jsThreadPool) , _convertToDiskJoin(false) { @@ -145,7 +146,7 @@ TupleJoiner::TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::R // Typeless joiner ctor TupleJoiner::TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::RowGroup& largeInput, const vector& smallJoinColumns, const vector& largeJoinColumns, - JoinType jt, threadpool::ThreadPool* jsThreadPool) + JoinType jt, threadpool::ThreadPool* jsThreadPool, const uint64_t numCores) : smallRG(smallInput) , largeRG(largeInput) , joinAlg(INSERTING) @@ -157,6 +158,7 @@ TupleJoiner::TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::R , bSignedUnsignedJoin(false) , uniqueLimit(100) , finished(false) + , numCores(numCores) , jobstepThreadPool(jsThreadPool) , _convertToDiskJoin(false) { @@ -254,11 +256,6 @@ bool TupleJoiner::operator<(const TupleJoiner& tj) const void TupleJoiner::getBucketCount() { - // get the # of cores, round up to nearest power of 2 - // make the bucket mask - numCores = sysconf(_SC_NPROCESSORS_ONLN); - if (numCores <= 0) - numCores = 8; bucketCount = (numCores == 1 ? 1 : (1 << (32 - __builtin_clz(numCores - 1)))); bucketMask = bucketCount - 1; } diff --git a/utils/joiner/tuplejoiner.h b/utils/joiner/tuplejoiner.h index aa4aaf647..98af11b31 100644 --- a/utils/joiner/tuplejoiner.h +++ b/utils/joiner/tuplejoiner.h @@ -268,12 +268,12 @@ class TupleJoiner /* ctor to use for numeric join */ TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::RowGroup& largeInput, uint32_t smallJoinColumn, uint32_t largeJoinColumn, joblist::JoinType jt, - threadpool::ThreadPool* jsThreadPool); + threadpool::ThreadPool* jsThreadPool, const uint64_t numCores); /* ctor to use for string & compound join */ TupleJoiner(const rowgroup::RowGroup& smallInput, const rowgroup::RowGroup& largeInput, const std::vector& smallJoinColumns, const std::vector& largeJoinColumns, - joblist::JoinType jt, threadpool::ThreadPool* jsThreadPool); + joblist::JoinType jt, threadpool::ThreadPool* jsThreadPool, const uint64_t numCores); ~TupleJoiner(); From 38fd96a663517350e7cf4e229e9d147a99a946ff Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Mon, 30 Sep 2024 14:50:35 +0300 Subject: [PATCH 26/65] fix(memory leaks): MCOL-5791 - get rid of memory leaks in plugin code There were numerous memory leaks in plugin's code and associated code. During typical run of MTR tests it leaked around 65 megabytes of objects. As a result they may severely affect long-lived connections. This patch fixes (almost) all leaks found in the plugin. The exceptions are two leaks associated with SHOW CREATE TABLE columnstore_table and getting information of columns of columnstore-handled table. These should be fixed on the server side and work is on the way. --- dbcon/ddlpackage/ddl.l | 8 +- dbcon/ddlpackage/ddl.y | 21 +- dbcon/dmlpackage/dml.l | 4 +- dbcon/mysql/ha_exists_sub.cpp | 2 +- dbcon/mysql/ha_from_sub.cpp | 2 +- dbcon/mysql/ha_in_sub.cpp | 2 +- dbcon/mysql/ha_mcs_ddl.cpp | 8 +- dbcon/mysql/ha_mcs_execplan.cpp | 231 +++++++++++++++++----- dbcon/mysql/ha_mcs_impl.cpp | 97 +++++---- dbcon/mysql/ha_mcs_impl_if.h | 35 +++- dbcon/mysql/ha_pseudocolumn.cpp | 1 + dbcon/mysql/ha_scalar_sub.cpp | 4 +- dbcon/mysql/ha_select_sub.cpp | 2 +- dbcon/mysql/ha_subquery.h | 19 ++ dbcon/mysql/ha_view.cpp | 12 +- dbcon/mysql/ha_window_function.cpp | 8 +- dbcon/mysql/is_columnstore_extents.cpp | 20 +- dbcon/mysql/sm.cpp | 6 +- dbcon/mysql/sm.h | 10 +- tests/CMakeLists.txt | 3 +- tests/scripts/fullmtr.sh | 16 +- versioning/BRM/brmshmimpl.cpp | 1 + writeengine/bulk/we_bulkload.cpp | 40 ++-- writeengine/bulk/we_bulkload.h | 4 +- writeengine/bulk/we_bulkloadbuffer.cpp | 32 +-- writeengine/bulk/we_extentstripealloc.cpp | 4 +- writeengine/bulk/we_extentstripealloc.h | 39 ++-- writeengine/bulk/we_tableinfo.cpp | 2 +- writeengine/bulk/we_workers.cpp | 60 +++--- writeengine/shared/we_brm.cpp | 5 + writeengine/shared/we_brm.h | 3 + 31 files changed, 472 insertions(+), 229 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index c0aec2a94..5404542f0 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -34,7 +34,7 @@ int ddldebug = 0; int lineno = 1; void ddlerror(struct pass_to_bison* x, char const *s); -static char* scanner_copy(char *str, yyscan_t yyscanner, copy_action_t action = NOOP ); +static char* scanner_copy(const char *str, yyscan_t yyscanner, copy_action_t action = NOOP ); %} @@ -115,9 +115,9 @@ CONSTRAINT {return CONSTRAINT;} CONSTRAINTS {return CONSTRAINTS;} CREATE {return CREATE;} CURRENT_USER {return CURRENT_USER;} -DATE {ddlget_lval(yyscanner)->str=strdup("date"); return DATE;} +DATE {ddlget_lval(yyscanner)->str = scanner_copy("date", yyscanner); return DATE;} DATETIME {return DATETIME;} -TIME {ddlget_lval(yyscanner)->str=strdup("time"); return TIME;} +TIME {ddlget_lval(yyscanner)->str = scanner_copy("time", yyscanner); return TIME;} TIMESTAMP {return TIMESTAMP;} DECIMAL {return DECIMAL;} DEC {return DECIMAL;} @@ -276,7 +276,7 @@ void scanner_finish(yyscan_t yyscanner) pScanData->valbuf.clear(); } -char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action) +char* scanner_copy (const char *str, yyscan_t yyscanner, copy_action_t action) { char* result; char* nv = strdup(str); diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 83f4913f9..1e42314b5 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -105,7 +105,6 @@ void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_c column->fType->fLength = 16777215; } } - %} %expect 17 @@ -255,6 +254,24 @@ ZEROFILL %type opt_quoted_literal %type opt_column_charset %type opt_column_collate + +// for pointers to vectors of pointers: +%destructor { if ($$) { for (auto p : *($$)) { delete p; } }; delete $$; } table_element_list + +// for objects allocated during parse:. +%destructor { delete $$; } qualified_name ident table_element column_def exact_numeric_type +%destructor { delete $$; } table_options opt_table_options table_name +%destructor { delete $$; } column_name_list data_type column_constraint +%destructor { delete $$; } column_constraint_def column_qualifier_list +%destructor { delete $$; } opt_referential_triggered_action referential_triggered_action +%destructor { delete $$; } table_option character_string_type binary_string_type blob_type +%destructor { delete $$; } text_type numeric_type table_constraint table_constraint_def + +// NOTE: if you have a leak in this code and do not know which one leaks +// add %destructor { printf("pop yykind %d\n"; fflush(stdout); } <*> +// this will print tags in residual stack after syntax error and you'll see +// what is not delete'd. + %% stmtblock: stmtmulti { x->fParseTree = $1; } ; @@ -685,7 +702,7 @@ qualified_name: if (x->fDBSchema.size()) $$ = new QualifiedName((char*)x->fDBSchema.c_str(), $1); else - $$ = new QualifiedName($1); + $$ = new QualifiedName($1); } | ident '.' ident { diff --git a/dbcon/dmlpackage/dml.l b/dbcon/dmlpackage/dml.l index fca2d3f25..babff264e 100644 --- a/dbcon/dmlpackage/dml.l +++ b/dbcon/dmlpackage/dml.l @@ -42,7 +42,7 @@ namespace dmlpackage { int lineno = 1; -static char* scanner_copy (char *str, yyscan_t yyscanner); +static char* scanner_copy (const char *str, yyscan_t yyscanner); /* macro to save the text and return a token */ @@ -256,7 +256,7 @@ void scanner_finish(yyscan_t yyscanner) pScanData->valbuf.clear(); } -char* scanner_copy (char *str, yyscan_t yyscanner) +char* scanner_copy (const char *str, yyscan_t yyscanner) { char* nv = strdup(str); if(nv) diff --git a/dbcon/mysql/ha_exists_sub.cpp b/dbcon/mysql/ha_exists_sub.cpp index a97abfa44..181d1c83c 100644 --- a/dbcon/mysql/ha_exists_sub.cpp +++ b/dbcon/mysql/ha_exists_sub.cpp @@ -96,7 +96,7 @@ execplan::ParseTree* ExistsSub::transform() csep->subType(CalpontSelectExecutionPlan::EXISTS_SUBS); // gwi for the sub query - gp_walk_info gwi(fGwip.timeZone); + gp_walk_info gwi(fGwip.timeZone, fGwip.subQueriesChain); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 9013c8fe6..177887323 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -424,7 +424,7 @@ SCSEP FromSubQuery::transform() csep->subType(CalpontSelectExecutionPlan::FROM_SUBS); // gwi for the sub query - gp_walk_info gwi(fGwip.timeZone); + gp_walk_info gwi(fGwip.timeZone, fGwip.subQueriesChain); gwi.thd = fGwip.thd; gwi.subQuery = this; gwi.viewName = fGwip.viewName; diff --git a/dbcon/mysql/ha_in_sub.cpp b/dbcon/mysql/ha_in_sub.cpp index 989e06ad5..8b2772efd 100644 --- a/dbcon/mysql/ha_in_sub.cpp +++ b/dbcon/mysql/ha_in_sub.cpp @@ -151,7 +151,7 @@ execplan::ParseTree* InSub::transform() csep->subType(CalpontSelectExecutionPlan::IN_SUBS); // gwi for the sub query - gp_walk_info gwi(fGwip.timeZone); + gp_walk_info gwi(fGwip.timeZone, fGwip.subQueriesChain); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 9695d21b2..fac2d176e 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -344,7 +344,7 @@ bool anyRowInTable(string& schema, string& tableName, int sessionID) rowgroup::RGData rgData; ByteStream::quadbyte qb = 4; msg << qb; - rowgroup::RowGroup* rowGroup = 0; + std::shared_ptr rowGroup = 0; bool anyRow = false; exemgrClient->write(msg); @@ -397,7 +397,7 @@ bool anyRowInTable(string& schema, string& tableName, int sessionID) if (!rowGroup) { // This is mete data - rowGroup = new rowgroup::RowGroup(); + rowGroup.reset(new rowgroup::RowGroup()); rowGroup->deserialize(msg); qb = 100; msg.restart(); @@ -515,7 +515,7 @@ bool anyTimestampColumn(string& schema, string& tableName, int sessionID) rowgroup::RGData rgData; ByteStream::quadbyte qb = 4; msg << qb; - rowgroup::RowGroup* rowGroup = 0; + std::shared_ptr rowGroup = 0; bool anyRow = false; exemgrClient->write(msg); @@ -568,7 +568,7 @@ bool anyTimestampColumn(string& schema, string& tableName, int sessionID) if (!rowGroup) { // This is mete data - rowGroup = new rowgroup::RowGroup(); + rowGroup.reset(new rowgroup::RowGroup()); rowGroup->deserialize(msg); qb = 100; msg.restart(); diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index e8b7b9a83..85a2b2968 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -293,13 +293,66 @@ string escapeBackTick(const char* str) return ret; } -void clearStacks(gp_walk_info& gwi) +cal_impl_if::gp_walk_info::~gp_walk_info() +{ + while (!rcWorkStack.empty()) + { + delete rcWorkStack.top(); + rcWorkStack.pop(); + } + + while (!ptWorkStack.empty()) + { + delete ptWorkStack.top(); + ptWorkStack.pop(); + } + for (uint32_t i=0;i& join_list, ParseTree* pt = new ParseTree(onFilter); outerJoinStack.push(pt); } + } else // inner join { @@ -1545,7 +1599,7 @@ ParseTree* buildRowPredicate(gp_walk_info* gwip, RowColumn* lhs, RowColumn* rhs, ParseTree* pt = new ParseTree(lo); sop->setOpType(lhs->columnVec()[0]->resultType(), rhs->columnVec()[0]->resultType()); - SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[0].get(), rhs->columnVec()[0].get()); + SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[0]->clone(), rhs->columnVec()[0]->clone()); sf->timeZone(gwip->timeZone); pt->left(new ParseTree(sf)); @@ -1553,7 +1607,7 @@ ParseTree* buildRowPredicate(gp_walk_info* gwip, RowColumn* lhs, RowColumn* rhs, { sop.reset(po->clone()); sop->setOpType(lhs->columnVec()[i]->resultType(), rhs->columnVec()[i]->resultType()); - SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[i].get(), rhs->columnVec()[i].get()); + SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[i]->clone(), rhs->columnVec()[i]->clone()); sf->timeZone(gwip->timeZone); pt->right(new ParseTree(sf)); @@ -1570,6 +1624,12 @@ ParseTree* buildRowPredicate(gp_walk_info* gwip, RowColumn* lhs, RowColumn* rhs, bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, Item_func* ifp) { + // rhs and lhs are being dismembered here and then get thrown away, leaking. + // So we create two scoped pointers to get them delete'd automatically at + // return. + // Also look below into heldoutVals vector - it contains values produced from + // ifp's arguments. + const std::unique_ptr rhsp(rhs), lhsp(lhs); if (ifp->functype() == Item_func::EQ_FUNC || ifp->functype() == Item_func::NE_FUNC) { // (c1,c2,..) = (v1,v2,...) transform to: c1=v1 and c2=v2 and ... @@ -1601,6 +1661,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It // two entries have been popped from the stack already: lhs and rhs stack tmpStack; vector valVec; + vector heldOutVals; // these vals are not rhs/lhs and need to be freed tmpStack.push(rhs); tmpStack.push(lhs); assert(gwip->rcWorkStack.size() >= ifp->argument_count() - 2); @@ -1608,6 +1669,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It for (uint32_t i = 2; i < ifp->argument_count(); i++) { tmpStack.push(gwip->rcWorkStack.top()); + heldOutVals.push_back(SRCP(gwip->rcWorkStack.top())); if (!gwip->rcWorkStack.empty()) gwip->rcWorkStack.pop(); @@ -1627,7 +1689,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It vals = dynamic_cast(tmpStack.top()); valVec.push_back(vals); tmpStack.pop(); - pt1->right(buildRowPredicate(gwip, columns->clone(), vals, predicateOp)); + pt1->right(buildRowPredicate(gwip, columns, vals, predicateOp)); pt = pt1; } @@ -1644,7 +1706,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It for (uint32_t i = 0; i < columns->columnVec().size(); i++) { - ConstantFilter* cf = new ConstantFilter(); + std::unique_ptr cf(new ConstantFilter()); sop.reset(lo->clone()); cf->op(sop); @@ -1652,7 +1714,9 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It // no optimization for non-simple column because CP won't apply if (!sc) + { continue; + } ssc.reset(sc->clone()); cf->col(ssc); @@ -1674,9 +1738,11 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It } if (j < valVec.size()) + { continue; + } - tmpPtStack.push(new ParseTree(cf)); + tmpPtStack.push(new ParseTree(cf.release())); } // "and" all the filters together @@ -2027,7 +2093,10 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) } if (!gwip->rcWorkStack.empty()) + { + delete gwip->rcWorkStack.top(); gwip->rcWorkStack.pop(); // pop gwip->scsp + } if (cf->filterList().size() < inp->argument_count() - 1) { @@ -2839,7 +2908,6 @@ void setError(THD* thd, uint32_t errcode, string errmsg) void setError(THD* thd, uint32_t errcode, string errmsg, gp_walk_info& gwi) { setError(thd, errcode, errmsg); - clearStacks(gwi); } int setErrorAndReturn(gp_walk_info& gwi) @@ -4114,6 +4182,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (!sptp) { nonSupport = true; + delete fc; return NULL; } @@ -4128,6 +4197,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non nonSupport = true; gwi.fatalParseError = true; gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SUB_EXPRESSION); + delete fc; return NULL; } @@ -4159,6 +4229,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (!rc || nonSupport) { nonSupport = true; + delete fc; return NULL; } @@ -4187,6 +4258,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non else { nonSupport = true; + delete fc; return NULL; } } @@ -4583,6 +4655,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS gwi.inCaseStmt = false; if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top() == *sptp.get()) { + delete gwi.ptWorkStack.top(); gwi.ptWorkStack.pop(); } } @@ -4603,6 +4676,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // We need to pop whichever stack is holding it, if any. if ((!gwi.rcWorkStack.empty()) && *gwi.rcWorkStack.top() == parm) { + delete gwi.rcWorkStack.top(); gwi.rcWorkStack.pop(); } else if (!gwi.ptWorkStack.empty()) @@ -4610,7 +4684,10 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS ReturnedColumn* ptrc = dynamic_cast(gwi.ptWorkStack.top()->data()); if (ptrc && *ptrc == *parm) + { + delete gwi.ptWorkStack.top(); gwi.ptWorkStack.pop(); + } } } else @@ -4620,6 +4697,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // We need to pop whichever stack is holding it, if any. if ((!gwi.ptWorkStack.empty()) && *gwi.ptWorkStack.top()->data() == sptp->data()) { + delete gwi.ptWorkStack.top(); gwi.ptWorkStack.pop(); } else if (!gwi.rcWorkStack.empty()) @@ -4630,6 +4708,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS if (ptrc && *ptrc == *gwi.rcWorkStack.top()) { + delete gwi.rcWorkStack.top(); gwi.rcWorkStack.pop(); } } @@ -5301,6 +5380,18 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) ac->constCol(SRCP(rc)); break; } + // the "rc" can be in gwi.no_parm_func_list. erase it from that list and + // then delete it. + // kludge, I know. + uint32_t i; + + for (i = 0; gwi.no_parm_func_list[i] != rc && i < gwi.no_parm_func_list.size(); i++) { } + + if (i < gwi.no_parm_func_list.size()) + { + gwi.no_parm_func_list.erase(gwi.no_parm_func_list.begin() + i); + delete rc; + } } } @@ -5996,7 +6087,10 @@ void gp_walk(const Item* item, void* arg) { // @bug 4215. remove the argument in rcWorkStack. if (!gwip->rcWorkStack.empty()) + { + delete gwip->rcWorkStack.top(); gwip->rcWorkStack.pop(); + } break; } @@ -6031,6 +6125,7 @@ void gp_walk(const Item* item, void* arg) for (uint32_t i = 0; i < ifp->argument_count() && !gwip->rcWorkStack.empty(); i++) { + delete gwip->rcWorkStack.top(); gwip->rcWorkStack.pop(); } @@ -6207,6 +6302,7 @@ void gp_walk(const Item* item, void* arg) } else { + delete rhs; gwip->ptWorkStack.push(lhs); continue; } @@ -6740,21 +6836,21 @@ int processFrom(bool& isUnion, SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& if (table_ptr->derived) { SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - FromSubQuery fromSub(gwi, select_cursor); + FromSubQuery* fromSub = new FromSubQuery(gwi, select_cursor); string alias(table_ptr->alias.str); if (lower_case_table_names) { boost::algorithm::to_lower(alias); } - fromSub.alias(alias); + fromSub->alias(alias); CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); // @bug 3852. check return execplan - SCSEP plan = fromSub.transform(); + SCSEP plan = fromSub->transform(); if (!plan) { - setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); + setError(gwi.thd, ER_INTERNAL_ERROR, fromSub->gwip().parseErrorText, gwi); CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); return ER_INTERNAL_ERROR; } @@ -6878,7 +6974,7 @@ int processFrom(bool& isUnion, SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& plan->data(csep->data()); // gwi for the union unit - gp_walk_info union_gwi(gwi.timeZone); + gp_walk_info union_gwi(gwi.timeZone, gwi.subQueriesChain); union_gwi.thd = gwi.thd; uint32_t err = 0; @@ -7081,14 +7177,28 @@ int processWhere(SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep, const s std::stack outerJoinStack; if ((failed = buildJoin(gwi, select_lex.top_join_list, outerJoinStack))) + { + while (!outerJoinStack.empty()) + { + delete outerJoinStack.top(); + outerJoinStack.pop(); + } return failed; + } if (gwi.subQuery) { for (uint i = 0; i < gwi.viewList.size(); i++) { if ((failed = gwi.viewList[i]->processJoin(gwi, outerJoinStack))) + { + while (!outerJoinStack.empty()) + { + delete outerJoinStack.top(); + outerJoinStack.pop(); + } return failed; + } } } @@ -7103,7 +7213,7 @@ int processWhere(SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep, const s // is pushed to rcWorkStack. if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) { - filters = new ParseTree(gwi.rcWorkStack.top()); + gwi.ptWorkStack.push(new ParseTree(gwi.rcWorkStack.top())); gwi.rcWorkStack.pop(); } @@ -7156,6 +7266,7 @@ int processWhere(SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep, const s "columnstore_max_allowed_in_values " "threshold.", gwi); + delete filters; return ER_CHECK_NOT_IMPLEMENTED; } @@ -7180,6 +7291,26 @@ int processWhere(SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep, const s csep->filters(filters); } + if (!gwi.rcWorkStack.empty()) + { + while(!gwi.rcWorkStack.empty()) + { + ReturnedColumn* t = gwi.rcWorkStack.top(); + delete t; + gwi.rcWorkStack.pop(); + } + } + if (!gwi.ptWorkStack.empty()) + { + while(!gwi.ptWorkStack.empty()) + { + ParseTree* t = gwi.ptWorkStack.top(); + delete t; + gwi.ptWorkStack.pop(); + } + } + + return 0; } @@ -7515,7 +7646,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i vector funcFieldVec; // empty rcWorkStack and ptWorkStack. They should all be empty by now. - clearStacks(gwi); + clearStacks(gwi, false, true); // indicate the starting pos of scalar returned column, because some join column // has been inserted to the returned column list. @@ -7852,6 +7983,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.parseErrorText = "Unsupported Item in SELECT subquery."; setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; } @@ -7963,8 +8095,8 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // Having clause handling gwi.clauseType = HAVING; - clearStacks(gwi); - ParseTree* havingFilter = 0; + clearStacks(gwi, false, true); + std::unique_ptr havingFilter; // clear fatalParseError that may be left from post process functions gwi.fatalParseError = false; gwi.parseErrorText = ""; @@ -7990,20 +8122,20 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // @bug 4215. some function filter will be in the rcWorkStack. if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) { - havingFilter = new ParseTree(gwi.rcWorkStack.top()); + gwi.ptWorkStack.push(new ParseTree(gwi.rcWorkStack.top())); gwi.rcWorkStack.pop(); } while (!gwi.ptWorkStack.empty()) { - havingFilter = gwi.ptWorkStack.top(); + havingFilter.reset(gwi.ptWorkStack.top()); gwi.ptWorkStack.pop(); if (gwi.ptWorkStack.empty()) break; ptp = new ParseTree(new LogicOperator("and")); - ptp->left(havingFilter); + ptp->left(havingFilter.release()); rhs = gwi.ptWorkStack.top(); gwi.ptWorkStack.pop(); ptp->right(rhs); @@ -8034,9 +8166,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i if (havingFilter) { ParseTree* ptp = new ParseTree(new LogicOperator("and")); - ptp->left(havingFilter); + ptp->left(havingFilter.release()); ptp->right(inToExistsFilter); - havingFilter = ptp; + havingFilter.reset(ptp); } else { @@ -8266,6 +8398,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i { if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) { + delete rc; rc = gwi.returnedCols[j].get()->clone(); rc->orderPos(j); break; @@ -8278,6 +8411,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i { if (ifp->name.length && string(ifp->name.str) == gwi.returnedCols[j].get()->alias()) { + delete rc; rc = gwi.returnedCols[j].get()->clone(); rc->orderPos(j); break; @@ -8629,21 +8763,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i { } } - else if (ord_item->type() == Item::FUNC_ITEM) - { - // @bug5636. check if this order by column is on the select list - ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (rc && rc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - break; - } - } - } } } @@ -8746,20 +8865,28 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i csep->orderByCols(gwi.orderByCols); csep->returnedCols(gwi.returnedCols); csep->columnMap(gwi.columnMap); - csep->having(havingFilter); + csep->having(havingFilter.release()); csep->derivedTableList(gwi.derivedTbList); csep->selectSubList(selectSubList); csep->subSelectList(gwi.subselectList); - clearStacks(gwi); return 0; } int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti, long timeZone) { - gp_walk_info* gwi = ti.condInfo; - if (!gwi) - gwi = new gp_walk_info(timeZone); + SubQueryChainHolder chainHolder; + bool allocated = false; + gp_walk_info* gwi; + if (ti.condInfo) + { + gwi = &ti.condInfo->gwi; + } + else + { + allocated = true; + gwi = new gp_walk_info(timeZone, &chainHolder.chain); + } gwi->thd = thd; LEX* lex = thd->lex; @@ -8810,7 +8937,7 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti, long timeZone) // get filter if (ti.condInfo) { - gp_walk_info* gwi = ti.condInfo; + gp_walk_info* gwi = &ti.condInfo->gwi; ParseTree* filters = 0; ParseTree* ptp = 0; ParseTree* rhs = 0; @@ -8856,6 +8983,10 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti, long timeZone) // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering csep->stringScanThreshold(get_string_scan_threshold(gwi->thd)); + if (allocated) + { + delete gwi; + } return 0; } @@ -8865,7 +8996,8 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) const char* timeZone = thd->variables.time_zone->get_name()->ptr(); long timeZoneOffset; dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); - gp_walk_info gwi(timeZoneOffset); + SubQuery* chain = nullptr; + gp_walk_info gwi(timeZoneOffset, &chain); gwi.thd = thd; gwi.isGroupByHandler = true; int status = getGroupPlan(gwi, *select_lex, csep, gi); @@ -9003,6 +9135,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro #ifdef DEBUG_WALK_COND cerr << "getGroupPlan()" << endl; #endif + idbassert_s(false, "getGroupPlan is utterly out of date"); // XXX: rollup is currently not supported (not tested) in this part. // but this is not triggered in any of tests. @@ -9103,21 +9236,21 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro SELECT_LEX* select_cursor = table_ptr->derived->first_select(); // Use Pushdown handler for subquery processing - FromSubQuery fromSub(gwi, select_cursor); + FromSubQuery* fromSub = new FromSubQuery(gwi, select_cursor); string alias(table_ptr->alias.str); if (lower_case_table_names) { boost::algorithm::to_lower(alias); } - fromSub.alias(alias); + fromSub->alias(alias); CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); // @bug 3852. check return execplan - SCSEP plan = fromSub.transform(); + SCSEP plan = fromSub->transform(); if (!plan) { - setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); + setError(gwi.thd, ER_INTERNAL_ERROR, fromSub->gwip().parseErrorText, gwi); CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); return ER_INTERNAL_ERROR; } @@ -9369,7 +9502,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro bool redo = false; // empty rcWorkStack and ptWorkStack. They should all be empty by now. - clearStacks(gwi); + clearStacks(gwi, false); // indicate the starting pos of scalar returned column, because some join column // has been inserted to the returned column list. @@ -9819,7 +9952,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro // Having clause handling gwi.clauseType = HAVING; - clearStacks(gwi); + clearStacks(gwi, false); ParseTree* havingFilter = 0; // clear fatalParseError that may be left from post process functions gwi.fatalParseError = false; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index fc0ec5078..955602b62 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -128,6 +128,7 @@ using namespace funcexp; #include "ha_mcs_datatype.h" #include "statistics.h" #include "ha_mcs_logging.h" +#include "ha_subquery.h" namespace cal_impl_if { @@ -332,7 +333,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, long t std::vector& colTypes = ti.tpl_scan_ctx->ctp; - RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; + std::shared_ptr rowGroup = ti.tpl_scan_ctx->rowGroup; // table mode mysql expects all columns of the table. mapping between columnoid and position in rowgroup // set coltype.position to be the position in rowgroup. only set once. @@ -656,7 +657,7 @@ vector getOnUpdateTimestampColumns(string& schema, string& tableName, in rowgroup::RGData rgData; ByteStream::quadbyte qb = 4; msg << qb; - rowgroup::RowGroup* rowGroup = 0; + std::unique_ptr rowGroup; uint32_t rowCount; exemgrClient->write(msg); @@ -709,7 +710,7 @@ vector getOnUpdateTimestampColumns(string& schema, string& tableName, in if (!rowGroup) { // This is mete data - rowGroup = new rowgroup::RowGroup(); + rowGroup.reset(new rowgroup::RowGroup()); rowGroup->deserialize(msg); qb = 100; msg.restart(); @@ -888,7 +889,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c //@Bug 2753. the memory already freed by destructor of UpdateSqlStatement if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command)) { - ColumnAssignment* columnAssignmentPtr; + ColumnAssignment* columnAssignmentPtr = nullptr; Item_field* item; List_iterator_fast field_it(thd->lex->first_select_lex()->item_list); List_iterator_fast value_it(thd->lex->value_list); @@ -919,6 +920,8 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c } else if (strcmp(tableName.c_str(), tmpTableName.c_str()) != 0) { + delete colAssignmentListPtr; + delete columnAssignmentPtr; //@ Bug3326 error out for multi table update string emsg(IDBErrorInfo::instance()->errorMsg(ERR_UPDATE_NOT_SUPPORT_FEATURE)); thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, emsg.c_str()); @@ -929,6 +932,8 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c if (!item->db_name.str) { + delete colAssignmentListPtr; + delete columnAssignmentPtr; //@Bug 5312. if subselect, wait until the schema info is available. if (thd->derived_tables_processing) return 0; @@ -1003,7 +1008,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c // sysdate() etc. if (!hasNonSupportItem && !cal_impl_if::nonConstFunc(ifp) && tmpVec.size() == 0) { - gp_walk_info gwi2(gwi.timeZone); + gp_walk_info gwi2(gwi.timeZone, gwi.subQueriesChain); gwi2.thd = thd; SRCP srcp(buildReturnedColumn(value, gwi2, gwi2.fatalParseError)); ConstantColumn* constCol = dynamic_cast(srcp.get()); @@ -1114,6 +1119,8 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c } else if (value->type() == Item::WINDOW_FUNC_ITEM) { + delete colAssignmentListPtr; + delete columnAssignmentPtr; setError(thd, ER_INTERNAL_ERROR, logging::IDBErrorInfo::instance()->errorMsg(ERR_WF_UPDATE)); return ER_CHECK_NOT_IMPLEMENTED; } @@ -1178,6 +1185,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c if (colAssignmentListPtr->empty() && ha_mcs_common::isUpdateStatement(thd->lex->sql_command)) { ci->affectedRows = 0; + delete colAssignmentListPtr; return 0; } @@ -1205,7 +1213,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c boost::algorithm::to_lower(aTableName.table); } - CalpontDMLPackage* pDMLPackage = 0; + std::shared_ptr pDMLPackage; // dmlStmt += ";"; IDEBUG(cout << "STMT: " << dmlStmt << " and sessionID " << thd->thread_id << endl); VendorDMLStatement dmlStatement(dmlStmt, sessionID); @@ -1215,7 +1223,6 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c else dmlStatement.set_DMLStatementType(DML_DELETE); - TableName* qualifiedTablName = new TableName(); UpdateSqlStatement updateStmt; //@Bug 2753. To make sure the momory is freed. @@ -1223,10 +1230,11 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command)) { + TableName* qualifiedTablName = new TableName(); qualifiedTablName->fName = tableName; qualifiedTablName->fSchema = schemaName; updateStmt.fNamePtr = qualifiedTablName; - pDMLPackage = CalpontDMLFactory::makeCalpontUpdatePackageFromMysqlBuffer(dmlStatement, updateStmt); + pDMLPackage.reset(CalpontDMLFactory::makeCalpontUpdatePackageFromMysqlBuffer(dmlStatement, updateStmt)); } else if ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) //@Bug 6121 error out on multi tables delete. { @@ -1246,9 +1254,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c boost::algorithm::to_lower(tableName); boost::algorithm::to_lower(aliasName); } - qualifiedTablName->fName = tableName; - qualifiedTablName->fSchema = schemaName; - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement); + pDMLPackage.reset(CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement)); } else { @@ -1271,9 +1277,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c boost::algorithm::to_lower(tableName); boost::algorithm::to_lower(aliasName); } - qualifiedTablName->fName = tableName; - qualifiedTablName->fSchema = schemaName; - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement); + pDMLPackage.reset(CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement)); } } else @@ -1288,9 +1292,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c boost::algorithm::to_lower(tableName); boost::algorithm::to_lower(aliasName); } - qualifiedTablName->fName = tableName; - qualifiedTablName->fSchema = schemaName; - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement); + pDMLPackage.reset(CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStatement)); } if (!pDMLPackage) @@ -1549,7 +1551,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c updateCP->serialize(*plan); pDMLPackage->write(bytestream); - delete pDMLPackage; + pDMLPackage.reset(); ByteStream::byte b = 0; ByteStream::octbyte rows = 0; @@ -1633,12 +1635,12 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c // cout << "doUpdateDelete start new DMLProc client for ctrl-c " << " for session " << sessionID // << endl; VendorDMLStatement cmdStmt("CTRL+C", DML_COMMAND, sessionID); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + std::shared_ptr pDMLPackage(CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt)); pDMLPackage->set_TimeZone(timeZoneOffset); ByteStream bytestream; bytestream << static_cast(sessionID); pDMLPackage->write(bytestream); - delete pDMLPackage; + pDMLPackage.reset(); b = 1; retry = maxRetries; errorMsg = "Command canceled by user"; @@ -1756,13 +1758,13 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c if (command != "") { VendorDMLStatement cmdStmt(command, DML_COMMAND, sessionID); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + std::shared_ptr pDMLPackage(CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt)); pDMLPackage->set_TimeZone(timeZoneOffset); pDMLPackage->setTableOid(ci->tableOid); ByteStream bytestream; bytestream << static_cast(sessionID); pDMLPackage->write(bytestream); - delete pDMLPackage; + pDMLPackage.reset(); ByteStream::byte bc; std::string errMsg; @@ -2145,7 +2147,8 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected_rows, const char* timeZone = thd->variables.time_zone->get_name()->ptr(); long timeZoneOffset; dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); - cal_impl_if::gp_walk_info gwi(timeZoneOffset); + SubQueryChainHolder chainHolder; + cal_impl_if::gp_walk_info gwi(timeZoneOffset, &chainHolder.chain); gwi.thd = thd; int rc = 0; @@ -2168,6 +2171,7 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected_rows, *affected_rows = ci->affectedRows; } + return rc; } @@ -2178,7 +2182,8 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) const char* timeZone = thd->variables.time_zone->get_name()->ptr(); long timeZoneOffset; dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); - gp_walk_info gwi(timeZoneOffset); + SubQueryChainHolder chainHolder; + gp_walk_info gwi(timeZoneOffset, &chainHolder.chain); gwi.thd = thd; if (thd->slave_thread && !get_replication_slave(thd) && @@ -2452,9 +2457,9 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) ti = ci->tableMap[table]; ti.msTablePtr = table; - if (ti.tpl_ctx == nullptr) + if (!ti.tpl_ctx) { - ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_ctx.reset(new sm::cpsm_tplh_t()); ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); } @@ -2729,7 +2734,6 @@ int ha_mcs_impl_rnd_end(TABLE* table, bool is_pushdown_hand) else ci->cal_conn_hndl = hndl; - ti.tpl_ctx = 0; } catch (IDBExcept& e) { @@ -3737,6 +3741,13 @@ int ha_mcs_impl_delete_row(const uchar* buf) return 0; } +// this place is as good as any. +ext_cond_info::ext_cond_info(long timeZone) + : chainHolder(new SubQueryChainHolder()) + , gwi(timeZone, &chainHolder->chain) +{ +} + COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector& condStack) { THD* thd = current_thd; @@ -3766,7 +3777,8 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector& condSt const char* timeZone = thd->variables.time_zone->get_name()->ptr(); long timeZoneOffset; dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); - gp_walk_info gwi(timeZoneOffset); + SubQueryChainHolder chainHolder; + gp_walk_info gwi(timeZoneOffset, &chainHolder.chain); gwi.condPush = true; gwi.sessionid = tid2sid(thd->thread_id); cout << "------------------ cond push -----------------------" << endl; @@ -3782,16 +3794,17 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector& condSt const char* timeZone = thd->variables.time_zone->get_name()->ptr(); long timeZoneOffset; dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); - ti.condInfo = new gp_walk_info(timeZoneOffset); + ti.condInfo = new ext_cond_info(timeZoneOffset); //new gp_walk_info(timeZoneOffset); } - gp_walk_info* gwi = ti.condInfo; + gp_walk_info* gwi = &ti.condInfo->gwi; gwi->dropCond = false; gwi->fatalParseError = false; gwi->condPush = true; gwi->thd = thd; gwi->sessionid = tid2sid(thd->thread_id); cond->traverse_cond(gp_walk, gwi, Item::POSTFIX); + clearDeleteStacks(*gwi); ci->tableMap[table] = ti; if (gwi->fatalParseError) @@ -4153,12 +4166,12 @@ int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table) mapiter = ci->tableMap.find(tl->table); if (mapiter != ci->tableMap.end() && mapiter->second.condInfo != NULL && - mapiter->second.condInfo->condPush) + mapiter->second.condInfo->gwi.condPush) { - while (!mapiter->second.condInfo->ptWorkStack.empty()) + while (!mapiter->second.condInfo->gwi.ptWorkStack.empty()) { - ptIt = mapiter->second.condInfo->ptWorkStack.top(); - mapiter->second.condInfo->ptWorkStack.pop(); + ptIt = mapiter->second.condInfo->gwi.ptWorkStack.top(); + mapiter->second.condInfo->gwi.ptWorkStack.pop(); gi.pushedPts.push_back(ptIt); } } @@ -4338,7 +4351,7 @@ int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table) { // MCOL-1601 Using stacks of ExeMgr conn hndls, table and scan contexts. - ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_ctx.reset(new sm::cpsm_tplh_t()); ti.tpl_ctx_st.push(ti.tpl_ctx); ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); ti.tpl_scan_ctx_st.push(ti.tpl_scan_ctx); @@ -4593,6 +4606,7 @@ int ha_mcs_impl_group_by_end(TABLE* table) { bool ask_4_stats = (ci->traceFlags) ? true : false; sm::tpl_close(ti.tpl_ctx, &hndl, ci->stats, ask_4_stats, clearScanCtx); + ti.tpl_ctx = 0; } // Normaly stats variables are set in external_lock method but we set it here // since they we pretend we are in vtable_disabled mode and the stats vars won't be set. @@ -4608,10 +4622,13 @@ int ha_mcs_impl_group_by_end(TABLE* table) ci->miniStats += hndl->miniStats; } } + else + { + ti.tpl_ctx.reset(); + } ci->cal_conn_hndl = hndl; - ti.tpl_ctx = 0; } catch (IDBExcept& e) { @@ -4684,6 +4701,8 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool IDEBUG(cout << "pushdown_init for table " << endl); THD* thd = current_thd; + SubQueryChainHolder chainHolder; + if (thd->slave_thread && !get_replication_slave(thd) && ha_mcs_common::isDMLStatement(thd->lex->sql_command)) return 0; @@ -4691,7 +4710,7 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool const char* timeZone = thd->variables.time_zone->get_name()->ptr(); long timeZoneOffset; dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); - gp_walk_info gwi(timeZoneOffset); + gp_walk_info gwi(timeZoneOffset, &chainHolder.chain); gwi.thd = thd; bool err = false; @@ -5049,7 +5068,7 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool { if (ti.tpl_ctx == 0) { - ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_ctx.reset(new sm::cpsm_tplh_t()); ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); } @@ -5174,7 +5193,7 @@ int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone) { if (ti.tpl_ctx == 0) { - ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_ctx.reset(new sm::cpsm_tplh_t()); ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); } diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 010b056c7..98b488f57 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -179,7 +179,15 @@ struct gp_walk_info TableOnExprList tableOnExprList; std::vector condList; - gp_walk_info(long timeZone_) + // All SubQuery allocations are single-linked into this chain. + // At the end of gp_walk_info processing we can free whole chain at once. + // This is done so because the juggling of SubQuery pointers in the + // ha_mcs_execplan code. + // There is a struct SubQueryChainHolder down below to hold chain root and free + // the chain using sorta kinda RAII. + SubQuery** subQueriesChain; + + gp_walk_info(long timeZone_, SubQuery** subQueriesChain_) : sessionid(0) , fatalParseError(false) , condPush(false) @@ -204,12 +212,21 @@ struct gp_walk_info , timeZone(timeZone_) , inSubQueryLHS(nullptr) , inSubQueryLHSItem(nullptr) + , subQueriesChain(subQueriesChain_) { } + ~gp_walk_info(); - ~gp_walk_info() - { - } +}; + +struct SubQueryChainHolder; +struct ext_cond_info +{ + // having this as a direct field would introduce + // circular dependency on header inclusion with ha_subquery.h. + boost::shared_ptr chainHolder; + gp_walk_info gwi; + ext_cond_info(long timeZone); // needs knowledge on SubQueryChainHolder, will be defined elsewhere }; struct cal_table_info @@ -223,17 +240,14 @@ struct cal_table_info cal_table_info() : tpl_ctx(0), c(0), msTablePtr(0), conn_hndl(0), condInfo(0), moreRows(false) { } - ~cal_table_info() - { - } - sm::cpsm_tplh_t* tpl_ctx; - std::stack tpl_ctx_st; + sm::sp_cpsm_tplh_t tpl_ctx; + std::stack tpl_ctx_st; sm::sp_cpsm_tplsch_t tpl_scan_ctx; std::stack tpl_scan_ctx_st; unsigned c; // for debug purpose TABLE* msTablePtr; // no ownership sm::cpsm_conhdl_t* conn_hndl; - gp_walk_info* condInfo; + ext_cond_info* condInfo; execplan::SCSEP csep; bool moreRows; // are there more rows to consume (b/c of limit) }; @@ -400,6 +414,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& cse void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); void setError(THD* thd, uint32_t errcode, const std::string errmsg); void gp_walk(const Item* item, void* arg); +void clearDeleteStacks(gp_walk_info& gwi); void parse_item(Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& parseInfo, gp_walk_info* gwip = NULL); const std::string bestTableName(const Item_field* ifp); diff --git a/dbcon/mysql/ha_pseudocolumn.cpp b/dbcon/mysql/ha_pseudocolumn.cpp index 7bd75b6a7..1da0eaa01 100644 --- a/dbcon/mysql/ha_pseudocolumn.cpp +++ b/dbcon/mysql/ha_pseudocolumn.cpp @@ -475,6 +475,7 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& funcexp::Func_idbpartition* idbpartition = new funcexp::Func_idbpartition(); fc->operationType(idbpartition->operationType(parms, fc->resultType())); fc->alias(ifp->full_name() ? ifp->full_name() : ""); + delete idbpartition; return fc; } diff --git a/dbcon/mysql/ha_scalar_sub.cpp b/dbcon/mysql/ha_scalar_sub.cpp index c1014c9f4..e2afd8367 100644 --- a/dbcon/mysql/ha_scalar_sub.cpp +++ b/dbcon/mysql/ha_scalar_sub.cpp @@ -230,6 +230,7 @@ execplan::ParseTree* ScalarSub::buildParseTree(PredicateOperator* op) { fGwip.fatalParseError = true; fGwip.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_OPERATOR_WITH_LIST); + delete op; return NULL; } @@ -244,7 +245,7 @@ execplan::ParseTree* ScalarSub::buildParseTree(PredicateOperator* op) csep->subType(CalpontSelectExecutionPlan::SINGLEROW_SUBS); // gwi for the sub query - gp_walk_info gwi(fGwip.timeZone); + gp_walk_info gwi(fGwip.timeZone, fGwip.subQueriesChain); gwi.thd = fGwip.thd; gwi.subQuery = this; @@ -270,6 +271,7 @@ execplan::ParseTree* ScalarSub::buildParseTree(PredicateOperator* op) fGwip.parseErrorText = gwi.parseErrorText; } + delete op; return NULL; } diff --git a/dbcon/mysql/ha_select_sub.cpp b/dbcon/mysql/ha_select_sub.cpp index 5a83e245b..3939c11b4 100644 --- a/dbcon/mysql/ha_select_sub.cpp +++ b/dbcon/mysql/ha_select_sub.cpp @@ -67,7 +67,7 @@ SCSEP SelectSubQuery::transform() csep->subType(CalpontSelectExecutionPlan::SELECT_SUBS); // gwi for the sub query - gp_walk_info gwi(fGwip.timeZone); + gp_walk_info gwi(fGwip.timeZone, fGwip.subQueriesChain); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_subquery.h b/dbcon/mysql/ha_subquery.h index 029db0051..8195c520e 100644 --- a/dbcon/mysql/ha_subquery.h +++ b/dbcon/mysql/ha_subquery.h @@ -45,6 +45,8 @@ class SubQuery public: SubQuery(gp_walk_info& gwip) : fGwip(gwip), fCorrelated(false) { + next = *gwip.subQueriesChain; + *gwip.subQueriesChain = this; } virtual ~SubQuery() { @@ -68,11 +70,28 @@ class SubQuery { } + SubQuery* next; protected: gp_walk_info& fGwip; bool fCorrelated; }; +struct SubQueryChainHolder +{ + SubQuery* chain; + SubQueryChainHolder () : chain(nullptr) { } + ~SubQueryChainHolder () + { + while (chain) + { + SubQuery* next = chain->next; + delete chain; + chain = next; + } + } +}; + + /** * @brief A class to represent a generic WHERE clause subquery */ diff --git a/dbcon/mysql/ha_view.cpp b/dbcon/mysql/ha_view.cpp index b419ee9ee..e78cea5da 100644 --- a/dbcon/mysql/ha_view.cpp +++ b/dbcon/mysql/ha_view.cpp @@ -66,7 +66,7 @@ void View::transform() csep->sessionID(fParentGwip->sessionid); // gwi for the sub query - gp_walk_info gwi(fParentGwip->timeZone); + gp_walk_info gwi(fParentGwip->timeZone, fParentGwip->subQueriesChain); gwi.thd = fParentGwip->thd; uint32_t sessionID = csep->sessionID(); @@ -151,6 +151,7 @@ void View::transform() if (gwi.fatalParseError) { setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText); + delete csep; return; } } @@ -158,6 +159,7 @@ void View::transform() { setError(gwi.thd, ER_INTERNAL_ERROR, ie.what()); CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + delete csep; return; } catch (...) @@ -165,6 +167,7 @@ void View::transform() string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); setError(gwi.thd, ER_INTERNAL_ERROR, emsg); CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + delete csep; return; } @@ -177,6 +180,7 @@ void View::transform() // merge view list to parent fParentGwip->viewList.insert(fParentGwip->viewList.end(), gwi.viewList.begin(), gwi.viewList.end()); + gwi.viewList.clear(); // merge non-collapsed outer join to parent select stack tmpstack; @@ -192,6 +196,12 @@ void View::transform() fParentGwip->ptWorkStack.push(tmpstack.top()); tmpstack.pop(); } + while (!gwi.rcWorkStack.empty()) { + delete gwi.rcWorkStack.top(); + gwi.rcWorkStack.pop(); + } + + delete csep; } uint32_t View::processJoin(gp_walk_info& gwi, std::stack& outerJoinStack) diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index a30ba0b71..b3d992077 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -316,7 +316,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n Item_window_func* wf = (Item_window_func*)item; Item_sum* item_sum = wf->window_func(); string funcName = ConvertFuncName(item_sum); - WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); + std::unique_ptr ac(new WindowFunctionColumn(funcName)); ac->timeZone(gwi.timeZone); ac->distinct(item_sum->has_with_distinct()); Window_spec* win_spec = wf->window_spec; @@ -902,8 +902,10 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n ac->charsetNumber(item->collation.collation->number); // put ac on windowFuncList - gwi.windowFuncList.push_back(ac); - return ac; + // we clone our managed pointer to put it into uunmanaged world. + WindowFunctionColumn* retAC = ac.release(); + gwi.windowFuncList.push_back(retAC); + return retAC; } } // namespace cal_impl_if diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index 0d6b00c3d..c12793cbf 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -187,7 +187,6 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th if (schema_table_store_record(thd, table)) { - delete emp; return 1; } @@ -197,13 +196,27 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th return 0; } +struct refresher +{ + BRM::DBRM* guarded; + refresher() + { + guarded = new BRM::DBRM(); + } + ~refresher() + { + delete guarded; + BRM::DBRM::refreshShm(); + } +}; static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) { BRM::OID_t cond_oid = 0; TABLE* table = tables->table; - BRM::DBRM::refreshShm(); - BRM::DBRM* emp = new BRM::DBRM(); + BRM::DBRM* emp; + refresher shmRefresher; + emp = shmRefresher.guarded; if (!emp || !emp->isDBRMReady()) { @@ -289,7 +302,6 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) return 1; } - delete emp; return 0; } diff --git a/dbcon/mysql/sm.cpp b/dbcon/mysql/sm.cpp index 098b2a655..caf6f156e 100644 --- a/dbcon/mysql/sm.cpp +++ b/dbcon/mysql/sm.cpp @@ -280,7 +280,7 @@ namespace sm { const std::string DEFAULT_SAVE_PATH = "/var/tmp"; -status_t tpl_open(tableid_t tableid, cpsm_tplh_t* ntplh, cpsm_conhdl_t* conn_hdl) +status_t tpl_open(tableid_t tableid, sp_cpsm_tplh_t& ntplh, cpsm_conhdl_t* conn_hdl) { SMDEBUGLOG << "tpl_open: ntplh: " << ntplh << " conn_hdl: " << conn_hdl << " tableid: " << tableid << endl; @@ -359,7 +359,7 @@ status_t tpl_scan_close(sp_cpsm_tplsch_t& ntplsch) return STATUS_OK; } -status_t tpl_close(cpsm_tplh_t* ntplh, cpsm_conhdl_t** conn_hdl, QueryStats& stats, bool ask_4_stats, +status_t tpl_close(sp_cpsm_tplh_t& ntplh, cpsm_conhdl_t** conn_hdl, QueryStats& stats, bool ask_4_stats, bool clear_scan_ctx) { cpsm_conhdl_t* hndl = *conn_hdl; @@ -369,7 +369,7 @@ status_t tpl_close(cpsm_tplh_t* ntplh, cpsm_conhdl_t** conn_hdl, QueryStats& sta SMDEBUGLOG << " tableid: " << ntplh->tableid; SMDEBUGLOG << endl; - delete ntplh; + ntplh.reset(); // determine end of result set and end of statement execution if (hndl->queryState == QUERY_IN_PROCESS) diff --git a/dbcon/mysql/sm.h b/dbcon/mysql/sm.h index 06ad59ba2..0e055f8e0 100644 --- a/dbcon/mysql/sm.h +++ b/dbcon/mysql/sm.h @@ -137,12 +137,11 @@ struct cpsm_tplsch_t } ~cpsm_tplsch_t() { - delete rowGroup; } tableid_t tableid; uint64_t rowsreturned; - rowgroup::RowGroup* rowGroup; + std::shared_ptr rowGroup; messageqcpp::ByteStream bs; // rowgroup bytestream. need to stay with the life span of rowgroup uint32_t traceFlags; // @bug 649 @@ -158,7 +157,7 @@ struct cpsm_tplsch_t { if (!rowGroup) { - rowGroup = new rowgroup::RowGroup(); + rowGroup.reset(new rowgroup::RowGroup()); rowGroup->deserialize(bs); } else @@ -280,6 +279,7 @@ struct cpsm_tplh_t uint16_t saveFlag; int bandsInTable; }; +typedef std::shared_ptr sp_cpsm_tplh_t; struct cpsm_tid_t { @@ -293,11 +293,11 @@ struct cpsm_tid_t extern status_t sm_init(uint32_t, cpsm_conhdl_t**, uint32_t columnstore_local_query = false); extern status_t sm_cleanup(cpsm_conhdl_t*); -extern status_t tpl_open(tableid_t, cpsm_tplh_t*, cpsm_conhdl_t*); +extern status_t tpl_open(tableid_t, sp_cpsm_tplh_t&, cpsm_conhdl_t*); extern status_t tpl_scan_open(tableid_t, sp_cpsm_tplsch_t&, cpsm_conhdl_t*); extern status_t tpl_scan_fetch(sp_cpsm_tplsch_t&, cpsm_conhdl_t*, int* k = 0); extern status_t tpl_scan_close(sp_cpsm_tplsch_t&); -extern status_t tpl_close(cpsm_tplh_t*, cpsm_conhdl_t**, querystats::QueryStats& stats, bool ask_4_stats, +extern status_t tpl_close(sp_cpsm_tplh_t&, cpsm_conhdl_t**, querystats::QueryStats& stats, bool ask_4_stats, bool clear_scan_ctx = false); } // namespace sm diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b7632f301..549f92a7c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -33,10 +33,9 @@ if (WITH_UNITTESTS) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) gtest_add_tests(TARGET rowgroup_tests TEST_PREFIX columnstore:) - add_executable(rewritetest rewritetest.cpp) add_dependencies(rewritetest googletest) - target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} messageqcpp execplan) + target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) gtest_add_tests(TARGET rewritetest TEST_PREFIX columnstore:) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) diff --git a/tests/scripts/fullmtr.sh b/tests/scripts/fullmtr.sh index 53f5ef20f..f3b16b570 100644 --- a/tests/scripts/fullmtr.sh +++ b/tests/scripts/fullmtr.sh @@ -7,6 +7,9 @@ CURRENT_DIR=`pwd` mysql -e "create database if not exists test;" SOCKET=`mysql -e "show variables like 'socket';" | grep socket | cut -f2` +export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=/core/asan.hz + + cd ${INSTALLED_MTR_PATH} if [[ ! -d ${COLUMSNTORE_MTR_INSTALLED} ]]; then @@ -21,7 +24,15 @@ if [[ ! -d '/data/qa/source/dbt3/' || ! -d '/data/qa/source/ssb/' ]]; then fi run_suite() { - ./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/$1 $2 | tee $CURRENT_DIR/mtr.$1.log 2>&1 + ls /core >$CURRENT_DIR/mtr.$1.cores-before + ./mtr --force --max-test-fail=0 --testcase-timeout=60 --suite=columnstore/$1 $2 | tee $CURRENT_DIR/mtr.$1.log 2>&1 + # dump analyses. + systemctl stop mariadb + systemctl start mariadb + ls /core >$CURRENT_DIR/mtr.$1.cores-after + echo "reports or coredumps:" + diff -u $CURRENT_DIR/mtr.$1.cores-before $CURRENT_DIR/mtr.$1.cores-after && echo "no new reports or coredumps" + rm $CURRENT_DIR/mtr.$1.cores-before $CURRENT_DIR/mtr.$1.cores-after } if (( $# == 2 )); then @@ -29,8 +40,9 @@ if (( $# == 2 )); then exit 1 fi -run_suite setup +run_suite basic run_suite bugfixes +run_suite setup run_suite devregression run_suite autopilot run_suite extended diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index 82bef3b7c..f901be3fc 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -422,6 +422,7 @@ BRMManagedShmImplRBTree::BRMManagedShmImplRBTree(unsigned key, off_t size, bool BRMManagedShmImplRBTree::~BRMManagedShmImplRBTree() { + delete fShmSegment; } void BRMManagedShmImplRBTree::setReadOnly() diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index 8911d0300..2335c99c2 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -70,7 +70,7 @@ const std::string ERR_LOG_SUFFIX = ".err"; // Job err log file suffix // extern WriteEngine::BRMWrapper* brmWrapperPtr; namespace WriteEngine { -/* static */ boost::ptr_vector BulkLoad::fTableInfo; +/* static */ std::vector> BulkLoad::fTableInfo; /* static */ boost::mutex* BulkLoad::fDDLMutex = 0; /* static */ const std::string BulkLoad::DIR_BULK_JOB("job"); @@ -519,7 +519,7 @@ void BulkLoad::spawnWorkers() // NO_ERROR if success // other if fail //------------------------------------------------------------------------------ -int BulkLoad::preProcess(Job& job, int tableNo, TableInfo* tableInfo) +int BulkLoad::preProcess(Job& job, int tableNo, std::shared_ptr& tableInfo) { int rc = NO_ERROR, minWidth = 9999; // give a big number HWM minHWM = 999999; // rp 9/25/07 Bug 473 @@ -701,7 +701,7 @@ int BulkLoad::preProcess(Job& job, int tableNo, TableInfo* tableInfo) << "Table-" << job.jobTableList[tableNo].tblName << "..."; fLog.logMsg(oss11.str(), MSGLVL_INFO2); - rc = saveBulkRollbackMetaData(job, tableInfo, segFileInfo, dbRootHWMInfoColVec); + rc = saveBulkRollbackMetaData(job, tableInfo.get(), segFileInfo, dbRootHWMInfoColVec); if (rc != NO_ERROR) { @@ -733,10 +733,10 @@ int BulkLoad::preProcess(Job& job, int tableNo, TableInfo* tableInfo) if (job.jobTableList[tableNo].colList[i].compressionType) info = new ColumnInfoCompressed(&fLog, i, job.jobTableList[tableNo].colList[i], pDBRootExtentTracker, - tableInfo); + tableInfo.get()); // tableInfo->rbMetaWriter()); else - info = new ColumnInfo(&fLog, i, job.jobTableList[tableNo].colList[i], pDBRootExtentTracker, tableInfo); + info = new ColumnInfo(&fLog, i, job.jobTableList[tableNo].colList[i], pDBRootExtentTracker, tableInfo.get()); if (pwd) info->setUIDGID(pwd->pw_uid, pwd->pw_gid); @@ -840,7 +840,7 @@ int BulkLoad::preProcess(Job& job, int tableNo, TableInfo* tableInfo) if (rc) return rc; - fTableInfo.push_back(tableInfo); + fTableInfo.push_back(std::shared_ptr(tableInfo)); return NO_ERROR; } @@ -1039,19 +1039,19 @@ int BulkLoad::processJob() //-------------------------------------------------------------------------- // Validate the existence of the import data files //-------------------------------------------------------------------------- - std::vector tables; + std::vector> tables; for (i = 0; i < curJob.jobTableList.size(); i++) { - TableInfo* tableInfo = new TableInfo(&fLog, fTxnID, fProcessName, curJob.jobTableList[i].mapOid, - curJob.jobTableList[i].tblName, fKeepRbMetaFiles); + std::shared_ptr tableInfo(new TableInfo(&fLog, fTxnID, fProcessName, curJob.jobTableList[i].mapOid, + curJob.jobTableList[i].tblName, fKeepRbMetaFiles)); if ((fBulkMode == BULK_MODE_REMOTE_SINGLE_SRC) || (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) tableInfo->setBulkLoadMode(fBulkMode, fBRMRptFileName); tableInfo->setErrorDir(string(getErrorDir())); tableInfo->setTruncationAsError(getTruncationAsError()); - rc = manageImportDataFileList(curJob, i, tableInfo); + rc = manageImportDataFileList(curJob, i, tableInfo.get()); if (rc != NO_ERROR) { @@ -1515,7 +1515,7 @@ int BulkLoad::rollbackLockedTables() for (unsigned i = 0; i < fTableInfo.size(); i++) { - if (fTableInfo[i].isTableLocked()) + if (fTableInfo[i]->isTableLocked()) { lockedTableFound = true; break; @@ -1529,10 +1529,10 @@ int BulkLoad::rollbackLockedTables() // Report the tables that were successfully loaded for (unsigned i = 0; i < fTableInfo.size(); i++) { - if (!fTableInfo[i].isTableLocked()) + if (!fTableInfo[i]->isTableLocked()) { ostringstream oss; - oss << "Table " << fTableInfo[i].getTableName() << " was successfully loaded. "; + oss << "Table " << fTableInfo[i]->getTableName() << " was successfully loaded. "; fLog.logMsg(oss.str(), MSGLVL_INFO1); } } @@ -1540,24 +1540,24 @@ int BulkLoad::rollbackLockedTables() // Report the tables that were not successfully loaded for (unsigned i = 0; i < fTableInfo.size(); i++) { - if (fTableInfo[i].isTableLocked()) + if (fTableInfo[i]->isTableLocked()) { - if (fTableInfo[i].hasProcessingBegun()) + if (fTableInfo[i]->hasProcessingBegun()) { ostringstream oss; - oss << "Table " << fTableInfo[i].getTableName() << " (OID-" << fTableInfo[i].getTableOID() << ")" + oss << "Table " << fTableInfo[i]->getTableName() << " (OID-" << fTableInfo[i]->getTableOID() << ")" << " was not successfully loaded. Rolling back."; fLog.logMsg(oss.str(), MSGLVL_INFO1); } else { ostringstream oss; - oss << "Table " << fTableInfo[i].getTableName() << " (OID-" << fTableInfo[i].getTableOID() << ")" + oss << "Table " << fTableInfo[i]->getTableName() << " (OID-" << fTableInfo[i]->getTableOID() << ")" << " did not start loading. No rollback necessary."; fLog.logMsg(oss.str(), MSGLVL_INFO1); } - rc = rollbackLockedTable(fTableInfo[i]); + rc = rollbackLockedTable(*fTableInfo[i]); if (rc != NO_ERROR) { @@ -1623,9 +1623,9 @@ bool BulkLoad::addErrorMsg2BrmUpdater(const std::string& tablename, const ostrin for (int tableId = 0; tableId < size; tableId++) { - if (fTableInfo[tableId].getTableName() == tablename) + if (fTableInfo[tableId]->getTableName() == tablename) { - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fTableInfo[tableId]->fBRMReporter.addToErrMsgEntry(oss.str()); return true; } } diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index 39841b23a..926ff6c93 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -77,7 +77,7 @@ class BulkLoad : public FileOp /** * @brief Pre process jobs to validate and assign values to the job structure */ - int preProcess(Job& job, int tableNo, TableInfo* tableInfo); + int preProcess(Job& job, int tableNo, std::shared_ptr& tableInfo); /** * @brief Print job information @@ -194,7 +194,7 @@ class BulkLoad : public FileOp std::string fAlternateImportDir; // Alternate bulk import directory std::string fErrorDir; // Opt. where error records record std::string fProcessName; // Application process name - static boost::ptr_vector fTableInfo; // Vector of Table information + static std::vector> fTableInfo; // Vector of Table information int fNoOfParseThreads; // Number of parse threads int fNoOfReadThreads; // Number of read threads boost::thread_group fReadThreads; // Read thread group diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 2b245181b..16739a686 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -1670,7 +1670,7 @@ int BulkLoadBuffer::parseColParquet(ColumnInfo& columnInfo) // not aux column if (isNonAuxColumn) { - columnData = fParquetBatch->column(columnId); + columnData = fParquetBatchParser->column(columnId); } else // aux column { @@ -1789,6 +1789,13 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un int width = column.width; + int bufIndex = 1; + + if (columnData->data()->buffers.size() < 2) + { + bufIndex = 0; + } + //-------------------------------------------------------------------------- // Parse based on column data type //-------------------------------------------------------------------------- @@ -1799,7 +1806,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un //---------------------------------------------------------------------- case WriteEngine::WR_FLOAT: { - const float* dataPtr = columnData->data()->GetValues(1); + const float* dataPtr = columnData->data()->GetValues(bufIndex); for (uint32_t i = 0; i < fTotalReadRowsParser; i++) { @@ -1855,7 +1862,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un //---------------------------------------------------------------------- case WriteEngine::WR_DOUBLE: { - const double* dataPtr = columnData->data()->GetValues(1); + const double* dataPtr = columnData->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2005,7 +2012,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un case WriteEngine::WR_SHORT: { long long origVal; - const short* dataPtr = columnData->data()->GetValues(1); + const short* dataPtr = columnData->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2085,7 +2092,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un case WriteEngine::WR_USHORT: { int64_t origVal = 0; - const uint16_t* dataPtr = columnData->data()->GetValues(1); + const uint16_t* dataPtr = columnData->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2161,7 +2168,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un // if use int8_t here, it will take 8 bool value of parquet array std::shared_ptr boolArray = std::static_pointer_cast(columnData); - const int8_t* dataPtr = columnData->data()->GetValues(1); + const int8_t* dataPtr = columnData->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2250,7 +2257,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un // special handling for aux column to fix segmentation error if (columnData->type_id() != arrow::Type::type::NA) { - const uint8_t* dataPtr = columnData->data()->GetValues(1); + const uint8_t* dataPtr = columnData->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2382,7 +2389,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un if (column.dataType != CalpontSystemCatalog::DATETIME && column.dataType != CalpontSystemCatalog::TIMESTAMP && column.dataType != CalpontSystemCatalog::TIME) { - const long long* dataPtr = columnData->data()->GetValues(1); + const long long* dataPtr = columnData->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2740,7 +2747,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un std::static_pointer_cast(columnData); std::shared_ptr fType = std::static_pointer_cast(decimalArray->type()); - const int128_t* dataPtr = decimalArray->data()->GetValues(1); + const int128_t* dataPtr = decimalArray->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2803,7 +2810,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un //---------------------------------------------------------------------- case WriteEngine::WR_ULONGLONG: { - const uint64_t* dataPtr = columnData->data()->GetValues(1); + const uint64_t* dataPtr = columnData->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2869,7 +2876,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un case WriteEngine::WR_UINT: { int64_t origVal; - const uint32_t* dataPtr = columnData->data()->GetValues(1); + const uint32_t* dataPtr = columnData->data()->GetValues(bufIndex); for (unsigned int i = 0; i < fTotalReadRowsParser; i++) { @@ -2947,7 +2954,7 @@ void BulkLoadBuffer::convertParquet(std::shared_ptr columnData, un { if (column.dataType != CalpontSystemCatalog::DATE) { - const int* dataPtr = columnData->data()->GetValues(1); + const int* dataPtr = columnData->data()->GetValues(bufIndex); long long origVal; for (unsigned int i = 0; i < fTotalReadRowsParser; i++) @@ -3724,6 +3731,7 @@ int BulkLoadBuffer::fillFromFileParquet(RID& totalReadRows, RID& correctTotalRow try { + fParquetBatch.reset(); PARQUET_THROW_NOT_OK(fParquetReader->ReadNext(&fParquetBatch)); fStartRow = correctTotalRows; fStartRowForLogging = totalReadRows; diff --git a/writeengine/bulk/we_extentstripealloc.cpp b/writeengine/bulk/we_extentstripealloc.cpp index 8482cd834..98463e1b3 100644 --- a/writeengine/bulk/we_extentstripealloc.cpp +++ b/writeengine/bulk/we_extentstripealloc.cpp @@ -140,7 +140,7 @@ int ExtentStripeAlloc::allocateExtent(OID oid, uint16_t dbRoot, startLbid = extentEntryIter->second.fStartLbid; allocSize = extentEntryIter->second.fAllocSize; hwm = extentEntryIter->second.fHwm; - errMsg = extentEntryIter->second.fStatusMsg; + errMsg = *extentEntryIter->second.fStatusMsg; retStatus = extentEntryIter->second.fStatus; fMap.erase(extentEntryIter); @@ -274,7 +274,7 @@ void ExtentStripeAlloc::print() << "; seg: " << iter->second.fSegNum << "; lbid: " << iter->second.fStartLbid << "; size: " << iter->second.fAllocSize << "; hwm: " << iter->second.fHwm << "; stripe: " << iter->second.fStripeKey << "; stat: " << iter->second.fStatus - << "; msg: " << iter->second.fStatusMsg; + << "; msg: " << *iter->second.fStatusMsg; } } else diff --git a/writeengine/bulk/we_extentstripealloc.h b/writeengine/bulk/we_extentstripealloc.h index 7d30d4dbe..acdee123e 100644 --- a/writeengine/bulk/we_extentstripealloc.h +++ b/writeengine/bulk/we_extentstripealloc.h @@ -46,21 +46,6 @@ class Log; class AllocExtEntry { public: - // Default constructor - AllocExtEntry() - : fOid(0) - , fColWidth(0) - , fDbRoot(0) - , fPartNum(0) - , fSegNum(0) - , fStartLbid(0) - , fAllocSize(0) - , fHwm(0) - , fStatus(NO_ERROR) - , fStripeKey(0) - { - } - // Used to create entry for an existing extent we are going to add data to. AllocExtEntry(OID& oid, int colWidth, uint16_t dbRoot, uint32_t partNum, uint16_t segNum, BRM::LBID_t startLbid, int allocSize, HWM hwm, int status, const std::string& statusMsg, @@ -74,22 +59,22 @@ class AllocExtEntry , fAllocSize(allocSize) , fHwm(hwm) , fStatus(status) - , fStatusMsg(statusMsg) + , fStatusMsg(new std::string(statusMsg)) , fStripeKey(stripeKey) { } - OID fOid; // column OID - int fColWidth; // colum width (in bytes) - uint16_t fDbRoot; // DBRoot of allocated extent - uint32_t fPartNum; // Partition number of allocated extent - uint16_t fSegNum; // Segment number of allocated extent - BRM::LBID_t fStartLbid; // Starting LBID of allocated extent - int fAllocSize; // Number of allocated LBIDS - HWM fHwm; // Starting fbo or hwm of allocated extent - int fStatus; // Status of extent allocation - std::string fStatusMsg; // Status msg of extent allocation - unsigned int fStripeKey; // "Stripe" identifier for this extent + OID fOid = 0; // column OID + int fColWidth = 0; // colum width (in bytes) + uint16_t fDbRoot = 0; // DBRoot of allocated extent + uint32_t fPartNum = 0; // Partition number of allocated extent + uint16_t fSegNum = 0; // Segment number of allocated extent + BRM::LBID_t fStartLbid = 0; // Starting LBID of allocated extent + int fAllocSize = 0; // Number of allocated LBIDS + HWM fHwm = 0; // Starting fbo or hwm of allocated extent + int fStatus = NO_ERROR; // Status of extent allocation + std::shared_ptr fStatusMsg{new std::string()}; // Status msg of extent allocation + unsigned int fStripeKey = 0; // "Stripe" identifier for this extent }; //------------------------------------------------------------------------------ diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 7b4c2d0ee..5c6c8eb3b 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -182,7 +182,7 @@ TableInfo::TableInfo(Log* logger, const BRM::TxnID txnID, const string& processN TableInfo::~TableInfo() { fBRMReporter.sendErrMsgToFile(fBRMRptFileName); - freeProcessingBuffers(); + //freeProcessingBuffers(); } //------------------------------------------------------------------------------ diff --git a/writeengine/bulk/we_workers.cpp b/writeengine/bulk/we_workers.cpp index dcd4f1d54..26870e88d 100644 --- a/writeengine/bulk/we_workers.cpp +++ b/writeengine/bulk/we_workers.cpp @@ -95,16 +95,16 @@ void BulkLoad::read(int id) #ifdef PROFILE Stats::stopReadEvent(WE_STATS_WAIT_TO_SELECT_TBL); #endif - int rc = fTableInfo[tableId].readTableData(); + int rc = fTableInfo[tableId]->readTableData(); if (rc != NO_ERROR) { // Error occurred while reading the data, break out of loop. BulkStatus::setJobStatus(EXIT_FAILURE); ostringstream oss; - oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId]->getTableName() << ". Terminating this job."; - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fTableInfo[tableId]->fBRMReporter.addToErrMsgEntry(oss.str()); fLog.logMsg(oss.str(), rc, MSGLVL_CRITICAL); break; } @@ -117,7 +117,7 @@ void BulkLoad::read(int id) if (tableId != -1) oss << "Bulkload Read (thread " << id << ") Stopped reading Table " - << fTableInfo[tableId].getTableName() << ". " << ex.what(); + << fTableInfo[tableId]->getTableName() << ". " << ex.what(); else oss << "Bulkload Read (thread " << id << ") Stopped reading Tables. " << ex.what(); @@ -129,14 +129,14 @@ void BulkLoad::read(int id) ostringstream oss; if (tableId != -1) - oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId]->getTableName() << ". " << ex.what() << ". Terminating this job."; else oss << "Bulkload Read (thread " << id << ") Failed for Table. " << ex.what() << ". Terminating this job."; if (tableId != -1) - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fTableInfo[tableId]->fBRMReporter.addToErrMsgEntry(oss.str()); fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL); } @@ -146,13 +146,13 @@ void BulkLoad::read(int id) ostringstream oss; if (tableId != -1) - oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + oss << "Bulkload Read (thread " << id << ") Failed for Table " << fTableInfo[tableId]->getTableName() << ". Terminating this job."; else oss << "Bulkload Read (thread " << id << ") Failed for Table. Terminating this job."; if (tableId != -1) - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fTableInfo[tableId]->fBRMReporter.addToErrMsgEntry(oss.str()); fLog.logMsg(oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL); } @@ -170,7 +170,7 @@ int BulkLoad::lockTableForRead(int id) for (unsigned i = 0; i < fTableInfo.size(); ++i) { - if (fTableInfo[i].lockForRead(id)) + if (fTableInfo[i]->lockForRead(id)) return i; } @@ -292,16 +292,16 @@ void BulkLoad::parse(int id) // Have obtained the table and column for parsing. // Start parsing the column data. double processingTime; - int rc = fTableInfo[tableId].parseColumn(columnId, myParseBuffer, processingTime); + int rc = fTableInfo[tableId]->parseColumn(columnId, myParseBuffer, processingTime); if (rc != NO_ERROR) { // Error occurred while parsing the data, break out of loop. BulkStatus::setJobStatus(EXIT_FAILURE); ostringstream oss; - oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId]->getTableName() << " during parsing. Terminating this job."; - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fTableInfo[tableId]->fBRMReporter.addToErrMsgEntry(oss.str()); fLog.logMsg(oss.str(), rc, MSGLVL_CRITICAL); setParseErrorOnTable(tableId, true); @@ -310,7 +310,7 @@ void BulkLoad::parse(int id) // Parsing is complete. Acquire the mutex and increment // the parsingComplete value for the buffer - if (fTableInfo[tableId].getStatusTI() != WriteEngine::ERR) + if (fTableInfo[tableId]->getStatusTI() != WriteEngine::ERR) { #ifdef PROFILE Stats::startParseEvent(WE_STATS_WAIT_TO_COMPLETE_PARSE); @@ -320,15 +320,15 @@ void BulkLoad::parse(int id) Stats::stopParseEvent(WE_STATS_WAIT_TO_COMPLETE_PARSE); Stats::startParseEvent(WE_STATS_COMPLETING_PARSE); #endif - rc = fTableInfo[tableId].setParseComplete(columnId, myParseBuffer, processingTime); + rc = fTableInfo[tableId]->setParseComplete(columnId, myParseBuffer, processingTime); if (rc != NO_ERROR) { BulkStatus::setJobStatus(EXIT_FAILURE); ostringstream oss; oss << "Bulkload Parse (thread " << id << ") Failed for Table " - << fTableInfo[tableId].getTableName() << " during parse completion. Terminating this job."; - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + << fTableInfo[tableId]->getTableName() << " during parse completion. Terminating this job."; + fTableInfo[tableId]->fBRMReporter.addToErrMsgEntry(oss.str()); fLog.logMsg(oss.str(), rc, MSGLVL_CRITICAL); setParseErrorOnTable(tableId, false); @@ -349,7 +349,7 @@ void BulkLoad::parse(int id) if (tableId != -1) { oss << "Bulkload Parse (thread " << id << ") Stopped parsing Table " - << fTableInfo[tableId].getTableName() << ". " << ex.what(); + << fTableInfo[tableId]->getTableName() << ". " << ex.what(); setParseErrorOnTable(tableId, true); } @@ -367,11 +367,11 @@ void BulkLoad::parse(int id) if (tableId != -1) { - oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId]->getTableName() << ". " << ex.what() << ". Terminating this job."; setParseErrorOnTable(tableId, true); - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fTableInfo[tableId]->fBRMReporter.addToErrMsgEntry(oss.str()); } else { @@ -388,11 +388,11 @@ void BulkLoad::parse(int id) if (tableId != -1) { - oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId].getTableName() + oss << "Bulkload Parse (thread " << id << ") Failed for Table " << fTableInfo[tableId]->getTableName() << ". Terminating this job."; setParseErrorOnTable(tableId, true); - fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str()); + fTableInfo[tableId]->fBRMReporter.addToErrMsgEntry(oss.str()); } else { @@ -421,10 +421,10 @@ bool BulkLoad::lockColumnForParse(int thrdId, int& tableId, int& columnId, int& for (unsigned i = 0; i < fTableInfo.size(); ++i) { - if (fTableInfo[i].getStatusTI() == WriteEngine::PARSE_COMPLETE) + if (fTableInfo[i]->getStatusTI() == WriteEngine::PARSE_COMPLETE) continue; - int currentParseBuffer = fTableInfo[i].getCurrentParseBuffer(); + int currentParseBuffer = fTableInfo[i]->getCurrentParseBuffer(); myParseBuffer = currentParseBuffer; do @@ -434,7 +434,7 @@ bool BulkLoad::lockColumnForParse(int thrdId, int& tableId, int& columnId, int& { ostringstream oss; std::string bufStatusStr; - Status stat = fTableInfo[i].getStatusTI(); + Status stat = fTableInfo[i]->getStatusTI(); ColumnInfo::convertStatusToString(stat, bufStatusStr); oss << " - " << pthread_self() << ":fTableInfo[" << i << "]" << bufStatusStr << " (" << stat << ")"; @@ -452,13 +452,13 @@ bool BulkLoad::lockColumnForParse(int thrdId, int& tableId, int& columnId, int& // @bug2099- // get a buffer and column to parse if available. - if ((columnId = fTableInfo[i].getColumnForParse(thrdId, myParseBuffer, report)) != -1) + if ((columnId = fTableInfo[i]->getColumnForParse(thrdId, myParseBuffer, report)) != -1) { tableId = i; return true; } - myParseBuffer = (myParseBuffer + 1) % fTableInfo[i].getNumberOfBuffers(); + myParseBuffer = (myParseBuffer + 1) % fTableInfo[i]->getNumberOfBuffers(); } while (myParseBuffer != currentParseBuffer); } @@ -479,10 +479,10 @@ bool BulkLoad::allTablesDone(Status status) { for (unsigned i = 0; i < fTableInfo.size(); ++i) { - if (fTableInfo[i].getStatusTI() == WriteEngine::ERR) + if (fTableInfo[i]->getStatusTI() == WriteEngine::ERR) return true; - if (fTableInfo[i].getStatusTI() != status) + if (fTableInfo[i]->getStatusTI() != status) return false; } @@ -499,11 +499,11 @@ void BulkLoad::setParseErrorOnTable(int tableId, bool lockParseMutex) if (lockParseMutex) { boost::mutex::scoped_lock lock(fParseMutex); - fTableInfo[tableId].setParseError(); + fTableInfo[tableId]->setParseError(); } else { - fTableInfo[tableId].setParseError(); + fTableInfo[tableId]->setParseError(); } } diff --git a/writeengine/shared/we_brm.cpp b/writeengine/shared/we_brm.cpp index e065630b5..ffc783c78 100644 --- a/writeengine/shared/we_brm.cpp +++ b/writeengine/shared/we_brm.cpp @@ -57,6 +57,7 @@ using namespace execplan; namespace WriteEngine { BRMWrapper* volatile BRMWrapper::m_instance = NULL; +std::atomic BRMWrapper::finishReported(false); boost::thread_specific_ptr BRMWrapper::m_ThreadDataPtr; boost::mutex BRMWrapper::m_instanceCreateMutex; @@ -750,6 +751,10 @@ uint8_t BRMWrapper::newCpimportJob(uint32_t &jobId) void BRMWrapper::finishCpimportJob(uint32_t jobId) { + if (finishReported.exchange(true)) // get old and set to true; if old is true, do nothing. + { + return; + } blockRsltnMgrPtr->finishCpimportJob(jobId); } diff --git a/writeengine/shared/we_brm.h b/writeengine/shared/we_brm.h index d8bc403ae..e091e38ae 100644 --- a/writeengine/shared/we_brm.h +++ b/writeengine/shared/we_brm.h @@ -23,6 +23,7 @@ #pragma once +#include #include #include #include @@ -474,6 +475,8 @@ class BRMWrapper : public WEObj static IDBDataFile* m_curVBFile; BRM::DBRM* blockRsltnMgrPtr; + + EXPORT static std::atomic finishReported; }; //------------------------------------------------------------------------------ From 61200f532f84f71be13faffdbc427ebfcb827b8d Mon Sep 17 00:00:00 2001 From: Alexey Antipovsky Date: Fri, 6 Dec 2024 16:10:24 +0100 Subject: [PATCH 27/65] fix(aggregation): remove double returnMemory() (#3366) --- dbcon/joblist/tupleaggregatestep.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index ae31f0fa6..470f96b2a 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -509,7 +509,6 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID) fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit); if (cancelled()) { - fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit); finishedSecondPhase = true; fEndOfResult = true; } From eb6b370287189ac57f3995b1aecc83afe0e6e2db Mon Sep 17 00:00:00 2001 From: Sergey Zefirov <72864488+mariadb-SergeyZefirov@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:22:28 +0300 Subject: [PATCH 28/65] fix(MTR tests): Fixes autopilot MTR tests (#3368) Fixes in UBSAN related commit introduced more server-compatible behavior that differ fom our old behavior. Thus, old tests broke and their results had to be changed. This is what this patch does. --- .../mcs4090_function_CNX_EXTRACT_DM_Archived.result | 4 ++-- ...9_function_CNX_TIMESTAMPDIFF_DM_KnownIssue.result | 12 ++++++------ ...mcs4315_function_CNXPP_EXTRACT_DM_Archived.result | 4 ++-- ...function_CNXPP_TIMESTAMPDIFF_DM_KnownIssue.result | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/mysql-test/columnstore/autopilot/r/mcs4090_function_CNX_EXTRACT_DM_Archived.result b/mysql-test/columnstore/autopilot/r/mcs4090_function_CNX_EXTRACT_DM_Archived.result index 8b7c767ae..666839e46 100644 --- a/mysql-test/columnstore/autopilot/r/mcs4090_function_CNX_EXTRACT_DM_Archived.result +++ b/mysql-test/columnstore/autopilot/r/mcs4090_function_CNX_EXTRACT_DM_Archived.result @@ -157,7 +157,7 @@ cidx CTIME EXTRACT(MINUTE_SECOND FROM CTIME) 1 13:00:00 0 select cidx, CTIME, EXTRACT(HOUR_MICROSECOND FROM CTIME) from datatypetestm order by cidx; cidx CTIME EXTRACT(HOUR_MICROSECOND FROM CTIME) -1 13:00:00 1150981120 +1 13:00:00 130000000000 select cidx, CTIME, EXTRACT(HOUR_SECOND FROM CTIME) from datatypetestm order by cidx; cidx CTIME EXTRACT(HOUR_SECOND FROM CTIME) 1 13:00:00 130000 @@ -166,7 +166,7 @@ cidx CTIME EXTRACT(HOUR_MINUTE FROM CTIME) 1 13:00:00 1300 select cidx, CTIME, EXTRACT(DAY_MICROSECOND FROM CTIME) from datatypetestm order by cidx; cidx CTIME EXTRACT(DAY_MICROSECOND FROM CTIME) -1 13:00:00 1150981120 +1 13:00:00 130000000000 select cidx, CTIME, EXTRACT(DAY_SECOND FROM CTIME) from datatypetestm order by cidx; cidx CTIME EXTRACT(DAY_SECOND FROM CTIME) 1 13:00:00 130000 diff --git a/mysql-test/columnstore/autopilot/r/mcs4219_function_CNX_TIMESTAMPDIFF_DM_KnownIssue.result b/mysql-test/columnstore/autopilot/r/mcs4219_function_CNX_TIMESTAMPDIFF_DM_KnownIssue.result index a488f7df2..c6db8407d 100644 --- a/mysql-test/columnstore/autopilot/r/mcs4219_function_CNX_TIMESTAMPDIFF_DM_KnownIssue.result +++ b/mysql-test/columnstore/autopilot/r/mcs4219_function_CNX_TIMESTAMPDIFF_DM_KnownIssue.result @@ -55,22 +55,22 @@ cidx CDATETIME TIMESTAMPDIFF(YEAR,CDATETIME, '2011-05-06 11:09:36') 1 1997-01-01 00:00:00 14 select cidx, CTIME, TIMESTAMPDIFF(MICROSECOND,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(MICROSECOND,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -2086818369232967680 +1 13:00:00 63471852576000000 select cidx, CTIME, TIMESTAMPDIFF(SECOND,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(SECOND,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -2086818369232 +1 13:00:00 63471852576 select cidx, CTIME, TIMESTAMPDIFF(MINUTE,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(MINUTE,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -34780306153 +1 13:00:00 1057864209 select cidx, CTIME, TIMESTAMPDIFF(HOUR,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(HOUR,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -579671769 +1 13:00:00 17631070 select cidx, CTIME, TIMESTAMPDIFF(DAY,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(DAY,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -24152990 +1 13:00:00 734627 select cidx, CTIME, TIMESTAMPDIFF(WEEK,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(WEEK,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -3450427 +1 13:00:00 104946 select cidx, CTIME, TIMESTAMPDIFF(MONTH,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(MONTH,CTIME, '2011-05-06 11:09:36') 1 13:00:00 24137 diff --git a/mysql-test/columnstore/autopilot/r/mcs4315_function_CNXPP_EXTRACT_DM_Archived.result b/mysql-test/columnstore/autopilot/r/mcs4315_function_CNXPP_EXTRACT_DM_Archived.result index e328f5e8f..8a0a0d07a 100644 --- a/mysql-test/columnstore/autopilot/r/mcs4315_function_CNXPP_EXTRACT_DM_Archived.result +++ b/mysql-test/columnstore/autopilot/r/mcs4315_function_CNXPP_EXTRACT_DM_Archived.result @@ -157,7 +157,7 @@ cidx CTIME EXTRACT(MINUTE_SECOND FROM CTIME) 1 13:00:00 0 select cidx, CTIME, EXTRACT(HOUR_MICROSECOND FROM CTIME) from datatypetestm order by cidx; cidx CTIME EXTRACT(HOUR_MICROSECOND FROM CTIME) -1 13:00:00 1150981120 +1 13:00:00 130000000000 select cidx, CTIME, EXTRACT(HOUR_SECOND FROM CTIME) from datatypetestm order by cidx; cidx CTIME EXTRACT(HOUR_SECOND FROM CTIME) 1 13:00:00 130000 @@ -166,7 +166,7 @@ cidx CTIME EXTRACT(HOUR_MINUTE FROM CTIME) 1 13:00:00 1300 select cidx, CTIME, EXTRACT(DAY_MICROSECOND FROM CTIME) from datatypetestm order by cidx; cidx CTIME EXTRACT(DAY_MICROSECOND FROM CTIME) -1 13:00:00 1150981120 +1 13:00:00 130000000000 select cidx, CTIME, EXTRACT(DAY_SECOND FROM CTIME) from datatypetestm order by cidx; cidx CTIME EXTRACT(DAY_SECOND FROM CTIME) 1 13:00:00 130000 diff --git a/mysql-test/columnstore/autopilot/r/mcs4444_function_CNXPP_TIMESTAMPDIFF_DM_KnownIssue.result b/mysql-test/columnstore/autopilot/r/mcs4444_function_CNXPP_TIMESTAMPDIFF_DM_KnownIssue.result index efc67c0e1..558eedef1 100644 --- a/mysql-test/columnstore/autopilot/r/mcs4444_function_CNXPP_TIMESTAMPDIFF_DM_KnownIssue.result +++ b/mysql-test/columnstore/autopilot/r/mcs4444_function_CNXPP_TIMESTAMPDIFF_DM_KnownIssue.result @@ -55,22 +55,22 @@ cidx CDATETIME TIMESTAMPDIFF(YEAR,CDATETIME, '2011-05-06 11:09:36') 1 1997-01-01 00:00:00 14 select cidx, CTIME, TIMESTAMPDIFF(MICROSECOND,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(MICROSECOND,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -2086818369232967680 +1 13:00:00 63471852576000000 select cidx, CTIME, TIMESTAMPDIFF(SECOND,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(SECOND,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -2086818369232 +1 13:00:00 63471852576 select cidx, CTIME, TIMESTAMPDIFF(MINUTE,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(MINUTE,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -34780306153 +1 13:00:00 1057864209 select cidx, CTIME, TIMESTAMPDIFF(HOUR,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(HOUR,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -579671769 +1 13:00:00 17631070 select cidx, CTIME, TIMESTAMPDIFF(DAY,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(DAY,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -24152990 +1 13:00:00 734627 select cidx, CTIME, TIMESTAMPDIFF(WEEK,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(WEEK,CTIME, '2011-05-06 11:09:36') -1 13:00:00 -3450427 +1 13:00:00 104946 select cidx, CTIME, TIMESTAMPDIFF(MONTH,CTIME, '2011-05-06 11:09:36') from datatypetestm order by cidx; cidx CTIME TIMESTAMPDIFF(MONTH,CTIME, '2011-05-06 11:09:36') 1 13:00:00 24137 From 87e2bb4cef7830ee19483b2135b859154664d243 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Wed, 11 Dec 2024 00:07:46 +0300 Subject: [PATCH 29/65] feat(fdb): MCOL-5802 Add support for blob insertion into FDB. (#3351) --- build/bootstrap_mcs.sh | 1 + utils/fdb_wrapper_cpp/include/fdbcs.hpp | 78 +++++ utils/fdb_wrapper_cpp/src/fdbcs.cpp | 327 +++++++++++++++++++- utils/fdb_wrapper_cpp/test/test_fdb_api.cpp | 69 +++++ 4 files changed, 474 insertions(+), 1 deletion(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 3821eef46..5bc1e706b 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -215,6 +215,7 @@ clean_old_installation() rm -rf /etc/mysql rm -rf /etc/my.cnf.d/columnstore.cnf rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf + fdbcli --exec "writemode on; clearrange \x00 \xff" } build() diff --git a/utils/fdb_wrapper_cpp/include/fdbcs.hpp b/utils/fdb_wrapper_cpp/include/fdbcs.hpp index 202b5318d..4298dfba9 100644 --- a/utils/fdb_wrapper_cpp/include/fdbcs.hpp +++ b/utils/fdb_wrapper_cpp/include/fdbcs.hpp @@ -21,6 +21,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include // https://apple.github.io/foundationdb/api-c.html // We have to define `FDB_API_VERSION` before include `fdb_c.h` header. @@ -104,5 +110,77 @@ class DataBaseCreator static std::shared_ptr createDataBase(const std::string clusterFilePath); }; +using Block = std::pair; +using Key = std::string; +using Keys = std::vector; +// Maps a key to associated block. +using KeyBlockMap = std::unordered_map; +using TreeLevelNumKeysMap = std::unordered_map; + +class KeyGenerator +{ + public: + virtual ~KeyGenerator() + { + } + virtual Key generateKey() = 0; + virtual uint32_t getKeySize() = 0; +}; + +class BoostUIDKeyGenerator : public KeyGenerator +{ + public: + Key generateKey() override; + uint32_t getKeySize() override; +}; + +// This class represetns a machinery to handle a data `blob`. +class BlobHandler +{ + public: + BlobHandler(std::shared_ptr keyGen, uint32_t blockSizeInBytes = 100000) + : keyGen_(keyGen), blockSizeInBytes_(blockSizeInBytes) + { + // Block size in 100KB shows the best performance. + keySizeInBytes_ = keyGen_->getKeySize(); + assert(keySizeInBytes_); + assert(blockSizeInBytes_); + assert((keySizeInBytes_ + keyBlockIdentifier.size()) <= blockSizeInBytes_); + numKeysInBlock_ = (blockSizeInBytes_ - keyBlockIdentifier.size()) / keySizeInBytes_; + assert(blockSizeInBytes_ > dataBlockIdentifier.size()); + dataBlockSizeInBytes_ = (blockSizeInBytes_ - dataBlockIdentifier.size()); + } + + // Writes the given `blob` with given `key`. + bool writeBlob(std::shared_ptr database, const ByteArray& key, const ByteArray& blob); + // Reads `blob` by the given `key`, on error returns false. + std::pair readBlob(std::shared_ptr database, const ByteArray& key); + // Removes a `blob` by the given `key`, on error returns false. + bool removeBlob(std::shared_ptr database, const ByteArray& key); + + private: + size_t insertData(Block& block, const std::string& blob, const size_t offset); + void insertKey(Block& block, const std::string& value); + std::pair getKeysFromBlock(const Block& block); + Keys generateKeys(const uint32_t num); + bool isDataBlock(const Block& block); + bool commitKeys(std::shared_ptr database, KeyBlockMap& keyBlockMap, const Keys& keys); + bool commitKey(std::shared_ptr database, const Key& key, const ByteArray& value); + bool removeKeys(std::shared_ptr database, const Keys& keys); + TreeLevelNumKeysMap computeNumKeysForEachTreeLevel(const int32_t treeLen, const uint32_t numBlocks); + inline float log(const uint32_t base, const uint32_t value); + + std::shared_ptr keyGen_; + uint32_t blockSizeInBytes_; + uint32_t keySizeInBytes_; + uint32_t numKeysInBlock_; + uint32_t dataBlockSizeInBytes_; + // FIXME: Doc says that 10MB is limit, currently taking in account `key` size and `value` size, but 10MB + // limit returns error on transaction. + const uint32_t maxTnxSize_{8192000}; + const std::string keyBlockIdentifier{"K"}; + const std::string dataBlockIdentifier{"D"}; +}; + bool setAPIVersion(); } // namespace FDBCS diff --git a/utils/fdb_wrapper_cpp/src/fdbcs.cpp b/utils/fdb_wrapper_cpp/src/fdbcs.cpp index f85314da9..a6de89676 100644 --- a/utils/fdb_wrapper_cpp/src/fdbcs.cpp +++ b/utils/fdb_wrapper_cpp/src/fdbcs.cpp @@ -19,10 +19,24 @@ #include #include #include -#include "../include/fdbcs.hpp" +#include +#include +#include +#include +#include +#include "fdbcs.hpp" namespace FDBCS { + +#define RETURN_ON_ERROR(exp) \ + do \ + { \ + auto rc = (exp); \ + if (!rc) \ + return rc; \ + } while (false); + Transaction::Transaction(FDBTransaction* tnx) : tnx_(tnx) { } @@ -111,6 +125,13 @@ bool Transaction::commit() const std::cerr << "fdb_future_block_until_ready error, code: " << (int)err << std::endl; return false; } + err = fdb_future_get_error(future); + if (err) + { + fdb_future_destroy(future); + std::cerr << "fdb_future_get_error(), code: " << (int)err << std::endl; + return false; + } fdb_future_destroy(future); } return true; @@ -206,6 +227,310 @@ std::shared_ptr DataBaseCreator::createDataBase(const std::string c return std::make_shared(database); } +Key BoostUIDKeyGenerator::generateKey() +{ + return boost::lexical_cast(boost::uuids::random_generator()()); +} + +uint32_t BoostUIDKeyGenerator::getKeySize() +{ + return boost::lexical_cast(boost::uuids::random_generator()()).size(); +} + +Keys BlobHandler::generateKeys(const uint32_t num) +{ + Keys keys; + keys.reserve(num); + for (uint32_t i = 0; i < num; ++i) + keys.push_back(keyGen_->generateKey()); + + return keys; +} +// FIXME: Put it to util? +float BlobHandler::log(const uint32_t base, const uint32_t value) +{ + return std::log(value) / std::log(base); +} + +void BlobHandler::insertKey(Block& block, const std::string& value) +{ + if (!block.first) + { + block.second.reserve(blockSizeInBytes_); + block.second.insert(block.second.end(), keyBlockIdentifier.begin(), keyBlockIdentifier.end()); + block.first += keyBlockIdentifier.size(); + } + block.second.insert(block.second.begin() + block.first, value.begin(), value.end()); + block.first += value.size(); +} + +size_t BlobHandler::insertData(Block& block, const std::string& blob, const size_t offset) +{ + const size_t endOfBlock = std::min(offset + dataBlockSizeInBytes_, blob.size()); + auto& dataBlock = block.second; + if (!block.first) + { + dataBlock.reserve(blockSizeInBytes_); + dataBlock.insert(dataBlock.end(), dataBlockIdentifier.begin(), dataBlockIdentifier.end()); + block.first += dataBlockIdentifier.size(); + } + dataBlock.insert(dataBlock.begin() + block.first, blob.begin() + offset, blob.begin() + endOfBlock); + return endOfBlock; +} + +bool BlobHandler::commitKeys(std::shared_ptr dataBase, KeyBlockMap& keyBlockMap, + const Keys& keys) +{ + auto tnx = dataBase->createTransaction(); + if (!tnx) + return false; + + for (const auto& key : keys) + tnx->set(key, keyBlockMap[key].second); + + return tnx->commit(); +} + +bool BlobHandler::commitKey(std::shared_ptr dataBase, const Key& key, + const ByteArray& value) +{ + auto tnx = dataBase->createTransaction(); + tnx->set(key, value); + return tnx->commit(); +} + +TreeLevelNumKeysMap BlobHandler::computeNumKeysForEachTreeLevel(const int32_t treeLen, + const uint32_t numBlocks) +{ + TreeLevelNumKeysMap levelMap; + levelMap[treeLen] = numBlocks; + if (!treeLen) + return levelMap; + + for (int32_t level = treeLen - 1; level >= 0; --level) + { + if (level + 1 == treeLen) + levelMap[level] = levelMap[level + 1]; + else + levelMap[level] = (levelMap[level + 1] + (numKeysInBlock_ - 1)) / numKeysInBlock_; + } + return levelMap; +} + +bool BlobHandler::writeBlob(std::shared_ptr dataBase, const ByteArray& key, + const ByteArray& blob) +{ + const size_t blobSizeInBytes = blob.size(); + if (!blobSizeInBytes) + return commitKey(dataBase, key, ""); + + const uint32_t numDataBlocks = (blobSizeInBytes + (dataBlockSizeInBytes_ - 1)) / dataBlockSizeInBytes_; + const uint32_t treeLen = std::ceil(log(numKeysInBlock_, numDataBlocks)); + Keys currentKeys{key}; + auto numKeyLevelMap = computeNumKeysForEachTreeLevel(treeLen, numDataBlocks); + + KeyBlockMap keyBlockMap; + keyBlockMap[key] = {0, std::string()}; + Keys keysInTnx; + size_t currentTnxSize = 0; + + for (uint32_t currentLevel = 0; currentLevel < treeLen; ++currentLevel) + { + const uint32_t nextLevelKeyNum = numKeyLevelMap[currentLevel]; + auto nextLevelKeys = generateKeys(nextLevelKeyNum); + uint32_t nextKeysIt = 0; + for (uint32_t i = 0, size = currentKeys.size(); i < size && nextKeysIt < nextLevelKeyNum; ++i) + { + const auto& currentKey = currentKeys[i]; + auto& block = keyBlockMap[currentKey]; + for (uint32_t j = 0; j < numKeysInBlock_ && nextKeysIt < nextLevelKeyNum; ++j, ++nextKeysIt) + { + const auto& nextKey = nextLevelKeys[nextKeysIt]; + insertKey(block, nextKey); + keyBlockMap[nextKey] = {0, std::string()}; + } + if (currentTnxSize + (keySizeInBytes_ + block.second.size()) >= maxTnxSize_) + { + RETURN_ON_ERROR(commitKeys(dataBase, keyBlockMap, keysInTnx)); + currentTnxSize = 0; + keysInTnx.clear(); + } + currentTnxSize += block.second.size() + keySizeInBytes_; + keysInTnx.push_back(currentKey); + } + currentKeys = std::move(nextLevelKeys); + } + + size_t offset = 0; + for (uint32_t i = 0; i < numDataBlocks; ++i) + { + const auto& currentKey = currentKeys[i]; + auto& block = keyBlockMap[currentKey]; + offset = insertData(block, blob, offset); + if (currentTnxSize + (keySizeInBytes_ + block.second.size()) >= maxTnxSize_) + { + RETURN_ON_ERROR(commitKeys(dataBase, keyBlockMap, keysInTnx)); + currentTnxSize = 0; + keysInTnx.clear(); + } + keysInTnx.push_back(currentKey); + currentTnxSize += block.second.size() + keySizeInBytes_; + } + + if (currentTnxSize) + RETURN_ON_ERROR(commitKeys(dataBase, keyBlockMap, keysInTnx)); + + return true; +} + +std::pair BlobHandler::getKeysFromBlock(const Block& block) +{ + Keys keys; + const auto& blockData = block.second; + if (blockData.size() > blockSizeInBytes_) + return {false, {""}}; + + uint32_t offset = 1; + for (uint32_t i = 0; i < numKeysInBlock_ && offset + keySizeInBytes_ <= blockData.size(); ++i) + { + Key key(blockData.begin() + offset, blockData.begin() + offset + keySizeInBytes_); + keys.push_back(std::move(key)); + offset += keySizeInBytes_; + } + + return {true, keys}; +} + +bool BlobHandler::isDataBlock(const Block& block) +{ + return block.second.compare(0, keyBlockIdentifier.size(), keyBlockIdentifier) != 0; +} + +std::pair BlobHandler::readBlob(std::shared_ptr database, + const ByteArray& key) +{ + Keys currentKeys{key}; + bool dataBlockReached = false; + + while (!dataBlockReached) + { + auto tnx = database->createTransaction(); + if (!tnx) + return {false, ""}; + + std::vector blocks; + for (const auto& key : currentKeys) + { + auto p = tnx->get(key); + if (!p.first) + return {false, ""}; + + Block block{0, p.second}; + if (isDataBlock(block)) + { + dataBlockReached = true; + break; + } + blocks.push_back(block); + } + + if (dataBlockReached) + break; + + Keys nextKeys; + for (const auto& block : blocks) + { + auto keysPair = getKeysFromBlock(block); + if (!keysPair.first) + return {false, ""}; + + auto& keys = keysPair.second; + nextKeys.insert(nextKeys.end(), keys.begin(), keys.end()); + } + currentKeys = std::move(nextKeys); + } + + std::string blob; + for (const auto& key : currentKeys) + { + auto tnx = database->createTransaction(); + if (!tnx) + return {false, ""}; + + auto resultPair = tnx->get(key); + if (!resultPair.first) + return {false, ""}; + + auto& dataBlock = resultPair.second; + if (!dataBlock.size()) + return {false, ""}; + + blob.insert(blob.end(), dataBlock.begin() + dataBlockIdentifier.size(), dataBlock.end()); + } + + return {true, blob}; +} + +bool BlobHandler::removeKeys(std::shared_ptr database, const Keys& keys) +{ + for (const auto& key : keys) + { + auto tnx = database->createTransaction(); + if (!tnx) + return false; + tnx->remove(key); + RETURN_ON_ERROR(tnx->commit()); + } + + return true; +} + +bool BlobHandler::removeBlob(std::shared_ptr database, const Key& key) +{ + std::unordered_map treeLevel; + auto tnx = database->createTransaction(); + if (!tnx) + return false; + + uint32_t currentLevel = 0; + treeLevel[0] = {key}; + + while (true) + { + const auto& currentKeys = treeLevel[currentLevel]; + std::vector blocks; + for (const auto& key : currentKeys) + { + auto p = tnx->get(key); + if (!p.first) + return false; + blocks.push_back({0, p.second}); + } + + if (isDataBlock(blocks.front()) || !currentKeys.size()) + break; + + Keys nextKeys; + for (const auto& block : blocks) + { + auto keysPair = getKeysFromBlock(block); + if (!keysPair.first) + return false; + + auto& keys = keysPair.second; + nextKeys.insert(nextKeys.end(), keys.begin(), keys.end()); + } + + ++currentLevel; + treeLevel[currentLevel] = std::move(nextKeys); + } + + for (uint32_t level = 0; level <= currentLevel; ++level) + RETURN_ON_ERROR(removeKeys(database, treeLevel[level])); + + return true; +} + bool setAPIVersion() { auto err = fdb_select_api_version(FDB_API_VERSION); diff --git a/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp b/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp index dc9edb3e7..4f8f1decd 100644 --- a/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp +++ b/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp @@ -20,6 +20,12 @@ using namespace std; using namespace FDBCS; +using std::chrono::duration; +using std::chrono::duration_cast; +using std::chrono::high_resolution_clock; +using std::chrono::milliseconds; + +// #define TEST_PERF 1 template static void assert_internal(const T& value, const std::string& errMessage) @@ -31,6 +37,68 @@ static void assert_internal(const T& value, const std::string& errMessage) } } +static std::string generateBlob(const uint32_t len) +{ + std::string blob; + blob.reserve(len); + for (uint32_t i = 0; i < len; ++i) + { + blob.push_back('a' + (i % 26)); + } + return blob; +} + +static void testBlobHandler(std::shared_ptr db) +{ +#ifdef TEST_PERF + std::vector blobSizes{0, 1, 11, 101, 1001, 10001, 100001, 1000001, 10000001, 100000001}; + std::vector blockSizes{10000, 100000}; +#else + std::vector blobSizes{0, 1, 10001, 100001, 10000001, 100000001}; + std::vector blockSizes{100000}; +#endif + + for (auto blobSize : blobSizes) + { + for (auto blockSize : blockSizes) + { + std::string rootKey = "root"; + auto blobA = generateBlob(blobSize); + std::shared_ptr gen = std::make_shared(); + BlobHandler handler(gen, blockSize); +#ifdef TEST_PERF + auto t1 = high_resolution_clock::now(); +#endif + handler.writeBlob(db, rootKey, blobA); +#ifdef TEST_PERF + auto t2 = high_resolution_clock::now(); + auto ms_int = duration_cast(t2 - t1); + std::cout << "Write blob time: " << ms_int.count() << std::endl; + t1 = high_resolution_clock::now(); +#endif + auto p = handler.readBlob(db, rootKey); +#ifdef TEST_PERF + t2 = high_resolution_clock::now(); + cout << "size readed " << p.second.size() << endl; +#endif + assert_internal(p.second == blobA, "Blobs not equal"); +#ifdef TEST_PERF + ms_int = duration_cast(t2 - t1); + std::cout << "Read blob time: " << ms_int.count() << std::endl; + t1 = high_resolution_clock::now(); +#endif + assert_internal(handler.removeBlob(db, rootKey), "Remove blob error"); +#ifdef TEST_PERF + t2 = high_resolution_clock::now(); + ms_int = duration_cast(t2 - t1); + std::cout << "Remove blob time: " << ms_int.count() << std::endl; +#endif + p = handler.readBlob(db, rootKey); + assert_internal(!p.first, "Blob present after remove"); + } + } +} + int main() { std::string path = "/etc/foundationdb/fdb.cluster"; @@ -98,5 +166,6 @@ int main() } } + testBlobHandler(db); return 0; } From 8f8620210b1fefffde185dfced65fb99abc18ed9 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Tue, 17 Dec 2024 13:05:18 +0300 Subject: [PATCH 30/65] feat(SM,FDB): MCOL-5720 Move meta to FDB. (#3369) --- storage-manager/src/IOCoordinator.cpp | 28 +++---- storage-manager/src/MetadataFile.cpp | 106 ++++++++++++++++++++++---- storage-manager/src/MetadataFile.h | 6 +- storage-manager/src/Ownership.cpp | 4 - 4 files changed, 106 insertions(+), 38 deletions(-) diff --git a/storage-manager/src/IOCoordinator.cpp b/storage-manager/src/IOCoordinator.cpp index 347899db4..5a6adaf9a 100644 --- a/storage-manager/src/IOCoordinator.cpp +++ b/storage-manager/src/IOCoordinator.cpp @@ -1021,17 +1021,6 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) int err; char errbuf[80]; - if (!bf::exists(metaFile1)) - { - errno = ENOENT; - return -1; - } - if (bf::exists(metaFile2)) - { - deleteMetaFile(metaFile2); - ++filesDeleted; - } - // since we don't implement mkdir(), assume the caller did that and // create any necessary parent dirs for filename2 try @@ -1046,12 +1035,18 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) return -1; } - vector > newJournalEntries; + vector> newJournalEntries; ScopedReadLock lock(this, filename1); ScopedWriteLock lock2(this, filename2); MetadataFile meta1(metaFile1, MetadataFile::no_create_t(), false); MetadataFile meta2(metaFile2, MetadataFile::no_create_t(), false); vector objects = meta1.metadataRead(0, meta1.getLength()); + + if (!meta1.exists()) + { + errno = ENOENT; + return -1; + } bytesCopied += meta1.getLength(); if (meta2.exists()) @@ -1197,9 +1192,8 @@ int IOCoordinator::mergeJournal(int objFD, int journalFD, uint8_t* buf, off_t of throw runtime_error("IOCoordinator::mergeJournal(int, int, etc) is not implemented yet."); } -std::shared_ptr IOCoordinator::mergeJournal(const char* object, const char* journal, - off_t offset, size_t len, - size_t* _bytesReadOut) const +std::shared_ptr IOCoordinator::mergeJournal(const char* object, const char* journal, off_t offset, + size_t len, size_t* _bytesReadOut) const { int objFD, journalFD; std::shared_ptr ret; @@ -1335,8 +1329,8 @@ out: // MergeJournalInMem is a specialized version of mergeJournal(). This is currently only used by Synchronizer // and mergeJournal(), and only for merging the whole object with the whole journal. -int IOCoordinator::mergeJournalInMem(std::shared_ptr& objData, size_t len, - const char* journalPath, size_t* _bytesReadOut) const +int IOCoordinator::mergeJournalInMem(std::shared_ptr& objData, size_t len, const char* journalPath, + size_t* _bytesReadOut) const { // if the journal is over some size threshold (100MB for now why not), // use the original low-mem-usage version diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index b4fb327f8..8cf964c69 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -19,18 +19,24 @@ * MetadataFile.cpp */ #include "MetadataFile.h" +#include "KVStorageInitializer.h" #include #include +#include +#include +#include +#include +#include #define BOOST_SPIRIT_THREADSAFE #ifndef __clang__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif #include #ifndef __clang__ - #pragma GCC diagnostic pop +#pragma GCC diagnostic pop #endif #include #include @@ -38,6 +44,8 @@ #include #include #include +#include "fdbcs.hpp" +#include "KVPrefixes.hpp" #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) @@ -55,6 +63,12 @@ uint64_t metadataFilesAccessed = 0; namespace storagemanager { + +inline std::string getKeyName(const std::string& fileName) +{ + return KVPrefixes[static_cast(KVPrefixId::SM_META)] + fileName; +} + MetadataFile::MetadataConfig* MetadataFile::MetadataConfig::get() { if (inst) @@ -123,15 +137,21 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename) _exists = true; mFilename = mpConfig->msMetadataPath / (filename.string() + ".meta"); + metaKVName_ = getKeyName(mFilename.string()); boost::unique_lock s(jsonCache.getMutex()); jsontree = jsonCache.get(mFilename); if (!jsontree) { - if (boost::filesystem::exists(mFilename)) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler blobReader(keyGen); + auto rPair = blobReader.readBlob(kvStorage, metaKVName_); + if (rPair.first) { jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); + stringstream stream(rPair.second); + boost::property_tree::read_json(stream, *jsontree); jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; @@ -161,19 +181,24 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, mpLogger = SMLogging::get(); mFilename = filename; - if (appendExt) mFilename = mpConfig->msMetadataPath / (mFilename.string() + ".meta"); + metaKVName_ = getKeyName(mFilename.string()); boost::unique_lock s(jsonCache.getMutex()); jsontree = jsonCache.get(mFilename); if (!jsontree) { - if (boost::filesystem::exists(mFilename)) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler blobReader(keyGen); + auto rPair = blobReader.readBlob(kvStorage, metaKVName_); + if (rPair.first) { _exists = true; jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); + stringstream stream(rPair.second); + boost::property_tree::read_json(stream, *jsontree); jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; @@ -215,16 +240,55 @@ void MetadataFile::printKPIs() cout << "Metadata files accessed = " << metadataFilesAccessed << endl; } -int MetadataFile::stat(struct stat* out) const +// FIXME: This one increases IO load, we create a new file if the `stat` is not cached, +// but I'm not currently sure how to generate a `stat` info without a file. +// Could we make it smarter? Need some research on this area. +int MetadataFile::generateStatStructInfo(struct stat* out) { - int err = ::stat(mFilename.c_str(), out); - if (err) - return err; + try + { + const std::string statFileName = + mpConfig->msMetadataPath.string() + "/" + boost::to_string(boost::uuids::random_generator()()); + std::ofstream statStream(statFileName); + statStream.close(); + + int err = ::stat(statFileName.c_str(), out); + if (err) + return -1; + + statCache.resize(sizeof(struct stat)); + std::memcpy(&statCache[0], out, sizeof(struct stat)); + statCached = true; + std::filesystem::remove(statFileName); + out->st_size = getLength(); + } + catch (const std::exception& ex) + { + SMLogging::get()->log(LOG_CRIT, "Metadatafile::stat() failed with error: %s", ex.what()); + return -1; + } - out->st_size = getLength(); return 0; } +int MetadataFile::stat(struct stat* out) +{ + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto rPair = tnx->get(metaKVName_); + if (rPair.first) + { + if (statCached) + { + std::memcpy(out, (uint8_t*)&statCache[0], sizeof(struct stat)); + out->st_size = getLength(); + return 0; + } + return generateStatStructInfo(out); + } + return -1; +} + size_t MetadataFile::getLength() const { size_t totalSize = 0; @@ -319,10 +383,20 @@ metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& fi // TODO: Error handling...s int MetadataFile::writeMetadata() { - if (!boost::filesystem::exists(mFilename.parent_path())) - boost::filesystem::create_directories(mFilename.parent_path()); + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler blobWriter(keyGen); + stringstream stream; + write_json(stream, *jsontree); + + if (!blobWriter.writeBlob(kvStorage, metaKVName_, stream.str())) + { + SMLogging::get()->log(LOG_CRIT, "Metadatafile: cannot commit tnx set()."); + throw runtime_error("Metadatafile: cannot commit tnx set()."); + } + } - write_json(mFilename.string(), *jsontree); _exists = true; boost::unique_lock s(jsonCache.getMutex()); diff --git a/storage-manager/src/MetadataFile.h b/storage-manager/src/MetadataFile.h index 5c302c618..f8115ae09 100644 --- a/storage-manager/src/MetadataFile.h +++ b/storage-manager/src/MetadataFile.h @@ -63,7 +63,7 @@ class MetadataFile bool exists() const; void printObjects() const; - int stat(struct stat*) const; + int stat(struct stat*); size_t getLength() const; // returns the objects needed to update std::vector metadataRead(off_t offset, size_t length) const; @@ -118,10 +118,14 @@ class MetadataFile int mVersion; int mRevision; boost::filesystem::path mFilename; + std::string metaKVName_; Jsontree_t jsontree; // std::set mObjects; bool _exists; void makeEmptyJsonTree(); + int generateStatStructInfo(struct stat *); + std::vector statCache; + bool statCached{false}; class MetadataCache { diff --git a/storage-manager/src/Ownership.cpp b/storage-manager/src/Ownership.cpp index 0fe670b76..35c8a19b4 100644 --- a/storage-manager/src/Ownership.cpp +++ b/storage-manager/src/Ownership.cpp @@ -242,10 +242,6 @@ void Ownership::_takeOwnership(const bf::path& p) void Ownership::takeOwnership(const bf::path& p) { - // If the prefix doesn't exist, ownership isn't possible yet. - if (!bf::is_directory(metadataPrefix / p)) - return; - boost::unique_lock s(mutex); auto it = ownedPrefixes.find(p); From 3bc8bd8cc645abf2f7e6f1366dc1c09599ca4f7e Mon Sep 17 00:00:00 2001 From: Sergey Zefirov <72864488+mariadb-SergeyZefirov@users.noreply.github.com> Date: Fri, 20 Dec 2024 22:12:32 +0300 Subject: [PATCH 31/65] fix(group by, having): MCOL-5776: GROUP BY/HAVING closer to server's (#3257) This patch introduces an internal aggregate operator SELECT_SOME that is automatically added to columns that are not in GROUP BY. It "computes" some plausible value of the column (actually, last one passed). Along the way it fixes incorrect handling of HAVING being transferred into WHERE, window function handling and a bit of other inconsistencies. --- dbcon/execplan/aggregatecolumn.cpp | 1 + dbcon/execplan/arithmeticcolumn.cpp | 6 +- dbcon/execplan/arithmeticoperator.h | 8 + dbcon/execplan/constantfilter.cpp | 6 +- dbcon/execplan/predicateoperator.cpp | 2 +- dbcon/execplan/simplecolumn.cpp | 11 +- dbcon/execplan/simplecolumn.h | 1 + dbcon/joblist/expressionstep.cpp | 1 + dbcon/joblist/joblistfactory.cpp | 22 -- dbcon/joblist/tuplehavingstep.cpp | 4 + dbcon/mysql/ha_mcs_execplan.cpp | 317 +++++++++++++----- dbcon/mysql/ha_mcs_impl_if.h | 17 +- .../columnstore/basic/r/mcs76_having.result | 3 + .../columnstore/basic/t/mcs76_having.test | 1 + ...-5776-GROUP-BY-HAVING-functions-use.result | 214 ++++++++++++ ...OL-5776-GROUP-BY-HAVING-functions-use.test | 99 ++++++ .../r/mcs7224_regression_MCOL-3503.result | 14 +- storage-manager/CMakeLists.txt | 2 +- tests/scripts/fullmtr.sh | 2 +- utils/rowgroup/rowgroup.h | 1 + 20 files changed, 601 insertions(+), 131 deletions(-) create mode 100644 mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.result create mode 100644 mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index 2546e9f18..8dd85727c 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -46,6 +46,7 @@ using namespace joblist; namespace execplan { + void getAggCols(execplan::ParseTree* n, void* obj) { vector* list = reinterpret_cast*>(obj); diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index 1caf1bb34..58d11db69 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -300,12 +300,12 @@ const string ArithmeticColumn::toString() const if (fAlias.length() > 0) oss << "Alias: " << fAlias << endl; - if (fExpression != 0) - fExpression->walk(walkfn, oss); - oss << "expressionId=" << fExpressionId << endl; oss << "joinInfo=" << fJoinInfo << " returnAll=" << fReturnAll << " sequence#=" << fSequence << endl; oss << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << endl; + if (fExpression != 0) + fExpression->walk(walkfn, oss); + return oss.str(); } diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 6c66e9253..974ac35c9 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -261,6 +261,10 @@ inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, Parse case execplan::CalpontSystemCatalog::SMALLINT: case execplan::CalpontSystemCatalog::TINYINT: fResult.intVal = execute(lop->getIntVal(row, isNull), rop->getIntVal(row, isNull), isNull); + if (isNull) + { + fResult.intVal = joblist::INTNULL; + } break; case execplan::CalpontSystemCatalog::UBIGINT: @@ -282,6 +286,10 @@ inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, Parse case execplan::CalpontSystemCatalog::USMALLINT: case execplan::CalpontSystemCatalog::UTINYINT: fResult.uintVal = execute(lop->getUintVal(row, isNull), rop->getUintVal(row, isNull), isNull); + if (isNull) + { + fResult.uintVal = joblist::UBIGINTNULL; + } break; case execplan::CalpontSystemCatalog::DOUBLE: diff --git a/dbcon/execplan/constantfilter.cpp b/dbcon/execplan/constantfilter.cpp index b66069887..e3bf5d623 100644 --- a/dbcon/execplan/constantfilter.cpp +++ b/dbcon/execplan/constantfilter.cpp @@ -62,16 +62,14 @@ ConstantFilter::ConstantFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColum { SSFP ssfp(new SimpleFilter(op, lhs, rhs)); fFilterList.push_back(ssfp); - SimpleColumn* sc = dynamic_cast(lhs); - fCol.reset(sc->clone()); + fCol.reset(lhs->clone()); } ConstantFilter::ConstantFilter(SimpleFilter* sf) { SSFP ssfp(sf); fFilterList.push_back(ssfp); - const SimpleColumn* sc = dynamic_cast(sf->lhs()); - fCol.reset(sc->clone()); + fCol.reset(sf->lhs()->clone()); } ConstantFilter::ConstantFilter(const ConstantFilter& rhs) : Filter(rhs), fOp(rhs.fOp), fCol(rhs.fCol) diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index fb8d1993a..01bfee8a0 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -412,7 +412,7 @@ bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedCol int64_t val2 = rop->getIntVal(row, isNull); - return numericCompare(val1, val2) && !isNull; + return !isNull && numericCompare(val1, val2); } case execplan::CalpontSystemCatalog::UBIGINT: diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index e3293527a..73118f030 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -208,6 +208,13 @@ SimpleColumn::SimpleColumn(const SimpleColumn& rhs, const uint32_t sessionID) { } +SimpleColumn::SimpleColumn(const ReturnedColumn& rhs, const uint32_t sessionID) + : ReturnedColumn(rhs, sessionID) + , fData(rhs.data()) + , fisColumnStore(true) +{ +} + SimpleColumn::~SimpleColumn() { } @@ -270,7 +277,9 @@ const string SimpleColumn::toString() const << returnAll() << delim << sequence() << delim << cardinality() << delim << joinInfo() << delim << colSource() << delim << (isColumnStore() ? "ColumnStore" : "ForeignEngine") << delim << colPosition() << delim << cs.getCharset().cs_name.str << delim << cs.getCharset().coll_name.str - << delim << endl; + << " inputindex/outputindex: " << fInputIndex << delim << fOutputIndex + << " eid " << fExpressionId + << endl; return output.str(); } diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index e3d5f6550..836c927f5 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -70,6 +70,7 @@ class SimpleColumn : public ReturnedColumn SimpleColumn(const std::string& schema, const std::string& table, const std::string& col, const bool isColumnStore, const uint32_t sessionID = 0, const int lower_case_table_names = 0); SimpleColumn(const SimpleColumn& rhs, const uint32_t sessionID = 0); + SimpleColumn(const ReturnedColumn& rhs, const uint32_t sessionID = 0); /** * Destructor diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 14d217d0b..57052bd8b 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -53,6 +53,7 @@ using namespace rowgroup; #include "expressionstep.h" + namespace joblist { ExpressionStep::ExpressionStep() diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index c6d363f63..8de958d20 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -354,7 +354,6 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) } } - bool aggInHaving = false; const vector& columns = ths->columns(); for (vector::const_iterator i = columns.begin(); i != columns.end(); i++) @@ -365,7 +364,6 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) if (agc) { addAggregateColumn(agc, -1, jobInfo.nonConstCols, jobInfo); - aggInHaving = true; } else { @@ -387,26 +385,6 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) } } - if (aggInHaving == false) - { - // treated the same as where clause if no aggregate column in having. - jobInfo.havingStep.reset(); - - // parse the having expression - ParseTree* filters = csep->having(); - - if (filters != 0) - { - JLF_ExecPlanToJobList::walkTree(filters, jobInfo); - } - - if (!jobInfo.stack.empty()) - { - idbassert(jobInfo.stack.size() == 1); - jobInfo.havingStepVec = jobInfo.stack.top(); - jobInfo.stack.pop(); - } - } } void preProcessFunctionOnAggregation(const vector& scs, const vector& aggs, diff --git a/dbcon/joblist/tuplehavingstep.cpp b/dbcon/joblist/tuplehavingstep.cpp index b9e9d5962..e1089b3ab 100644 --- a/dbcon/joblist/tuplehavingstep.cpp +++ b/dbcon/joblist/tuplehavingstep.cpp @@ -87,7 +87,9 @@ void TupleHavingStep::initialize(const RowGroup& rgIn, const JobInfo& jobInfo) for (uint64_t i = 0; i < fRowGroupIn.getKeys().size(); ++i) if (keyToIndexMap.find(fRowGroupIn.getKeys()[i]) == keyToIndexMap.end()) + { keyToIndexMap.insert(make_pair(fRowGroupIn.getKeys()[i], i)); + } updateInputIndex(keyToIndexMap, jobInfo); @@ -125,9 +127,11 @@ void TupleHavingStep::expressionFilter(const ParseTree* filter, JobInfo& jobInfo ExpressionStep::expressionFilter(filter, jobInfo); // extract simple columns from parse tree +#if 01 vector acv; fExpressionFilter->walk(getAggCols, &acv); fColumns.insert(fColumns.end(), acv.begin(), acv.end()); +#endif } void TupleHavingStep::run() diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 85a2b2968..6ec3990ce 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -96,6 +96,7 @@ const uint64_t SUB_BIT = 0x02; const uint64_t AF_BIT = 0x04; const uint64_t CORRELATED = 0x08; + // In certain cases, gp_walk is called recursively. When done so, // we need to bookmark the rcWorkStack for those cases where a constant // expression such as 1=1 is used in an if statement or function call. @@ -161,6 +162,51 @@ void calculateNotNullTables(const std::vector& condList, table_map& not_n } } +bool itemDisablesWrapping(Item* item, gp_walk_info& gwi); + +void pushReturnedCol(gp_walk_info& gwi, Item* from, SRCP rc) +{ + uint32_t i; + for ( i = 0; i < gwi.processed.size(); i++) + { + Item* ith = gwi.processed[i].first; + + bool same = ith->eq(from, false); + + if (same && ith->type() == Item::FUNC_ITEM) + { + // an exception for cast(column as decimal(X,Y)) - they are equal (in the eq() call sense) + // even if Xs and Ys are different. + string funcName = ((Item_func*)ith)->func_name(); + + if (funcName == "decimal_typecast") + { + Item_decimal_typecast* ithdtc = (Item_decimal_typecast*)ith; + Item_decimal_typecast* fromdtc = (Item_decimal_typecast*)from; + same = ithdtc->decimals == fromdtc->decimals && ithdtc->max_length == fromdtc->max_length; + } + } + if (same) + { + break; + } + } + bool needChange = true; + if (dynamic_cast(rc.get()) == nullptr && gwi.select_lex && !itemDisablesWrapping(from, gwi)) + { + needChange = false; + } + if (needChange && i < gwi.processed.size()) + { + rc->expressionId(gwi.processed[i].second); + } + else + { + gwi.processed.push_back(std::make_pair(from, rc->expressionId())); + } + gwi.returnedCols.push_back(rc); +} + // Recursively iterate through the join_list and store all non-null // TABLE_LIST::on_expr items to a hash map keyed by the TABLE_LIST ptr. // This is then used by convertOuterJoinToInnerJoin(). @@ -605,7 +651,6 @@ ReturnedColumn* buildAggFrmTempField(Item* item, gp_walk_info& gwi) Item_field* ifip = NULL; Item_ref* irip; Item_func_or_sum* isfp; - switch (item->type()) { case Item::FIELD_ITEM: ifip = static_cast(item); break; @@ -2069,15 +2114,18 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) } sop.reset(new PredicateOperator(eqop)); - sop->setOpType(gwip->scsp->resultType(), rhs->resultType()); + SRCP scsp = gwip->scsp; + idbassert(scsp.get() != nullptr); + //sop->setOpType(gwip->scsp->resultType(), rhs->resultType()); + sop->setOpType(scsp->resultType(), rhs->resultType()); ConstantFilter* cf = 0; - cf = new ConstantFilter(sop, gwip->scsp->clone(), rhs); + cf = new ConstantFilter(sop, scsp->clone(), lhs); sop.reset(new LogicOperator(cmbop)); cf->op(sop); sop.reset(new PredicateOperator(eqop)); - sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->timeZone)); + sop->setOpType(scsp->resultType(), rhs->resultType()); + cf->pushFilter(new SimpleFilter(sop, scsp->clone(), rhs->clone(), gwip->timeZone)); while (!gwip->rcWorkStack.empty()) { @@ -2088,8 +2136,8 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) gwip->rcWorkStack.pop(); sop.reset(new PredicateOperator(eqop)); - sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->timeZone)); + sop->setOpType(scsp->resultType(), lhs->resultType()); + cf->pushFilter(new SimpleFilter(sop, scsp->clone(), lhs->clone(), gwip->timeZone)); } if (!gwip->rcWorkStack.empty()) @@ -2414,7 +2462,9 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) RowColumn* rlhs = dynamic_cast(lhs); if (rrhs && rlhs) + { return buildRowColumnFilter(gwip, rrhs, rlhs, ifp); + } vector itemList; @@ -3326,6 +3376,78 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB(const Item* item) return ct; } +bool itemDisablesWrapping(Item* item, gp_walk_info& gwi) +{ + if (gwi.select_lex == nullptr) + { + return true; + } + ORDER* groupcol = static_cast(gwi.select_lex->group_list.first); + + while (groupcol) + { + Item* gci = *groupcol->item; + while (gci->type() == Item::REF_ITEM) + { + if (item->eq(gci, false)) + { + return true; + } + Item_ref* ref = (Item_ref*)gci; + gci = *(ref->ref); + } + if (item->eq(gci, false)) + { + return true; + } + groupcol = groupcol->next; + } + return false; +} +ReturnedColumn* wrapIntoAggregate(ReturnedColumn* rc, gp_walk_info& gwi, Item* baseItem) +{ + if (!gwi.implicitExplicitGroupBy || gwi.disableWrapping || !gwi.select_lex) + { + return rc; + } + + if (dynamic_cast(rc) != nullptr || dynamic_cast(rc) != nullptr) + { + return rc; + } + + if (itemDisablesWrapping(baseItem, gwi)) + { + return rc; + } + + cal_connection_info* ci = static_cast(get_fe_conn_info_ptr()); + + AggregateColumn* ac = new AggregateColumn(gwi.sessionid); + ac->timeZone(gwi.timeZone); + ac->alias(rc->alias()); + ac->aggOp(AggregateColumn::SELECT_SOME); + ac->asc(rc->asc()); + ac->charsetNumber(rc->charsetNumber()); + ac->orderPos(rc->orderPos()); + uint32_t i; + for(i=0; i < gwi.processed.size() && !gwi.processed[i].first->eq(baseItem, false);i++) + { } + if (i < gwi.processed.size()) + { + ac->expressionId(gwi.processed[i].second); + } + else + { + ac->expressionId(ci->expressionId++); + } + + ac->aggParms().push_back(SRCP(rc)); + ac->resultType(rc->resultType()); + return ac; +} + + ReturnedColumn* buildReturnedColumnNull(gp_walk_info& gwi) { if (gwi.condPush) @@ -3492,7 +3614,7 @@ static ConstantColumn* buildConstantColumnNotNullUsingValNative(Item* item, gp_w return rc; } -ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool isRefItem) +ReturnedColumn* buildReturnedColumnBody(Item* item, gp_walk_info& gwi, bool& nonSupport, bool isRefItem) { ReturnedColumn* rc = NULL; @@ -3514,12 +3636,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp { Item_field* ifp = (Item_field*)item; - if (isRefItem && gwi.isGroupByHandler && !gwi.extSelAggColsItems.empty()) - { - return buildAggFrmTempField(ifp, gwi); - } - - return buildSimpleColumn(ifp, gwi); + return wrapIntoAggregate(buildSimpleColumn(ifp, gwi), gwi, ifp); } case Item::NULL_ITEM: return buildReturnedColumnNull(gwi); case Item::CONST_ITEM: @@ -3560,7 +3677,9 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp if (func_name == "+" || func_name == "-" || func_name == "*" || func_name == "/") return buildArithmeticColumn(ifp, gwi, nonSupport); else + { return buildFunctionColumn(ifp, gwi, nonSupport); + } } case Item::SUM_FUNC_ITEM: @@ -3681,6 +3800,14 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp return rc; } +ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool isRefItem) +{ + bool disableWrapping = gwi.disableWrapping; + gwi.disableWrapping = gwi.disableWrapping || itemDisablesWrapping(item, gwi); + ReturnedColumn* rc = buildReturnedColumnBody(item, gwi, nonSupport, isRefItem); + gwi.disableWrapping = disableWrapping; + return rc; +} // parse the boolean fields to string "true" or "false" ReturnedColumn* buildBooleanConstantColumn(Item* item, gp_walk_info& gwi, bool& nonSupport) @@ -3711,7 +3838,7 @@ ReturnedColumn* buildBooleanConstantColumn(Item* item, gp_walk_info& gwi, bool& return cc; } -ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport) +ReturnedColumn* buildArithmeticColumnBody(Item_func* item, gp_walk_info& gwi, bool& nonSupport) { if (get_fe_conn_info_ptr() == NULL) { @@ -3754,7 +3881,8 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool // Could have it set if there are aggregation funcs as this function arguments. gwi.fatalParseError = false; - ReturnedColumn* rc = buildAggFrmTempField(sfitempp[0], gwi); + //ReturnedColumn* rc = buildAggFrmTempField(sfitempp[0], gwi); + ReturnedColumn* rc = buildReturnedColumn(sfitempp[0], gwi, nonSupport); if (rc) lhs = new ParseTree(rc); } @@ -3773,12 +3901,13 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool // Could have it set if there are aggregation funcs as this function arguments. gwi.fatalParseError = false; - ReturnedColumn* rc = buildAggFrmTempField(sfitempp[1], gwi); + //ReturnedColumn* rc = buildAggFrmTempField(sfitempp[1], gwi); + ReturnedColumn* rc = buildReturnedColumn(sfitempp[1], gwi, nonSupport); if (rc) rhs = new ParseTree(rc); } } - else // where clause + else // where clause SZ: XXX: is it also HAVING clause??? it appears so judging from condition above. { if (isPredicateFunction(sfitempp[1], &gwi)) { @@ -3945,7 +4074,8 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool ac->expressionId(ci->expressionId++); // @3391. optimization. try to associate expression ID to the expression on the select list - if (gwi.clauseType != SELECT) + bool isOnSelectList = false; + if (gwi.clauseType != SELECT || gwi.havingDespiteSelect) { for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) { @@ -3953,6 +4083,7 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool strcasecmp(ac->alias().c_str(), gwi.returnedCols[i]->alias().c_str()) == 0) { ac->expressionId(gwi.returnedCols[i]->expressionId()); + isOnSelectList = true; break; } } @@ -3971,10 +4102,24 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool } } + if (isOnSelectList && gwi.havingDespiteSelect) + { + SimpleColumn* sc = new SimpleColumn(*ac); + delete ac; + return sc; + } return ac; } +ReturnedColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport) +{ + bool disableWrapping = gwi.disableWrapping; + gwi.disableWrapping = gwi.disableWrapping || itemDisablesWrapping(item, gwi); + ReturnedColumn* rc = buildArithmeticColumnBody(item, gwi, nonSupport); + gwi.disableWrapping = disableWrapping; + return rc; +} -ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& nonSupport, bool selectBetweenIn) +ReturnedColumn* buildFunctionColumnBody(Item_func* ifp, gp_walk_info& gwi, bool& nonSupport, bool selectBetweenIn) { if (get_fe_conn_info_ptr() == NULL) { @@ -4037,8 +4182,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non // Arithmetic exp if (funcName == "+" || funcName == "-" || funcName == "*" || funcName == "/") { - ArithmeticColumn* ac = buildArithmeticColumn(ifp, gwi, nonSupport); - return ac; + return buildArithmeticColumn(ifp, gwi, nonSupport); } else if (funcName == "case") @@ -4217,7 +4361,9 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (mayHasBoolArg && isBoolType) rc = buildBooleanConstantColumn(ifp->arguments()[i], gwi, nonSupport); else + { rc = buildReturnedColumn(ifp->arguments()[i], gwi, nonSupport); + } // MCOL-1510 It must be a temp table field, so find the corresponding column. if (!rc && ifp->arguments()[i]->type() == Item::REF_ITEM) @@ -4583,6 +4729,14 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non return fc; } +ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& nonSupport, bool selectBetweenIn) +{ + bool disableWrapping = gwi.disableWrapping; + gwi.disableWrapping = gwi.disableWrapping || itemDisablesWrapping(ifp, gwi); + ReturnedColumn* rc = buildFunctionColumnBody(ifp, gwi, nonSupport, selectBetweenIn); + gwi.disableWrapping = disableWrapping; + return rc; +} FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonSupport) { @@ -5041,44 +5195,7 @@ void analyzeForImplicitGroupBy(Item* item, gp_walk_info& gwi) } } -ReturnedColumn* wrapIntoAggregate(ReturnedColumn* rc, gp_walk_info& gwi, SELECT_LEX& select_lex, Item* baseItem) -{ - if (!gwi.implicitExplicitGroupBy) - { - return rc; - } - - if (dynamic_cast(rc) != nullptr || dynamic_cast(rc) != nullptr) - { - return rc; - } - - ORDER* groupcol = static_cast(select_lex.group_list.first); - - while (groupcol) - { - if (baseItem->eq(*groupcol->item, false)) - { - return rc; - } - groupcol = groupcol->next; - } - - cal_connection_info* ci = static_cast(get_fe_conn_info_ptr()); - - AggregateColumn* ac = new AggregateColumn(gwi.sessionid); - ac->timeZone(gwi.timeZone); - ac->alias(rc->alias()); - ac->aggOp(AggregateColumn::SELECT_SOME); - ac->asc(rc->asc()); - ac->charsetNumber(rc->charsetNumber()); - ac->expressionId(ci->expressionId++); - - ac->aggParms().push_back(SRCP(rc)); - return ac; -} - -ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) +ReturnedColumn* buildAggregateColumnBody(Item* item, gp_walk_info& gwi) { // MCOL-1201 For UDAnF multiple parameters vector selCols; @@ -5378,6 +5495,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) { //@bug5229. handle constant function on aggregate argument ac->constCol(SRCP(rc)); + // XXX: this skips restoration of clauseType. break; } // the "rc" can be in gwi.no_parm_func_list. erase it from that list and @@ -5759,6 +5877,14 @@ because it has multiple arguments."; ac->charsetNumber(item->collation.collation->number); return ac; } +ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) +{ + bool disableWrapping = gwi.disableWrapping; + gwi.disableWrapping = true; + ReturnedColumn* rc = buildAggregateColumnBody(item, gwi); + gwi.disableWrapping = disableWrapping; + return rc; +} void addIntervalArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { @@ -5890,6 +6016,7 @@ void gp_walk(const Item* item, void* arg) if (ifp) { + // XXX: this looks awfuly wrong. SimpleColumn* scp = buildSimpleColumn(ifp, *gwip); if (!scp) @@ -5898,7 +6025,7 @@ void gp_walk(const Item* item, void* arg) string aliasTableName(scp->tableAlias()); scp->tableAlias(aliasTableName); gwip->rcWorkStack.push(scp->clone()); - boost::shared_ptr scsp(scp); + boost::shared_ptr scsp(scp); gwip->scsp = scsp; gwip->funcName.clear(); @@ -5993,7 +6120,7 @@ void gp_walk(const Item* item, void* arg) } ostringstream oss; - oss << "Unhandled Item type: " << item->type(); + oss << "Unhandled Item type(): " << item->type(); gwip->parseErrorText = oss.str(); gwip->fatalParseError = true; break; @@ -6343,7 +6470,7 @@ void gp_walk(const Item* item, void* arg) gwip->fatalParseError = false; } - SimpleColumn* sc = dynamic_cast(rc); + SimpleColumn* sc = clauseType == HAVING ? nullptr : dynamic_cast(rc); if (sc) { @@ -6437,19 +6564,27 @@ void gp_walk(const Item* item, void* arg) } else if (col->type() == Item::FIELD_ITEM && gwip->clauseType == HAVING) { - ReturnedColumn* rc = buildAggFrmTempField(const_cast(item), *gwip); + //ReturnedColumn* rc = buildAggFrmTempField(const_cast(item), *gwip); + ReturnedColumn* rc = buildReturnedColumn(const_cast(item), *gwip, gwip->fatalParseError); if (rc) gwip->rcWorkStack.push(rc); break; } else + { cando = false; + } - if (!cando) + SimpleColumn* thisSC = dynamic_cast(rc); + if (thisSC) + { + gwip->scsp.reset(thisSC->clone()); + } + if (!rc && !cando) { ostringstream oss; - oss << "Unhandled Item type: " << item->type(); + oss << "Unhandled Item type(): " << item->type(); gwip->parseErrorText = oss.str(); gwip->fatalParseError = true; } @@ -6502,7 +6637,6 @@ void gp_walk(const Item* item, void* arg) vector cols; // temp change clause type because the elements of row column are not walked yet gwip->clauseType = SELECT; - for (uint32_t i = 0; i < row->cols(); i++) cols.push_back(SRCP(buildReturnedColumn(row->element_index(i), *gwip, gwip->fatalParseError))); @@ -6561,7 +6695,7 @@ void gp_walk(const Item* item, void* arg) } ostringstream oss; - oss << "Unhandled Item type: " << item->type(); + oss << "Unhandled Item type (2): " << item->type(); gwip->parseErrorText = oss.str(); gwip->fatalParseError = true; break; @@ -6664,15 +6798,15 @@ void parse_item(Item* item, vector& field_vec, bool& hasNonSupportI // MCOL-1510. This could be a non-supported function // argument in form of a temp_table_field, so check // and set hasNonSupportItem if it is so. - ReturnedColumn* rc = NULL; - if (gwi) - rc = buildAggFrmTempField(ref, *gwi); + //ReturnedColumn* rc = NULL; + //if (gwi) + // rc = buildAggFrmTempField(ref, *gwi); - if (!rc) - { + //if (!rc) + //{ Item_field* ifp = static_cast(*(ref->ref)); field_vec.push_back(ifp); - } + //} break; } else if ((*(ref->ref))->type() == Item::FUNC_ITEM) @@ -7599,6 +7733,8 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i } gwi.clauseType = SELECT; + SELECT_LEX* oldSelectLex = gwi.select_lex; // XXX: SZ: should it be restored in case of error return? + gwi.select_lex = &select_lex; #ifdef DEBUG_WALK_COND { cerr << "------------------- SELECT --------------------" << endl; @@ -7707,10 +7843,10 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i } // We need to look into GROUP BY columns to decide if we need to wrap a column. - ReturnedColumn* rc = wrapIntoAggregate(sc, gwi, select_lex, baseItem); + ReturnedColumn* rc = wrapIntoAggregate(sc, gwi, baseItem); SRCP sprc(rc); - gwi.returnedCols.push_back(sprc); + pushReturnedCol(gwi, baseItem, sprc); gwi.columnMap.insert( CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name.str), sprc)); @@ -7747,7 +7883,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // add this agg col to returnedColumnList boost::shared_ptr spac(ac); - gwi.returnedCols.push_back(spac); + pushReturnedCol(gwi, item, spac); break; } @@ -7806,7 +7942,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i if (!hasNonSupportItem && ifp->const_item() && !(parseInfo & AF_BIT) && tmpVec.size() == 0) { srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); + pushReturnedCol(gwi, item, srcp); if (ifp->name.length) srcp->alias(ifp->name.str); @@ -7814,7 +7950,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i continue; } - gwi.returnedCols.push_back(srcp); + pushReturnedCol(gwi, item, srcp); } else // This was a vtable post-process block { @@ -7836,7 +7972,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i if (ifp->name.length) cc->alias(ifp->name.str); - gwi.returnedCols.push_back(srcp); + pushReturnedCol(gwi, ifp, srcp); // clear the error set by buildFunctionColumn gwi.fatalParseError = false; @@ -7914,7 +8050,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i if (item->name.length) srcp->alias(item->name.str); - gwi.returnedCols.push_back(srcp); + pushReturnedCol(gwi, item, srcp); } break; @@ -7938,7 +8074,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i else { SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); + pushReturnedCol(gwi, item, srcp); if (item->name.length) srcp->alias(item->name.str); @@ -8034,7 +8170,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } - gwi.returnedCols.push_back(srcp); + pushReturnedCol(gwi, item, srcp); break; } case Item::TYPE_HOLDER: @@ -8095,12 +8231,16 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // Having clause handling gwi.clauseType = HAVING; + gwi.havingDespiteSelect = true; clearStacks(gwi, false, true); std::unique_ptr havingFilter; + // clear fatalParseError that may be left from post process functions gwi.fatalParseError = false; gwi.parseErrorText = ""; + gwi.disableWrapping = false; + gwi.havingDespiteSelect = true; if (select_lex.having != 0) { #ifdef DEBUG_WALK_COND @@ -8142,6 +8282,8 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.ptWorkStack.push(ptp); } } + gwi.havingDespiteSelect = false; + gwi.disableWrapping = false; // MCOL-4617 If this is an IN subquery, then create the in-to-exists // predicate and inject it into the csep @@ -8227,7 +8369,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i funcFieldVec[i]->print(&str, QT_ORDINARY); sc->alias(string(str.c_ptr())); sc->tableAlias(sc->tableAlias()); - SRCP srcp(sc); + SRCP srcp(wrapIntoAggregate(sc, gwi, funcFieldVec[i])); uint32_t j = 0; for (; j < gwi.returnedCols.size(); j++) @@ -8244,6 +8386,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i if (j == gwi.returnedCols.size()) { gwi.returnedCols.push_back(srcp); + // XXX: SZ: deduplicate here? gwi.columnMap.insert( CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name.str), srcp)); @@ -8288,6 +8431,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.hasWindowFunc = hasWindowFunc; groupcol = static_cast(select_lex.group_list.first); + gwi.disableWrapping = true; for (; groupcol; groupcol = groupcol->next) { Item* groupItem = *(groupcol->item); @@ -8511,6 +8655,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i nonSupportItem = groupItem; } } + gwi.disableWrapping = false; // @bug 4756. Add internal groupby column for correlated join to the groupby list if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) @@ -8629,7 +8774,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i { rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); - rc = wrapIntoAggregate(rc, gwi, select_lex, ord_item); + rc = wrapIntoAggregate(rc, gwi, ord_item); } // @bug5501 try item_ptr if item can not be fixed. For some // weird dml statement state, item can not be fixed but the @@ -8856,6 +9001,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i for (uint32_t i = 0; i < gwi.localCols.size(); i++) gwi.localCols[i]->sequence(i); + gwi.select_lex = oldSelectLex; // append additionalRetCols to returnedCols gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), gwi.additionalRetCols.end()); @@ -9000,6 +9146,7 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) gp_walk_info gwi(timeZoneOffset, &chain); gwi.thd = thd; gwi.isGroupByHandler = true; + idbassert(0); int status = getGroupPlan(gwi, *select_lex, csep, gi); #ifdef DEBUG_WALK_COND diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 98b488f57..3ab43564e 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -117,7 +117,7 @@ struct gp_walk_info std::vector localCols; std::stack rcWorkStack; std::stack ptWorkStack; - boost::shared_ptr scsp; + boost::shared_ptr scsp; // while defined as SSCP, it is used as SRCP, nothing specific to SimpleColumn is used in use sites. uint32_t sessionid; bool fatalParseError; std::string parseErrorText; @@ -144,6 +144,7 @@ struct gp_walk_info // we can have explicit GROUP BY and implicit one, triggered by aggregate in pojection or ORDER BY. // this flag tells us whether we have either case. bool implicitExplicitGroupBy; + bool disableWrapping; bool aggOnSelect; bool hasWindowFunc; bool hasSubSelect; @@ -179,6 +180,15 @@ struct gp_walk_info TableOnExprList tableOnExprList; std::vector condList; + // Item* associated with returnedCols. + std::vector> processed; + + // SELECT_LEX is needed for aggergate wrapping + SELECT_LEX* select_lex; + + // we are processing HAVING despite having (pun not intented) clauseType equal to SELECT. + bool havingDespiteSelect; + // All SubQuery allocations are single-linked into this chain. // At the end of gp_walk_info processing we can free whole chain at once. // This is done so because the juggling of SubQuery pointers in the @@ -198,6 +208,7 @@ struct gp_walk_info , subQuery(0) , clauseType(INIT) , implicitExplicitGroupBy(false) + , disableWrapping(false) , aggOnSelect(false) , hasWindowFunc(false) , hasSubSelect(false) @@ -212,6 +223,8 @@ struct gp_walk_info , timeZone(timeZone_) , inSubQueryLHS(nullptr) , inSubQueryLHSItem(nullptr) + , select_lex(nullptr) + , havingDespiteSelect(false) , subQueriesChain(subQueriesChain_) { } @@ -424,7 +437,7 @@ execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, boo bool isRefItem = false); execplan::ReturnedColumn* buildFunctionColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, bool selectBetweenIn = false); -execplan::ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport); +execplan::ReturnedColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport); execplan::ConstantColumn* buildDecimalColumn(const Item* item, const std::string& str, gp_walk_info& gwi); execplan::SimpleColumn* buildSimpleColumn(Item_field* item, gp_walk_info& gwi); execplan::FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonSupport); diff --git a/mysql-test/columnstore/basic/r/mcs76_having.result b/mysql-test/columnstore/basic/r/mcs76_having.result index 3b68e51be..d03e79c2c 100644 --- a/mysql-test/columnstore/basic/r/mcs76_having.result +++ b/mysql-test/columnstore/basic/r/mcs76_having.result @@ -26,6 +26,9 @@ SELECT col1, col2, SUM(LENGTH(col2)) FROM t1 GROUP BY col1 HAVING col1 > 1 AND c col1 col2 SUM(LENGTH(col2)) 2 oooooooooooooooooooo 40 4 ooo 6 +SELECT 'some output'; +some output +some output CREATE TABLE t2(col1 INT, col2 DATETIME)ENGINE=Columnstore; INSERT INTO t2 VALUES(1, '2020-2-2'),(2, '2020-3-3'),(5,'2020-6-6'),(6, '2020-7-7'); SELECT t1.col1, SUM(t1.col1*t2.col1) AS a FROM t1 JOIN t2 ON t1.col1 = t2.col1 GROUP BY t1.col1 HAVING a>1 ORDER BY t1.col1; diff --git a/mysql-test/columnstore/basic/t/mcs76_having.test b/mysql-test/columnstore/basic/t/mcs76_having.test index 5286cee4e..521a74a86 100644 --- a/mysql-test/columnstore/basic/t/mcs76_having.test +++ b/mysql-test/columnstore/basic/t/mcs76_having.test @@ -21,6 +21,7 @@ SELECT col1, col2 FROM t1 GROUP BY col1, col2 HAVING col1 > 1 OR col2 LIKE '%o%' --sorted_result SELECT col1, col2, SUM(LENGTH(col2)) FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +SELECT 'some output'; CREATE TABLE t2(col1 INT, col2 DATETIME)ENGINE=Columnstore; INSERT INTO t2 VALUES(1, '2020-2-2'),(2, '2020-3-3'),(5,'2020-6-6'),(6, '2020-7-7'); SELECT t1.col1, SUM(t1.col1*t2.col1) AS a FROM t1 JOIN t2 ON t1.col1 = t2.col1 GROUP BY t1.col1 HAVING a>1 ORDER BY t1.col1; diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.result b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.result new file mode 100644 index 000000000..d9a16c366 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.result @@ -0,0 +1,214 @@ +DROP DATABASE IF EXISTS MCOL5776; +CREATE DATABASE MCOL5776; +USE MCOL5776; +CREATE TABLE t(x INTEGER, y INTEGER) ENGINE=Columnstore; +INSERT INTO t(x,y) VALUES (1,2), (2,3), (3,3); +SELECT COUNT(y) OVER (PARTITION BY y) FROM t GROUP BY x; +COUNT(y) OVER (PARTITION BY y) +1 +2 +2 +SELECT COUNT(y) OVER (PARTITION BY LEFT(y, 10)) FROM t GROUP BY x; +COUNT(y) OVER (PARTITION BY LEFT(y, 10)) +1 +2 +2 +DROP TABLE t; +CREATE TABLE t(ci1 integer, ci2 integer) engine=Columnstore; +INSERT INTO t(ci1, ci2) VALUES (NULL, 1), (NULL, 2), (1,3), (1,4), (2,5), (2,6), (3,7), (3,8); +SELECT ci1+ci2, ci1+ci2, SUM(ci2), AVG(ci2) FROM t GROUP BY ci1+ci2, ci1+ci2; +ci1+ci2 ci1+ci2 SUM(ci2) AVG(ci2) +10 10 7 7.0000 +11 11 8 8.0000 +4 4 3 3.0000 +5 5 4 4.0000 +7 7 5 5.0000 +8 8 6 6.0000 +NULL NULL 3 1.5000 +SELECT CONCAT(ci1,ci2), CONCAT(ci1,ci2), SUM(ci2), AVG(ci2) FROM t GROUP BY ci1; +CONCAT(ci1,ci2) CONCAT(ci1,ci2) SUM(ci2) AVG(ci2) +14 14 7 3.5000 +26 26 11 5.5000 +38 38 15 7.5000 +NULL NULL 3 1.5000 +SELECT sum(ci1), abs(ci1) FROM t GROUP BY abs(ci1), abs(ci1); +sum(ci1) abs(ci1) +2 1 +4 2 +6 3 +NULL NULL +SELECT sum(ci1), abs(ci1) FROM t GROUP BY abs(ci1); +sum(ci1) abs(ci1) +2 1 +4 2 +6 3 +NULL NULL +DROP TABLE t; +CREATE TABLE t1(col1 INT, col2 TEXT)ENGINE=Columnstore; +INSERT INTO t1 VALUES(1, repeat('s',20)),(2, repeat('o',20)),(3, 'sss'),(4, 'ooo'); +INSERT INTO t1 SELECT * FROM t1; +SELECT * FROM t1; +col1 col2 +1 ssssssssssssssssssss +2 oooooooooooooooooooo +3 sss +4 ooo +1 ssssssssssssssssssss +2 oooooooooooooooooooo +3 sss +4 ooo +SELECT col2 FROM t1 GROUP BY col2 HAVING col2 LIKE '%o%' ORDER BY col2; +col2 +ooo +oooooooooooooooooooo +SELECT col1 FROM t1 GROUP BY col1 HAVING col1 > 1 ORDER BY col1; +col1 +2 +3 +4 +SELECT col1, col2 FROM t1 GROUP BY col2 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +col1 col2 +2 oooooooooooooooooooo +4 ooo +SELECT col1, col2 FROM t1 GROUP BY col2 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +col1 col2 +2 oooooooooooooooooooo +4 ooo +SELECT col2, col1 FROM t1 GROUP BY col2 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +col2 col1 +oooooooooooooooooooo 2 +ooo 4 +SELECT col2, col1 FROM t1 GROUP BY col2 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +col2 col1 +oooooooooooooooooooo 2 +ooo 4 +SELECT col1, col2 FROM t1 GROUP BY col2 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +col1 col2 +2 oooooooooooooooooooo +4 ooo +SELECT col1, col2 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +col1 col2 +2 oooooooooooooooooooo +4 ooo +SELECT col1, col2 FROM t1 GROUP BY col1 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +col1 col2 +2 oooooooooooooooooooo +4 ooo +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +col2 col1 +oooooooooooooooooooo 2 +ooo 4 +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +col2 col1 +oooooooooooooooooooo 2 +ooo 4 +SELECT col1, col2 FROM t1 GROUP BY col1 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +col1 col2 +2 oooooooooooooooooooo +4 ooo +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 = 'ooo' ORDER BY col1; +col2 col1 +ooo 4 +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col2 = 'ooo' AND col1 > 1 ORDER BY col1; +col2 col1 +ooo 4 +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 >= 'ooo' ORDER BY col1; +col2 col1 +oooooooooooooooooooo 2 +sss 3 +ooo 4 +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col2 >= 'ooo' AND col1 > 1 ORDER BY col1; +col2 col1 +oooooooooooooooooooo 2 +sss 3 +ooo 4 +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 < 'ooo' ORDER BY col1; +col2 col1 +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col2 < 'ooo' AND col1 > 1 ORDER BY col1; +col2 col1 +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND 'ooo' < col2 ORDER BY col1; +col2 col1 +oooooooooooooooooooo 2 +sss 3 +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING 'ooo' < col2 AND col1 > 1 ORDER BY col1; +col2 col1 +oooooooooooooooooooo 2 +sss 3 +SELECT col1, col2, SUM(LENGTH(col2)) FROM t1 GROUP BY col1 HAVING SUM(LENGTH(col2)) > 10 ORDER BY col1; +col1 col2 SUM(LENGTH(col2)) +1 ssssssssssssssssssss 40 +2 oooooooooooooooooooo 40 +SELECT col1, col2, SUM(LENGTH(col2)) a FROM t1 GROUP BY col1 HAVING a > 1 AND col2 LIKE '%o%' ORDER BY col1; +col1 col2 a +2 oooooooooooooooooooo 40 +4 ooo 6 +DROP TABLE t1; +CREATE TABLE empsalary ( +depname VARCHAR(100), +empno BIGINT, +salary INT, +enroll_date DATE +) ENGINE=Columnstore; +INSERT INTO empsalary VALUES ('develop' , 10, 5200, '2007-08-01'); +INSERT INTO empsalary VALUES ('sales' , 1, 5000, '2006-10-01'); +INSERT INTO empsalary VALUES ('personnel', 5, 3500, '2007-12-10'); +INSERT INTO empsalary VALUES ('sales' , 4, 4800, '2007-08-08'); +INSERT INTO empsalary VALUES ('personnel', 2, 3900, '2006-12-23'); +INSERT INTO empsalary VALUES ('develop' , 7, 4200, '2008-01-01'); +INSERT INTO empsalary VALUES ('develop' , 9, 4500, '2008-01-01'); +INSERT INTO empsalary VALUES ('sales' , 3, 4800, '2007-08-01'); +INSERT INTO empsalary VALUES ('develop' , 8, 6000, '2006-10-01'); +INSERT INTO empsalary VALUES ('develop' , 11, 5200, '2007-08-15'); +INSERT INTO empsalary VALUES ('develop' , 12, null, '2008-08-09'); +SELECT depname, empno, MODA(salary) OVER(PARTITION BY depname ORDER BY enroll_date) FROM empsalary ORDER BY depname, empno, enroll_date; +depname empno MODA(salary) OVER(PARTITION BY depname ORDER BY enroll_date) +develop 7 5200 +develop 8 6000 +develop 9 5200 +develop 10 5200 +develop 11 5200 +develop 12 5200 +personnel 2 3900 +personnel 5 3500 +sales 1 5000 +sales 3 4800 +sales 4 4800 +SELECT AVG(salary),depname, MODA(salary) OVER(PARTITION BY depname ORDER BY enroll_date) FROM empsalary GROUP BY depname ORDER BY depname, AVG(salary); +AVG(salary) depname MODA(salary) OVER(PARTITION BY depname ORDER BY enroll_date) +5020.0000 develop 0 +3700.0000 personnel 3900 +4866.6667 sales 4800 +DROP TABLE empsalary; +CREATE TABLE orders(o_custkey INT) ENGINE=Columnstore; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) < 20 ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) <= 20 ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) > 20 ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) >= 20 ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) = 20 ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) <> 20 ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) IN (15, 20) ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) NOT IN (15, 20) ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) BETWEEN 15 AND 20 ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) NOT BETWEEN 15 and 20 ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) IS NULL ORDER BY 1; +o_custkey COUNT(*) +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) IS NOT NULL ORDER BY 1; +o_custkey COUNT(*) +DROP TABLE orders; +CREATE TABLE t(k INT) ENGINE = Columnstore; +INSERT INTO t(k) VALUES (1), (2), (2), (3), (3), (4), (4),(4),(4),(4),(995), (NULL); +SELECT k + k a FROM t GROUP BY a HAVING a >= 8; +a +1990 +8 +DROP DATABASE MCOL5776; diff --git a/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test new file mode 100644 index 000000000..6f1776a1b --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-5776-GROUP-BY-HAVING-functions-use.test @@ -0,0 +1,99 @@ +--disable_warnings +DROP DATABASE IF EXISTS MCOL5776; +--enable_warnings +CREATE DATABASE MCOL5776; +USE MCOL5776; + +CREATE TABLE t(x INTEGER, y INTEGER) ENGINE=Columnstore; +INSERT INTO t(x,y) VALUES (1,2), (2,3), (3,3); +SELECT COUNT(y) OVER (PARTITION BY y) FROM t GROUP BY x; +SELECT COUNT(y) OVER (PARTITION BY LEFT(y, 10)) FROM t GROUP BY x; +DROP TABLE t; + +CREATE TABLE t(ci1 integer, ci2 integer) engine=Columnstore; +INSERT INTO t(ci1, ci2) VALUES (NULL, 1), (NULL, 2), (1,3), (1,4), (2,5), (2,6), (3,7), (3,8); +--sorted_result +SELECT ci1+ci2, ci1+ci2, SUM(ci2), AVG(ci2) FROM t GROUP BY ci1+ci2, ci1+ci2; +--sorted_result +SELECT CONCAT(ci1,ci2), CONCAT(ci1,ci2), SUM(ci2), AVG(ci2) FROM t GROUP BY ci1; +--sorted_result +SELECT sum(ci1), abs(ci1) FROM t GROUP BY abs(ci1), abs(ci1); +--sorted_result +SELECT sum(ci1), abs(ci1) FROM t GROUP BY abs(ci1); +DROP TABLE t; + +CREATE TABLE t1(col1 INT, col2 TEXT)ENGINE=Columnstore; +INSERT INTO t1 VALUES(1, repeat('s',20)),(2, repeat('o',20)),(3, 'sss'),(4, 'ooo'); +INSERT INTO t1 SELECT * FROM t1; +SELECT * FROM t1; +SELECT col2 FROM t1 GROUP BY col2 HAVING col2 LIKE '%o%' ORDER BY col2; +SELECT col1 FROM t1 GROUP BY col1 HAVING col1 > 1 ORDER BY col1; +SELECT col1, col2 FROM t1 GROUP BY col2 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +SELECT col1, col2 FROM t1 GROUP BY col2 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col2 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col2 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +SELECT col1, col2 FROM t1 GROUP BY col2 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +SELECT col1, col2 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +SELECT col1, col2 FROM t1 GROUP BY col1 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 LIKE '%o%' ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +SELECT col1, col2 FROM t1 GROUP BY col1 HAVING col2 LIKE '%o%' AND col1 > 1 ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 = 'ooo' ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col2 = 'ooo' AND col1 > 1 ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 >= 'ooo' ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col2 >= 'ooo' AND col1 > 1 ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND col2 < 'ooo' ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col2 < 'ooo' AND col1 > 1 ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING col1 > 1 AND 'ooo' < col2 ORDER BY col1; +SELECT col2, col1 FROM t1 GROUP BY col1 HAVING 'ooo' < col2 AND col1 > 1 ORDER BY col1; +SELECT col1, col2, SUM(LENGTH(col2)) FROM t1 GROUP BY col1 HAVING SUM(LENGTH(col2)) > 10 ORDER BY col1; +SELECT col1, col2, SUM(LENGTH(col2)) a FROM t1 GROUP BY col1 HAVING a > 1 AND col2 LIKE '%o%' ORDER BY col1; +DROP TABLE t1; + +CREATE TABLE empsalary ( + depname VARCHAR(100), + empno BIGINT, + salary INT, + enroll_date DATE +) ENGINE=Columnstore; + +INSERT INTO empsalary VALUES ('develop' , 10, 5200, '2007-08-01'); +INSERT INTO empsalary VALUES ('sales' , 1, 5000, '2006-10-01'); +INSERT INTO empsalary VALUES ('personnel', 5, 3500, '2007-12-10'); +INSERT INTO empsalary VALUES ('sales' , 4, 4800, '2007-08-08'); +INSERT INTO empsalary VALUES ('personnel', 2, 3900, '2006-12-23'); +INSERT INTO empsalary VALUES ('develop' , 7, 4200, '2008-01-01'); +INSERT INTO empsalary VALUES ('develop' , 9, 4500, '2008-01-01'); +INSERT INTO empsalary VALUES ('sales' , 3, 4800, '2007-08-01'); +INSERT INTO empsalary VALUES ('develop' , 8, 6000, '2006-10-01'); +INSERT INTO empsalary VALUES ('develop' , 11, 5200, '2007-08-15'); +INSERT INTO empsalary VALUES ('develop' , 12, null, '2008-08-09'); + +SELECT depname, empno, MODA(salary) OVER(PARTITION BY depname ORDER BY enroll_date) FROM empsalary ORDER BY depname, empno, enroll_date; +SELECT AVG(salary),depname, MODA(salary) OVER(PARTITION BY depname ORDER BY enroll_date) FROM empsalary GROUP BY depname ORDER BY depname, AVG(salary); +DROP TABLE empsalary; + +CREATE TABLE orders(o_custkey INT) ENGINE=Columnstore; + +# These checks for absence of exceptions and SIGSEGV's +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) < 20 ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) <= 20 ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) > 20 ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) >= 20 ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) = 20 ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) <> 20 ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) IN (15, 20) ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) NOT IN (15, 20) ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) BETWEEN 15 AND 20 ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) NOT BETWEEN 15 and 20 ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) IS NULL ORDER BY 1; +SELECT o_custkey, COUNT(*) FROM orders WHERE o_custkey < 100 GROUP BY o_custkey HAVING COUNT(*) IS NOT NULL ORDER BY 1; + +DROP TABLE orders; + +CREATE TABLE t(k INT) ENGINE = Columnstore; +INSERT INTO t(k) VALUES (1), (2), (2), (3), (3), (4), (4),(4),(4),(4),(995), (NULL); +--sorted_result +SELECT k + k a FROM t GROUP BY a HAVING a >= 8; + +DROP DATABASE MCOL5776; diff --git a/mysql-test/columnstore/devregression/r/mcs7224_regression_MCOL-3503.result b/mysql-test/columnstore/devregression/r/mcs7224_regression_MCOL-3503.result index e725da196..ac8132e29 100644 --- a/mysql-test/columnstore/devregression/r/mcs7224_regression_MCOL-3503.result +++ b/mysql-test/columnstore/devregression/r/mcs7224_regression_MCOL-3503.result @@ -14,14 +14,6 @@ sales 3 4800 sales 4 4800 select avg(salary),depname, moda(salary) over(partition by depname order by enroll_date) from empsalary group by depname order by depname, avg(salary); avg(salary) depname moda(salary) over(partition by depname order by enroll_date) -NULL develop 5200 -4200.0000 develop 5200 -4500.0000 develop 5200 -5200.0000 develop 5200 -5200.0000 develop 5200 -6000.0000 develop 6000 -3500.0000 personnel 3500 -3900.0000 personnel 3900 -4800.0000 sales 4800 -4800.0000 sales 4800 -5000.0000 sales 5000 +5020.0000 develop 0 +3700.0000 personnel 3900 +4866.6667 sales 4800 diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 707595b70..9e5a47d37 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10) project(storagemanager) -include_directories(include ${CMAKE_BINARY_DIR}/include ${ENGINE_UTILS_COMMON_INCLUDE} ${S3API_DIR}) +include_directories(include ${CMAKE_BINARY_DIR}/include ${ENGINE_COMMON_INCLUDES} ${S3API_DIR}) set(storagemanager_SRCS src/AppendTask.cpp diff --git a/tests/scripts/fullmtr.sh b/tests/scripts/fullmtr.sh index f3b16b570..8ed3f2133 100644 --- a/tests/scripts/fullmtr.sh +++ b/tests/scripts/fullmtr.sh @@ -25,7 +25,7 @@ fi run_suite() { ls /core >$CURRENT_DIR/mtr.$1.cores-before - ./mtr --force --max-test-fail=0 --testcase-timeout=60 --suite=columnstore/$1 $2 | tee $CURRENT_DIR/mtr.$1.log 2>&1 + ./mtr --force --extern=socket=/run/mysqld/mysqld.sock --max-test-fail=0 --testcase-timeout=60 --suite=columnstore/$1 $2 | tee $CURRENT_DIR/mtr.$1.log 2>&1 # dump analyses. systemctl stop mariadb systemctl start mariadb diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 62fd3a306..927dc3c7e 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1252,6 +1252,7 @@ inline void Row::setUintField(uint64_t val, uint32_t colIndex) template inline void Row::setIntField(int64_t val, uint32_t colIndex) { +// idbassert(getColumnWidth(colIndex) == len); switch (len) { case 1: *((int8_t*)&data[offsets[colIndex]]) = val; break; From 50a9f82f76e78b5e8090fc22e276afe0ff4d024f Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Tue, 3 Dec 2024 04:13:47 +0300 Subject: [PATCH 32/65] fix(client): MCOL-5587: Add quick-max-column-width for maridb clients. --- dbcon/mysql/columnstore.cnf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/columnstore.cnf b/dbcon/mysql/columnstore.cnf index 989978790..cafda6bd4 100644 --- a/dbcon/mysql/columnstore.cnf +++ b/dbcon/mysql/columnstore.cnf @@ -1,9 +1,9 @@ [mariadb-client] quick +quick-max-column-width=0 [mysqld] plugin-load-add=ha_columnstore.so collation_server = utf8_general_ci character_set_server = utf8 loose-columnstore_use_import_for_batchinsert = ON - From acce8d091a45364366e4a143d6e9877384169f8d Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Fri, 18 Oct 2024 13:19:48 +0000 Subject: [PATCH 33/65] MCOL-5724 Move journal to FDB --- storage-manager/src/IOCoordinator.cpp | 345 ++++++------------ storage-manager/src/IOCoordinator.h | 10 +- storage-manager/src/KVPrefixes.cpp | 9 +- .../src/{KVPrefixes.hpp => KVPrefixes.h} | 7 +- storage-manager/src/MetadataFile.cpp | 2 +- storage-manager/src/Ownership.cpp | 2 +- storage-manager/src/PrefixCache.cpp | 21 +- storage-manager/src/Replicator.cpp | 269 +++++--------- storage-manager/src/Replicator.h | 3 +- storage-manager/src/Synchronizer.cpp | 114 +++--- 10 files changed, 308 insertions(+), 474 deletions(-) rename storage-manager/src/{KVPrefixes.hpp => KVPrefixes.h} (88%) diff --git a/storage-manager/src/IOCoordinator.cpp b/storage-manager/src/IOCoordinator.cpp index 5a6adaf9a..4bc938834 100644 --- a/storage-manager/src/IOCoordinator.cpp +++ b/storage-manager/src/IOCoordinator.cpp @@ -16,6 +16,8 @@ MA 02110-1301, USA. */ #include "IOCoordinator.h" +#include "KVStorageInitializer.h" +#include "KVPrefixes.h" #include "MetadataFile.h" #include "Synchronizer.h" #include @@ -24,6 +26,7 @@ #include #include #include +#include "cstring" #include #define BOOST_SPIRIT_THREADSAFE #include @@ -189,7 +192,8 @@ ssize_t IOCoordinator::read(const char* _filename, uint8_t* data, off_t offset, } vector relevants = meta.metadataRead(offset, length); - map journalFDs, objectFDs; + map objectFDs; + unordered_set journalFDs; map keyToJournalName, keyToObjectName; utils::VLArray fdMinders(relevants.size() * 2); int mindersIndex = 0; @@ -211,29 +215,21 @@ ssize_t IOCoordinator::read(const char* _filename, uint8_t* data, off_t offset, // later. not thinking about it for now. // open all of the journal files that exist - string jFilename = (journalPath / firstDir / (key + ".journal")).string(); - int fd = ::open(jFilename.c_str(), O_RDONLY); - if (fd >= 0) + const string journal = getJournalName((journalPath / firstDir / (key + ".journal")).string()); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto resultPairJournal = tnx->get(journal); + if (resultPairJournal.first) { - keyToJournalName[key] = jFilename; - journalFDs[key] = fd; - fdMinders[mindersIndex++].fd = fd; - // fdMinders.push_back(SharedCloser(fd)); - } - else if (errno != ENOENT) - { - int l_errno = errno; - fileLock.unlock(); - cache->doneReading(firstDir, keys); - logger->log(LOG_CRIT, "IOCoordinator::read(): Got an unexpected error opening %s, error was '%s'", - jFilename.c_str(), strerror_r(l_errno, buf, 80)); - errno = l_errno; - return -1; + keyToJournalName[key] = journal; + journalFDs.insert(key); } + // else + // Journal already merged with object file. // open all of the objects string oFilename = (cachePath / firstDir / key).string(); - fd = ::open(oFilename.c_str(), O_RDONLY); + auto fd = ::open(oFilename.c_str(), O_RDONLY); if (fd < 0) { int l_errno = errno; @@ -247,9 +243,7 @@ ssize_t IOCoordinator::read(const char* _filename, uint8_t* data, off_t offset, keyToObjectName[key] = oFilename; objectFDs[key] = fd; fdMinders[mindersIndex++].fd = fd; - // fdMinders.push_back(SharedCloser(fd)); } - // fileLock.unlock(); // ^^^ TODO: The original version unlocked the file at the above position. On second glance, // I'm not sure how we can guarantee the journal files won't be modified during the loads below. @@ -857,7 +851,14 @@ int IOCoordinator::_truncate(const bf::path& bfpath, size_t newSize, ScopedFileL if (result & 0x1) replicator->remove(cachePath / firstDir / objects[i].key); if (result & 0x2) - replicator->remove(journalPath / firstDir / (objects[i].key + ".journal")); + { + const auto journalName = + getJournalName((journalPath / firstDir / (objects[i].key + ".journal")).string()); + const auto journalSizeName = + getJournalName((journalPath / firstDir / (objects[i].key + "_size.journal")).string()); + replicator->removeJournal(journalName); + replicator->removeJournalSize(journalSizeName); + } deletedObjects.push_back(objects[i].key); } if (!deletedObjects.empty()) @@ -905,7 +906,11 @@ void IOCoordinator::deleteMetaFile(const bf::path& file) if (result & 0x2) { ++iocFilesDeleted; - replicator->remove(journalPath / firstDir / (object.key + ".journal")); + const auto journalName = getJournalName((journalPath / firstDir / (object.key + ".journal")).string()); + const auto journalSizeName = + getJournalName((journalPath / firstDir / (object.key + "_size.journal")).string()); + replicator->removeJournalSize(journalSizeName); + replicator->removeJournal(journalName); } deletedObjects.push_back(object.key); } @@ -1021,20 +1026,6 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) int err; char errbuf[80]; - // since we don't implement mkdir(), assume the caller did that and - // create any necessary parent dirs for filename2 - try - { - bf::create_directories(metaFile2.parent_path()); - } - catch (bf::filesystem_error& e) - { - logger->log(LOG_CRIT, "IOCoordinator::copyFile(): failed to create directory %s. Got %s", - metaFile2.parent_path().string().c_str(), strerror_r(e.code().value(), errbuf, 80)); - errno = e.code().value(); - return -1; - } - vector> newJournalEntries; ScopedReadLock lock(this, filename1); ScopedWriteLock lock2(this, filename2); @@ -1060,8 +1051,6 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) { for (const auto& object : objects) { - bf::path journalFile = journalPath / firstDir1 / (object.key + ".journal"); - // originalLength = the length of the object before journal entries. // the length in the metadata is the length after journal entries size_t originalLength = MetadataFile::getLengthFromKey(object.key); @@ -1095,27 +1084,36 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) object.key + ": " + strerror_r(errno, errbuf, 80)); } - // if there's a journal file for this object, make a copy - if (bf::exists(journalFile)) + const auto journalName = getJournalName((journalPath / firstDir1 / (object.key + ".journal")).string()); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler journalReader(keyGen); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto resultPair = journalReader.readBlob(kvStorage, journalName); + if (resultPair.first) { - bf::path newJournalFile = journalPath / firstDir2 / (newObj.key + ".journal"); - try + const auto& journalData = resultPair.second; + const std::string oldJournalDataHeader = resultPair.second; + const auto newJournalName = + getJournalName((journalPath / firstDir2 / (newObj.key + ".journal")).string()); + const auto newJournalSizeName = + getJournalName((journalPath / firstDir2 / (newObj.key + "_size.journal")).string()); + + FDBCS::BlobHandler journalWriter(keyGen); + if (!journalWriter.writeBlob(kvStorage, newJournalName, journalData)) + logger->log(LOG_CRIT, "Cannot write new journal, while copying files."); + { - bf::copy_file(journalFile, newJournalFile); - size_t tmp = bf::file_size(newJournalFile); - ++iocJournalsCreated; - iocBytesRead += tmp; - iocBytesWritten += tmp; - cache->newJournalEntry(firstDir2, tmp); - newJournalEntries.push_back(pair(newObj.key, tmp)); - } - catch (bf::filesystem_error& e) - { - throw CFException(e.code().value(), string("IOCoordinator::copyFile(): source = ") + filename1 + - ", dest = " + filename2 + ". Got an error copying " + - journalFile.string() + ": " + - strerror_r(e.code().value(), errbuf, 80)); + auto tnx = kvStorage->createTransaction(); + tnx->set(newJournalSizeName, to_string(journalData.size())); + if (!tnx->commit()) + logger->log(LOG_CRIT, "Cannot write new journal size, while copying files."); } + + ++iocJournalsCreated; + iocBytesRead += journalData.size(); + iocBytesWritten += journalData.size(); + cache->newJournalEntry(firstDir2, journalData.size()); + newJournalEntries.push_back(pair(newObj.key, journalData.size())); } } } @@ -1126,9 +1124,12 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) cs->deleteObject(newObject.key); for (auto& jEntry : newJournalEntries) { - bf::path fullJournalPath = journalPath / firstDir2 / (jEntry.first + ".journal"); + const auto fullJournalPath = + getJournalName((journalPath / firstDir2 / (jEntry.first + ".journal")).string()); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + tnx->remove(fullJournalPath); cache->deletedJournal(firstDir2, bf::file_size(fullJournalPath)); - bf::remove(fullJournalPath); } errno = e.l_errno; return -1; @@ -1163,28 +1164,21 @@ const bf::path& IOCoordinator::getMetadataPath() const // first byte after the header. // update: had to make it also return the header; the boost json parser does not stop at either // a null char or the end of an object. -std::shared_ptr seekToEndOfHeader1(int fd, size_t* _bytesRead) +std::shared_ptr seekToEndOfHeader1_(const std::string& dataStr, size_t* _bytesRead) { - //::lseek(fd, 0, SEEK_SET); - std::shared_ptr ret(new char[100]); - int err; + const size_t numBytesToRead = dataStr.size(); + std::shared_ptr ret(new char[numBytesToRead]); + std::memcpy(ret.get(), &dataStr[0], numBytesToRead); - err = ::read(fd, ret.get(), 100); - if (err < 0) - { - char buf[80]; - throw runtime_error("seekToEndOfHeader1 got: " + string(strerror_r(errno, buf, 80))); - } - for (int i = 0; i < err; i++) + for (uint32_t i = 0; i < numBytesToRead; i++) { if (ret[i] == 0) { - ::lseek(fd, i + 1, SEEK_SET); *_bytesRead = i + 1; return ret; } } - throw runtime_error("seekToEndOfHeader1: did not find the end of the header"); + throw runtime_error("seekToEndOfHeader1_: did not find the end of the header"); } int IOCoordinator::mergeJournal(int objFD, int journalFD, uint8_t* buf, off_t offset, size_t* len) const @@ -1195,7 +1189,7 @@ int IOCoordinator::mergeJournal(int objFD, int journalFD, uint8_t* buf, off_t of std::shared_ptr IOCoordinator::mergeJournal(const char* object, const char* journal, off_t offset, size_t len, size_t* _bytesReadOut) const { - int objFD, journalFD; + int objFD; std::shared_ptr ret; size_t l_bytesRead = 0; @@ -1208,7 +1202,6 @@ std::shared_ptr IOCoordinator::mergeJournal(const char* object, const ScopedCloser s1(objFD); ret.reset(new uint8_t[len]); - // read the object into memory size_t count = 0; if (offset != 0) @@ -1249,29 +1242,34 @@ std::shared_ptr IOCoordinator::mergeJournal(const char* object, const return ret; } - journalFD = ::open(journal, O_RDONLY); - if (journalFD < 0) + // Read journal. + auto keyGen = std::make_shared(); + FDBCS::BlobHandler journalReader(keyGen); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto resultPairJournal = journalReader.readBlob(kvStorage, journal); + if (!resultPairJournal.first) { *_bytesReadOut = l_bytesRead; return ret; } - ScopedCloser s2(journalFD); - std::shared_ptr headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); + const std::string& journalData = resultPairJournal.second; + size_t journalOffset = 0; + std::shared_ptr headertxt = seekToEndOfHeader1_(journalData, &journalOffset); stringstream ss; ss << headertxt.get(); boost::property_tree::ptree header; boost::property_tree::json_parser::read_json(ss, header); assert(header.get("version") == 1); + l_bytesRead += journalOffset; + const size_t journalSize = journalData.size(); // start processing the entries - while (1) + while (journalOffset < journalSize) { uint64_t offlen[2]; - int err = ::read(journalFD, &offlen, 16); - if (err == 0) // got EOF - break; - assert(err == 16); + std::memcpy(&offlen, &journalData[journalOffset], 16); + journalOffset += 16; l_bytesRead += 16; // if this entry overlaps, read the overlapping section @@ -1284,45 +1282,15 @@ std::shared_ptr IOCoordinator::mergeJournal(const char* object, const // seek to the portion of the entry to start reading at if (startReadingAt != offlen[0]) - ::lseek(journalFD, startReadingAt - offlen[0], SEEK_CUR); - - uint count = 0; - while (count < lengthOfRead) - { - err = ::read(journalFD, &ret[startReadingAt - offset + count], lengthOfRead - count); - if (err < 0) - { - int l_errno = errno; - char buf[80]; - logger->log(LOG_ERR, "mergeJournal: got %s", strerror_r(l_errno, buf, 80)); - ret.reset(); - errno = l_errno; - l_bytesRead += count; - goto out; - } - else if (err == 0) - { - logger->log(LOG_ERR, - "mergeJournal: got early EOF. offset=%ld, len=%ld, jOffset=%ld, jLen=%ld," - " startReadingAt=%ld, lengthOfRead=%ld", - offset, len, offlen[0], offlen[1], startReadingAt, lengthOfRead); - ret.reset(); - l_bytesRead += count; - goto out; - } - count += err; - } - l_bytesRead += lengthOfRead; - - // advance the file pos if we didn't read to the end of the entry - if (startReadingAt - offlen[0] + lengthOfRead != offlen[1]) - ::lseek(journalFD, offlen[1] - (lengthOfRead + startReadingAt - offlen[0]), SEEK_CUR); + journalOffset += startReadingAt - offlen[0]; + //::lseek(journalFD, startReadingAt - offlen[0], SEEK_CUR); + std::memcpy(&ret[startReadingAt - offset], &journalData[journalOffset], lengthOfRead); + journalOffset += lengthOfRead; } else // skip over this journal entry - ::lseek(journalFD, offlen[1], SEEK_CUR); + journalOffset += offlen[1]; } -out: *_bytesReadOut = l_bytesRead; return ret; } @@ -1332,19 +1300,18 @@ out: int IOCoordinator::mergeJournalInMem(std::shared_ptr& objData, size_t len, const char* journalPath, size_t* _bytesReadOut) const { - // if the journal is over some size threshold (100MB for now why not), - // use the original low-mem-usage version - if (len > (100 << 20)) - return mergeJournalInMem_bigJ(objData, len, journalPath, _bytesReadOut); - size_t l_bytesRead = 0; - int journalFD = ::open(journalPath, O_RDONLY); - if (journalFD < 0) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler blobReader(keyGen); + auto resultPair = blobReader.readBlob(kvStorage, journalPath); + if (!resultPair.first) return -1; - ScopedCloser s(journalFD); + const std::string& journalData = resultPair.second; + size_t journalOffset = 0; // grab the journal header and make sure the version is 1 - std::shared_ptr headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); + std::shared_ptr headertxt = seekToEndOfHeader1_(journalData, &l_bytesRead); stringstream ss; ss << headertxt.get(); boost::property_tree::ptree header; @@ -1352,136 +1319,46 @@ int IOCoordinator::mergeJournalInMem(std::shared_ptr& objData, size_t assert(header.get("version") == 1); // read the journal file into memory - size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead; - ::lseek(journalFD, l_bytesRead, SEEK_SET); - boost::scoped_array journalData(new uint8_t[journalBytes]); + size_t journalBytes = journalData.size() - l_bytesRead; + journalOffset += l_bytesRead; size_t readCount = 0; - while (readCount < journalBytes) - { - ssize_t err = ::read(journalFD, &journalData[readCount], journalBytes - readCount); - if (err < 0) - { - char buf[80]; - int l_errno = errno; - logger->log(LOG_ERR, "mergeJournalInMem: got %s", strerror_r(errno, buf, 80)); - errno = l_errno; - return -1; - } - else if (err == 0) - { - logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); - errno = ENODATA; // is there a better errno for early EOF? - return -1; - } - readCount += err; - l_bytesRead += err; - } + readCount += journalBytes; + l_bytesRead += journalBytes; + const size_t journalSize = journalData.size(); // start processing the entries - size_t offset = 0; - while (offset < journalBytes) + while (journalOffset < journalSize) { - if (offset + 16 >= journalBytes) + if (journalOffset + 16 >= journalSize) { logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); errno = ENODATA; // is there a better errno for early EOF? return -1; } - uint64_t* offlen = (uint64_t*)&journalData[offset]; - offset += 16; + uint64_t* offlen = (uint64_t*)&journalData[journalOffset]; + journalOffset += 16; uint64_t startReadingAt = offlen[0]; uint64_t lengthOfRead = offlen[1]; - - if (startReadingAt > len) - { - offset += offlen[1]; - continue; - } - - if (startReadingAt + lengthOfRead > len) - lengthOfRead = len - startReadingAt; - if (offset + lengthOfRead > journalBytes) - { - logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); - errno = ENODATA; // is there a better errno for early EOF? - return -1; - } - memcpy(&objData[startReadingAt], &journalData[offset], lengthOfRead); - offset += offlen[1]; - } - *_bytesReadOut = l_bytesRead; - return 0; -} - -int IOCoordinator::mergeJournalInMem_bigJ(std::shared_ptr& objData, size_t len, - const char* journalPath, size_t* _bytesReadOut) const -{ - size_t l_bytesRead = 0; - int journalFD = ::open(journalPath, O_RDONLY); - if (journalFD < 0) - return -1; - ScopedCloser s(journalFD); - - // grab the journal header and make sure the version is 1 - std::shared_ptr headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); - stringstream ss; - ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); - - // start processing the entries - while (1) - { - uint64_t offlen[2]; - int err = ::read(journalFD, &offlen, 16); - if (err == 0) // got EOF + if (lengthOfRead == 0) break; - else if (err < 16) - { - // punting on this - cout << "mergeJournalInMem: failed to read a journal entry header in one attempt. fixme..." << endl; - errno = ENODATA; - return -1; - } - l_bytesRead += 16; - - uint64_t startReadingAt = offlen[0]; - uint64_t lengthOfRead = offlen[1]; if (startReadingAt > len) { - ::lseek(journalFD, offlen[1], SEEK_CUR); + journalOffset += offlen[1]; continue; } if (startReadingAt + lengthOfRead > len) lengthOfRead = len - startReadingAt; - - uint count = 0; - while (count < lengthOfRead) + if (journalOffset + lengthOfRead > journalSize) { - err = ::read(journalFD, &objData[startReadingAt + count], lengthOfRead - count); - if (err < 0) - { - char buf[80]; - int l_errno = errno; - logger->log(LOG_ERR, "mergeJournalInMem: got %s", strerror_r(errno, buf, 80)); - errno = l_errno; - return -1; - } - else if (err == 0) - { - logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); - errno = ENODATA; // is there a better errno for early EOF? - return -1; - } - count += err; + logger->log(LOG_ERR, "mergeJournalInMem: got early EOF"); + errno = ENODATA; // is there a better errno for early EOF? + return -1; } - l_bytesRead += lengthOfRead; - if (lengthOfRead < offlen[1]) - ::lseek(journalFD, offlen[1] - lengthOfRead, SEEK_CUR); + std::memcpy(&objData[startReadingAt], &journalData[journalOffset], lengthOfRead); + journalOffset += offlen[1]; } *_bytesReadOut = l_bytesRead; return 0; diff --git a/storage-manager/src/IOCoordinator.h b/storage-manager/src/IOCoordinator.h index f499aad3e..6ad0ab5f6 100644 --- a/storage-manager/src/IOCoordinator.h +++ b/storage-manager/src/IOCoordinator.h @@ -37,7 +37,7 @@ namespace storagemanager { -std::shared_ptr seekToEndOfHeader1(int fd, size_t* bytesRead); +std::shared_ptr seekToEndOfHeader1_(const std::string& dataStr, size_t* bytesRead); class IOCoordinator : public boost::noncopyable { @@ -58,16 +58,12 @@ class IOCoordinator : public boost::noncopyable // The shared logic for merging a journal file with its base file. // len should be set to the length of the data requested std::shared_ptr mergeJournal(const char* objectPath, const char* journalPath, off_t offset, - size_t len, size_t* sizeRead) const; + size_t len, size_t* sizeRead) const; // this version modifies object data in memory, given the journal filename. Processes the whole object // and whole journal file. int mergeJournalInMem(std::shared_ptr& objData, size_t len, const char* journalPath, - size_t* sizeRead) const; - - // this version of MJIM has a higher IOPS requirement and lower mem usage. - int mergeJournalInMem_bigJ(std::shared_ptr& objData, size_t len, const char* journalPath, - size_t* sizeRead) const; + size_t* sizeRead) const; // this version takes already-open file descriptors, and an already-allocated buffer as input. // file descriptor are positioned, eh, best not to assume anything about their positions diff --git a/storage-manager/src/KVPrefixes.cpp b/storage-manager/src/KVPrefixes.cpp index 8183d9f69..0e5bc2986 100644 --- a/storage-manager/src/KVPrefixes.cpp +++ b/storage-manager/src/KVPrefixes.cpp @@ -15,10 +15,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "KVPrefixes.hpp" +#include "KVPrefixes.h" namespace storagemanager { // FDB recommends keep the key size up to 32 bytes. -const char* KVPrefixes[2] = {/*OWNERSHIP*/ "SM_O_", /*META*/ "SM_M_"}; +const char* KVPrefixes[3] = {/*OWNERSHIP*/ "SM_O_", /*META*/ "SM_M_", /*JOURNAL*/ "SM_J_"}; + +std::string getJournalName(const std::string& key) +{ + return KVPrefixes[static_cast(KVPrefixId::SM_JOURNAL)] + key; +} } // namespace storagemanager diff --git a/storage-manager/src/KVPrefixes.hpp b/storage-manager/src/KVPrefixes.h similarity index 88% rename from storage-manager/src/KVPrefixes.hpp rename to storage-manager/src/KVPrefixes.h index 92d8b191c..34ad4ce96 100644 --- a/storage-manager/src/KVPrefixes.hpp +++ b/storage-manager/src/KVPrefixes.h @@ -16,14 +16,19 @@ MA 02110-1301, USA. */ #pragma once +#include +#include namespace storagemanager { enum class KVPrefixId { SM_OWNERSHIP = 0, - SM_META + SM_META, + SM_JOURNAL }; extern const char* KVPrefixes[]; + +std::string getJournalName(const std::string &key); } // namespace storagemanager diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index 8cf964c69..7070deac5 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -45,7 +45,7 @@ #include #include #include "fdbcs.hpp" -#include "KVPrefixes.hpp" +#include "KVPrefixes.h" #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) diff --git a/storage-manager/src/Ownership.cpp b/storage-manager/src/Ownership.cpp index 35c8a19b4..972f92260 100644 --- a/storage-manager/src/Ownership.cpp +++ b/storage-manager/src/Ownership.cpp @@ -20,7 +20,7 @@ #include "Cache.h" #include "Synchronizer.h" #include "KVStorageInitializer.h" -#include "KVPrefixes.hpp" +#include "KVPrefixes.h" #include "fdbcs.hpp" #include #include diff --git a/storage-manager/src/PrefixCache.cpp b/storage-manager/src/PrefixCache.cpp index 220346aed..584b10d65 100644 --- a/storage-manager/src/PrefixCache.cpp +++ b/storage-manager/src/PrefixCache.cpp @@ -17,6 +17,8 @@ #include "PrefixCache.h" #include "Cache.h" +#include "KVPrefixes.h" +#include "KVStorageInitializer.h" #include "Config.h" #include "Downloader.h" #include "Synchronizer.h" @@ -575,7 +577,8 @@ void PrefixCache::rename(const string& oldKey, const string& newKey, ssize_t siz int PrefixCache::ifExistsThenDelete(const string& key) { bf::path cachedPath = cachePrefix / key; - bf::path journalPath = journalPrefix / (key + ".journal"); + const auto journalName = getJournalName((journalPrefix / (key + ".journal")).string()); + const auto journalSizeName = getJournalName((journalPrefix / (key + "_size.journal")).string()); boost::unique_lock s(lru_mutex); bool objectExists = false; @@ -593,16 +596,18 @@ int PrefixCache::ifExistsThenDelete(const string& key) else // let makeSpace() delete it if it's already in progress return 0; } - bool journalExists = bf::exists(journalPath); - // assert(objectExists == bf::exists(cachedPath)); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto journalResult = tnx->get(journalName); + auto journalSizeResult = tnx->get(journalSizeName); + bool journalExists = journalSizeResult.first; size_t objectSize = (objectExists ? bf::file_size(cachedPath) : 0); - // size_t objectSize = (objectExists ? MetadataFile::getLengthFromKey(key) : 0); - size_t journalSize = (journalExists ? bf::file_size(journalPath) : 0); + size_t journalSize = 0; + if (journalExists) + journalSize = std::atoi(journalSizeResult.second.c_str()); + currentCacheSize -= (objectSize + journalSize); - - // assert(!objectExists || objectSize == bf::file_size(cachedPath)); - return (objectExists ? 1 : 0) | (journalExists ? 2 : 0); } diff --git a/storage-manager/src/Replicator.cpp b/storage-manager/src/Replicator.cpp index c4778a990..e77b7e105 100644 --- a/storage-manager/src/Replicator.cpp +++ b/storage-manager/src/Replicator.cpp @@ -20,6 +20,9 @@ #include "SMLogging.h" #include "Utilities.h" #include "Cache.h" +#include "KVStorageInitializer.h" +#include "KVPrefixes.h" +#include "fdbcs.hpp" #include #include #include @@ -121,9 +124,8 @@ void Replicator::printKPIs() const int Replicator::newObject(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, size_t length) { + const string objectFilename = msCachePath + "/" + filename.string(); int fd, err; - string objectFilename = msCachePath + "/" + filename.string(); - OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT); size_t count = 0; while (count < length) @@ -215,54 +217,48 @@ ssize_t Replicator::_write(int fd, const void* data, size_t length) Benefits would be data integrity, and possibly add'l parallelism. The downside is of course, a higher number of IO ops for the same operation. */ + int Replicator::addJournalEntry(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, size_t length) { - int fd, err; uint64_t offlen[] = {(uint64_t)offset, length}; - size_t count = 0; - int version = 1; - int l_errno; - char errbuf[80]; - bool bHeaderChanged = false; - string headerRollback = ""; - string journalFilename = msJournalPath + "/" + filename.string() + ".journal"; + const int version = 1; + const auto journalName = getJournalName(msJournalPath + "/" + filename.string() + ".journal"); + const auto journalSizeName = getJournalName(msJournalPath + "/" + filename.string() + "_size" + ".journal"); boost::filesystem::path firstDir = *((filename).begin()); - uint64_t thisEntryMaxOffset = (offset + length - 1); + const uint64_t thisEntryMaxOffset = (offset + length - 1); + string dataStr; + size_t dataStrOffset = 0; - uint64_t currentMaxOffset = 0; - bool exists = boost::filesystem::exists(journalFilename); - OPEN(journalFilename.c_str(), (exists ? O_RDWR : O_WRONLY | O_CREAT)) - - if (!exists) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler journalHandler(keyGen); + auto resultPair = journalHandler.readBlob(kvStorage, journalName); + const std::string& journalData = resultPair.second; + const bool journalExists = resultPair.first; + if (!journalExists) { - bHeaderChanged = true; // create new journal file with header string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % thisEntryMaxOffset) .str(); - err = _write(fd, header.c_str(), header.length() + 1); - l_errno = errno; - repHeaderDataWritten += (header.length() + 1); - if ((uint)err != (header.length() + 1)) - { - // return the error because the header for this entry on a new journal file failed - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Writing journal header failed (%s).", - strerror_r(l_errno, errbuf, 80)); - errno = l_errno; - return err; - } - Cache::get()->newJournalEntry(firstDir, header.length() + 1); + const size_t headerLength = header.length(); + dataStr.resize(headerLength + 1 + JOURNAL_ENTRY_HEADER_SIZE + length); + std::memcpy(&dataStr[dataStrOffset], header.c_str(), headerLength); + // Specifies the end of the header. + dataStr[headerLength] = 0; + dataStrOffset = headerLength + 1; + repHeaderDataWritten += headerLength + 1; + Cache::get()->newJournalEntry(firstDir, headerLength + 1); ++replicatorJournalsCreated; } else { - // read the existing header and check if max_offset needs to be updated size_t tmp; std::shared_ptr headertxt; try { - headertxt = seekToEndOfHeader1(fd, &tmp); + headertxt = seekToEndOfHeader1_(journalData, &tmp); } catch (std::runtime_error& e) { @@ -278,7 +274,6 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u } stringstream ss; ss << headertxt.get(); - headerRollback = headertxt.get(); boost::property_tree::ptree header; try { @@ -297,164 +292,61 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u return -1; } assert(header.get("version") == 1); - uint64_t currentMaxOffset = header.get("max_offset"); + const uint64_t currentMaxOffset = header.get("max_offset"); + dataStr.resize(journalData.size() + JOURNAL_ENTRY_HEADER_SIZE + length); + size_t journalOffset = 0; + if (thisEntryMaxOffset > currentMaxOffset) { - bHeaderChanged = true; string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % thisEntryMaxOffset) .str(); - err = _pwrite(fd, header.c_str(), header.length() + 1, 0); - l_errno = errno; - repHeaderDataWritten += (header.length() + 1); - if ((uint)err != (header.length() + 1)) - { - // only the header was possibly changed rollback attempt - mpLogger->log(LOG_CRIT, - "Replicator::addJournalEntry: Updating journal header failed. " - "Attempting to rollback and continue."); - int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1, 0); - if ((uint)rollbackErr == (headerRollback.length() + 1)) - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header success."); - else - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed!"); - errno = l_errno; - if (err < 0) - return err; - else - return 0; - } + const size_t headerLenght = header.length(); + std::memcpy(&dataStr[0], header.c_str(), headerLenght); + dataStr[headerLenght] = 0; + dataStrOffset = headerLenght + 1; + journalOffset = headerLenght + 1; + repHeaderDataWritten += headerLenght + 1; } + + std::memcpy(&dataStr[dataStrOffset], &journalData[journalOffset], journalData.size() - journalOffset); + dataStrOffset = journalData.size(); } - off_t entryHeaderOffset = ::lseek(fd, 0, SEEK_END); - - err = _write(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE); - l_errno = errno; + std::memcpy(&dataStr[dataStrOffset], offlen, JOURNAL_ENTRY_HEADER_SIZE); + dataStrOffset += JOURNAL_ENTRY_HEADER_SIZE; repHeaderDataWritten += JOURNAL_ENTRY_HEADER_SIZE; - if (err != JOURNAL_ENTRY_HEADER_SIZE) + std::memcpy(&dataStr[dataStrOffset], data, length); + dataStrOffset += length; + assert(dataStr.size() == dataStrOffset); + + if (journalExists && !journalHandler.removeBlob(kvStorage, journalName)) { - // this entry failed so if the header was updated roll it back - if (bHeaderChanged) - { - mpLogger->log(LOG_CRIT, - "Replicator::addJournalEntry: write journal entry header failed. Attempting to rollback " - "and continue."); - // attempt to rollback top level header - int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1, 0); - if ((uint)rollbackErr != (headerRollback.length() + 1)) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed! (%s)", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - if (err < 0) - return err; - else - return 0; - } - } - int rollbackErr = ::ftruncate(fd, entryHeaderOffset); - if (rollbackErr != 0) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Truncate to previous EOF failed! (%s)", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - if (err < 0) - return err; - else - return 0; - } - l_errno = errno; - return err; - } - while (count < length) - { - err = _write(fd, &data[count], length - count); - if (err < 0) - { - l_errno = errno; - /* XXXBEN: Attempt to update entry header with the partial write and write it. - IF the write fails to update entry header report an error to IOC and logging. - */ - if (count > 0) // return what was successfully written - { - mpLogger->log(LOG_CRIT, - "Replicator::addJournalEntry: Got '%s' writing a journal entry. " - "Attempting to update and continue.", - strerror_r(l_errno, errbuf, 80)); - // Update the file header max_offset if necessary and possible - thisEntryMaxOffset = (offset + count - 1); - if (thisEntryMaxOffset > currentMaxOffset) - { - string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % - thisEntryMaxOffset) - .str(); - int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1, 0); - if ((uint)rollbackErr != (header.length() + 1)) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal header failed! (%s)", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - return err; - } - } - // Update the journal entry header - offlen[1] = count; - int rollbackErr = _pwrite(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE, entryHeaderOffset); - if ((uint)rollbackErr != JOURNAL_ENTRY_HEADER_SIZE) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal entry header failed! (%s)", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - return err; - } - // return back what we did write - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Partial write success."); - repUserDataWritten += count; - return count; - } - else - { - // If the header was changed rollback and reset EOF - // Like this never happened - // Currently since this returns the err from the first write. IOC returns -1 and writeTask returns an - // error So system is likely broken in some way - if (bHeaderChanged) - { - mpLogger->log(LOG_CRIT, - "Replicator::addJournalEntry: write journal entry failed (%s). " - "Attempting to rollback and continue.", - strerror_r(l_errno, errbuf, 80)); - // attempt to rollback top level header - string header = - (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % 0).str(); - int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1, 0); - if ((uint)rollbackErr != (header.length() + 1)) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed (%s)!", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - return err; - } - } - int rollbackErr = ::ftruncate(fd, entryHeaderOffset); - if (rollbackErr != 0) - { - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Remove entry header failed (%s)!", - strerror_r(errno, errbuf, 80)); - errno = l_errno; - return err; - } - mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Write failed. Journal file restored."); - errno = l_errno; - return err; - } - } - count += err; + mpLogger->log(LOG_CRIT, "Cannot remove journal blob."); + errno = EIO; + return -1; } - repUserDataWritten += count; - return count; + if (!journalHandler.writeBlob(kvStorage, journalName, dataStr)) + { + mpLogger->log(LOG_CRIT, "Cannot write journal blob."); + errno = EIO; + return -1; + } + + { + auto tnx = kvStorage->createTransaction(); + tnx->set(journalSizeName, std::to_string(dataStr.size())); + if (!tnx->commit()) + { + mpLogger->log(LOG_CRIT, "Cannot write journal size."); + errno = EIO; + return -1; + } + } + + repUserDataWritten += length; + return length; } int Replicator::remove(const boost::filesystem::path& filename, Flags flags) @@ -484,6 +376,27 @@ int Replicator::remove(const boost::filesystem::path& filename, Flags flags) return ret; } +int Replicator::removeJournal(const boost::filesystem::path& filename) +{ + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler journalHandler(keyGen); + if (!journalHandler.removeBlob(kvStorage, filename.string())) + { + return -1; + } + return 0; +} + +int Replicator::removeJournalSize(const boost::filesystem::path& filename) +{ + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + tnx->remove(filename.string()); + return tnx->commit(); +} + + int Replicator::updateMetadata(MetadataFile& meta) { return meta.writeMetadata(); diff --git a/storage-manager/src/Replicator.h b/storage-manager/src/Replicator.h index 165c60e45..bc237b308 100644 --- a/storage-manager/src/Replicator.h +++ b/storage-manager/src/Replicator.h @@ -43,13 +43,14 @@ class Replicator LOCAL_ONLY = 0x1, NO_LOCAL = 0x2 }; - int addJournalEntry(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, size_t length); int newObject(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, size_t length); int newNullObject(const boost::filesystem::path& filename, size_t length); int remove(const boost::filesystem::path& file, Flags flags = NONE); + int removeJournal(const boost::filesystem::path &file); + int removeJournalSize(const boost::filesystem::path &file); int updateMetadata(MetadataFile& meta); diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index 62315d990..99a7ff0fb 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -20,6 +20,8 @@ #include "IOCoordinator.h" #include "MetadataFile.h" #include "Utilities.h" +#include "KVStorageInitializer.h" +#include "KVPrefixes.h" #include #include @@ -238,8 +240,12 @@ void Synchronizer::flushObject(const bf::path& prefix, const string& _key) sleep(5); } } while (err); - journalExists = bf::exists(journalPath / (key + ".journal")); + const auto jounralName = getJournalName((journalPath / (key + ".journal")).string()); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto resultPair = tnx->get(jounralName); + journalExists = resultPair.first; if (journalExists) { logger->log(LOG_DEBUG, @@ -292,8 +298,8 @@ void Synchronizer::periodicSync() else ++flushesTriggeredByTimer; } - // cout << "Sync'ing " << pendingOps.size() << " objects" << " queue size is " << - // threadPool.currentQueueSize() << endl; + // cout << "Sync'ing " << pendingOps.size() << " objects" + // << " queue size is " << threadPool->currentQueueSize() << endl; for (auto& job : pendingOps) makeJob(job.first); for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) @@ -575,11 +581,18 @@ void Synchronizer::synchronizeWithJournal(const string& sourceFile, list cache->deletedObject(prefix, cloudKey, objSize); cs->deleteObject(cloudKey); } - bf::path jPath = journalPath / (key + ".journal"); - if (bf::exists(jPath)) + + const auto journalName = getJournalName((journalPath / (key + ".journal")).string()); + const auto journalSizeName = getJournalName((journalPath / (key + "_size.journal")).string()); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto resultPairJournal = tnx->get(journalName); + auto resultPairJournalSize = tnx->get(journalName); + if (resultPairJournal.first && resultPairJournalSize.first) { - size_t jSize = bf::file_size(jPath); - replicator->remove(jPath); + size_t jSize = std::atoi(resultPairJournalSize.second.c_str()); + replicator->removeJournal(journalName); + replicator->removeJournalSize(journalSizeName); cache->deletedJournal(prefix, jSize); } } @@ -605,43 +618,48 @@ void Synchronizer::synchronizeWithJournal(const string& sourceFile, list // sync queue bf::path oldCachePath = cachePath / key; - string journalName = (journalPath / (key + ".journal")).string(); - - if (!bf::exists(journalName)) + const string journalName = getJournalName((journalPath / (key + ".journal")).string()); + const auto journalSizeName = getJournalName((journalPath / (key + "_size.journal")).string()); { - logger->log(LOG_DEBUG, "synchronizeWithJournal(): no journal file found for %s", key.c_str()); - - // sanity check + add'l info. Test whether the object exists in cloud storage. If so, complain, - // and run synchronize() instead. - bool existsOnCloud; - int err = cs->exists(cloudKey, &existsOnCloud); - if (err) - throw runtime_error(string("Synchronizer: cs->exists() failed: ") + strerror_r(errno, buf, 80)); - if (!existsOnCloud) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto resultPair = tnx->get(journalName); + if (!resultPair.first) { - if (cache->exists(prefix, cloudKey)) + logger->log(LOG_DEBUG, "synchronizeWithJournal(): no journal file found for %s", key.c_str()); + + // sanity check + add'l info. Test whether the object exists in cloud storage. If so, complain, + // and run synchronize() instead. + bool existsOnCloud; + int err = cs->exists(cloudKey, &existsOnCloud); + if (err) + throw runtime_error(string("Synchronizer: cs->exists() failed: ") + strerror_r(errno, buf, 80)); + if (!existsOnCloud) { - logger->log(LOG_DEBUG, - "synchronizeWithJournal(): %s has no journal and does not exist in the cloud, calling " - "synchronize() instead. Need to explain how this happens.", - key.c_str()); - s.unlock(); - synchronize(sourceFile, lit); + if (cache->exists(prefix, cloudKey)) + { + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): %s has no journal and does not exist in the cloud, calling " + "synchronize() instead. Need to explain how this happens.", + key.c_str()); + s.unlock(); + synchronize(sourceFile, lit); + } + else + logger->log(LOG_DEBUG, + "synchronizeWithJournal(): %s has no journal, and does not exist in the cloud or in " + " the local cache. Need to explain how this happens.", + key.c_str()); + return; } else logger->log(LOG_DEBUG, - "synchronizeWithJournal(): %s has no journal, and does not exist in the cloud or in " - " the local cache. Need to explain how this happens.", + "synchronizeWithJournal(): %s has no journal, but it does exist in the cloud. " + " This indicates that an overlapping syncWithJournal() call handled it properly.", key.c_str()); + return; } - else - logger->log(LOG_DEBUG, - "synchronizeWithJournal(): %s has no journal, but it does exist in the cloud. " - " This indicates that an overlapping syncWithJournal() call handled it properly.", - key.c_str()); - - return; } int err; @@ -649,7 +667,6 @@ void Synchronizer::synchronizeWithJournal(const string& sourceFile, list size_t count = 0, size = mdEntry.length, originalSize = 0; bool oldObjIsCached = cache->exists(prefix, cloudKey); - // get the base object if it is not already cached // merge it with its journal file if (!oldObjIsCached) @@ -685,7 +702,10 @@ void Synchronizer::synchronizeWithJournal(const string& sourceFile, list err = ioc->mergeJournalInMem(data, size, journalName.c_str(), &_bytesRead); if (err) { - if (!bf::exists(journalName)) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto resultPair = tnx->get(journalName); + if (!resultPair.first) logger->log(LOG_DEBUG, "synchronizeWithJournal(): journal %s was deleted mid-operation, check locking", journalName.c_str()); @@ -704,7 +724,10 @@ void Synchronizer::synchronizeWithJournal(const string& sourceFile, list data = ioc->mergeJournal(oldCachePath.string().c_str(), journalName.c_str(), 0, size, &_bytesRead); if (!data) { - if (!bf::exists(journalName)) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto resultPair = tnx->get(journalName); + if (!resultPair.first) logger->log(LOG_DEBUG, "synchronizeWithJournal(): journal %s was deleted mid-operation, check locking", journalName.c_str()); @@ -790,8 +813,17 @@ void Synchronizer::synchronizeWithJournal(const string& sourceFile, list rename(key, newKey); // delete the old object & journal file - cache->deletedJournal(prefix, bf::file_size(journalName)); - replicator->remove(journalName); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + { + auto tnx = kvStorage->createTransaction(); + auto result = tnx->get(journalSizeName); + if (result.first) + cache->deletedJournal(prefix, std::atoi(result.second.c_str())); + } + + // Remove journal size name from kv storage. + replicator->removeJournalSize(journalSizeName); + replicator->removeJournal(journalName); cs->deleteObject(cloudKey); } @@ -887,7 +919,7 @@ void Synchronizer::configListener() { maxUploads = 20; threadPool->setMaxThreads(maxUploads); - logger->log(LOG_INFO, "max_concurrent_uploads = %u",maxUploads); + logger->log(LOG_INFO, "max_concurrent_uploads = %u", maxUploads); } if (stmp.empty()) { From dd8fac35ae8bab1835b349a1dc0db367117cfe3c Mon Sep 17 00:00:00 2001 From: drrtuy Date: Thu, 26 Dec 2024 16:26:34 +0000 Subject: [PATCH 34/65] fix(syscat): MCOL-5816 23.02 -> 23.10 upgrade issues (#3346) (#3374) * feat(BRM,tools): couple utilities to watch/operate shared memory locks and extent map * feat(BRM,tools): merged two utilities and added some extra dbbuilder output in case of upgrade * fix(dbbuilder): extra output to log upgrade detection. --- CMakeLists.txt | 1 + cmake/CLI11.cmake | 15 ++ debian/mariadb-plugin-columnstore.install | 3 + tools/dbbuilder/dbbuilder.cpp | 85 +++----- versioning/BRM/CMakeLists.txt | 12 ++ versioning/BRM/extentmap.cpp | 72 ++++--- versioning/BRM/extentmap.h | 10 + versioning/BRM/load_brm_from_file.cpp | 248 ++++++++-------------- versioning/BRM/lock_grabber.cpp | 118 ---------- versioning/BRM/lock_state.cpp | 91 -------- versioning/BRM/shmem_locks.cpp | 128 +++++++++++ versioning/BRM/tablelockserver.cpp | 2 +- 12 files changed, 339 insertions(+), 446 deletions(-) create mode 100644 cmake/CLI11.cmake delete mode 100644 versioning/BRM/lock_grabber.cpp delete mode 100644 versioning/BRM/lock_state.cpp create mode 100644 versioning/BRM/shmem_locks.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f8cd0737..e26483967 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,7 @@ SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE(columnstore_version) INCLUDE(misc) INCLUDE(boost) +INCLUDE(CLI11) INCLUDE(thrift) INCLUDE(arrow) diff --git a/cmake/CLI11.cmake b/cmake/CLI11.cmake new file mode 100644 index 000000000..d86e9f99c --- /dev/null +++ b/cmake/CLI11.cmake @@ -0,0 +1,15 @@ +set(EXTERNAL_INCLUDE_DIR "${CMAKE_BINARY_DIR}/external/include") +file(MAKE_DIRECTORY "${EXTERNAL_INCLUDE_DIR}") + +set(C11CLI_URL "https://github.com/CLIUtils/CLI11/releases/download/v2.4.2/CLI11.hpp") +set(C11CLI_HEADER "${EXTERNAL_INCLUDE_DIR}/CLI11.hpp") +set(CLI11_INCLUDE_DIR "${EXTERNAL_INCLUDE_DIR}") + +file(DOWNLOAD + ${C11CLI_URL} + ${C11CLI_HEADER} + SHOW_PROGRESS STATUS download_status +) + +add_library(CLI11 INTERFACE) +target_include_directories(CLI11 INTERFACE ${CLI11_INCLUDE_DIR}) \ No newline at end of file diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index 17539eaa9..6f0ca724b 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -37,6 +37,9 @@ usr/bin/mcs-loadbrm.py usr/bin/mcs_parquet_ddl usr/bin/mcs_parquet_gen usr/bin/mcs-stop-controllernode.sh +usr/bin/mcs-load-brm-from-file +usr/bin/mcs-load-em +usr/bin/mcs-shmem-locks usr/bin/mcsGetConfig usr/bin/mcsSetConfig usr/bin/mycnfUpgrade diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index 33c3323e3..9958d2eef 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -84,12 +84,12 @@ void usage() const unsigned sysCatalogErr = logging::M0060; -void errorHandler(const unsigned mid, const string& src, const string& msg, bool isCritErr = true) +void messageHandler(const string& src, const string& msg, bool isCritErr = true) { logging::LoggingID lid(19); logging::MessageLog ml(lid); logging::Message::Args args; - logging::Message message(mid); + logging::Message message(sysCatalogErr); if (isCritErr) { @@ -143,28 +143,6 @@ int main(int argc, char* argv[]) return 1; } - oamModuleInfo_t t; - bool parentOAMModuleFlag = false; - - // get local module info; validate running on Active Parent OAM Module - try - { - t = oam.getModuleInfo(); - parentOAMModuleFlag = boost::get<4>(t); - } - catch (exception&) - { - parentOAMModuleFlag = true; - } - - if (!parentOAMModuleFlag) - { - cerr << "Exiting, dbbuilder can only be run on the Active " - "Parent OAM Module" - << endl; - return 1; - } - logFile = string(MCSLOGDIR) + "/install/dbbuilder.status"; buildOption = atoi(argv[optind++]); @@ -193,24 +171,21 @@ int main(int argc, char* argv[]) bool isUpgrade = false; std::unordered_map> upgradeOidMap; - upgradeOidMap[OID_SYSTABLE_AUXCOLUMNOID] = // This is the candidate OID for the upgrade. - std::make_pair(OID_SYSTABLE_OBJECTID, false); // std::pair::first is the reference OID used - // to fill the candidate OID with default vals - // std::pair::second is set to false by default - // which means the candidate OID will not be - // upgraded. This is set to true in the code - // below if a specific condition is met. Please - // note that the candidate and reference OID - // datatypes and colwidths are assumed to be the - // same in SystemCatalog::upgrade(). - upgradeOidMap[OID_SYSCOLUMN_CHARSETNUM] = - std::make_pair(OID_SYSCOLUMN_OBJECTID, false); + upgradeOidMap[OID_SYSTABLE_AUXCOLUMNOID] = // This is the candidate OID for the upgrade. + std::make_pair(OID_SYSTABLE_OBJECTID, false); // std::pair::first is the reference OID used + // to fill the candidate OID with default vals + // std::pair::second is set to false by default + // which means the candidate OID will not be + // upgraded. This is set to true in the code + // below if a specific condition is met. Please + // note that the candidate and reference OID + // datatypes and colwidths are assumed to be the + // same in SystemCatalog::upgrade(). + upgradeOidMap[OID_SYSCOLUMN_CHARSETNUM] = std::make_pair(OID_SYSCOLUMN_OBJECTID, false); std::unordered_map upgradeOidTypeMap; - upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] = - std::make_pair(CalpontSystemCatalog::INT, 4); - upgradeOidTypeMap[OID_SYSCOLUMN_CHARSETNUM] = - std::make_pair(CalpontSystemCatalog::INT, 4); + upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] = std::make_pair(CalpontSystemCatalog::INT, 4); + upgradeOidTypeMap[OID_SYSCOLUMN_CHARSETNUM] = std::make_pair(CalpontSystemCatalog::INT, 4); std::unordered_map upgradeOidDefaultValStrMap; upgradeOidDefaultValStrMap[OID_SYSTABLE_AUXCOLUMNOID] = "0"; @@ -227,11 +202,12 @@ int main(int argc, char* argv[]) (iter->second).second = true; isUpgrade = true; } + messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false); } if (!isUpgrade) { - string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; + string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; if (canWrite) { @@ -242,10 +218,10 @@ int main(int argc, char* argv[]) cerr << cmd << endl; } - errorHandler(sysCatalogErr, "Build system catalog", - "System catalog appears to exist. It will remain intact " - "for reuse. The database is not recreated.", - false); + messageHandler("Build system catalog", + "System catalog appears to exist. It will remain intact " + "for reuse. The database is not recreated.", + false); return 1; } } @@ -276,7 +252,7 @@ int main(int argc, char* argv[]) else cerr << cmd << endl; - errorHandler(sysCatalogErr, "Build system catalog", ex.what(), false); + messageHandler("Build system catalog", ex.what(), false); return 1; } catch (...) @@ -288,7 +264,7 @@ int main(int argc, char* argv[]) else cerr << cmd << endl; - errorHandler(sysCatalogErr, "Build system catalog", "HDFS check failed.", false); + messageHandler("Build system catalog", "HDFS check failed.", false); return 1; } @@ -305,6 +281,15 @@ int main(int argc, char* argv[]) } else { + string cmd = "echo 'Upgrade system catalog' > " + logFile; + if (canWrite) + { + rc = system(cmd.c_str()); + } + else + { + cerr << cmd << endl; + } sysCatalog.upgrade(upgradeOidMap, upgradeOidTypeMap, upgradeOidDefaultValStrMap); } @@ -326,7 +311,7 @@ int main(int argc, char* argv[]) ostringstream os; os << "Warning: running " << cmd << " failed. This is usually non-fatal."; cerr << os.str() << endl; - errorHandler(sysCatalogErr, "Save BRM", os.str()); + messageHandler("Save BRM", os.str()); } } else @@ -343,7 +328,7 @@ int main(int argc, char* argv[]) else cerr << cmd << endl; - errorHandler(sysCatalogErr, "Build system catalog", ex.what()); + messageHandler("Build system catalog", ex.what()); } catch (...) { @@ -355,7 +340,7 @@ int main(int argc, char* argv[]) cerr << cmd << endl; string err("Caught unknown exception!"); - errorHandler(sysCatalogErr, "Build system catalog", err); + messageHandler("Build system catalog", err); } } else diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 2b4b74db5..203360bad 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -116,3 +116,15 @@ add_executable(load_brm ${load_brm_SRCS}) target_link_libraries(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) install(TARGETS load_brm DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) + +add_executable(mcs-load-em load_em.cpp) +target_link_libraries(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +install(TARGETS mcs-load-em DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) + +add_executable(mcs-load-brm-from-file load_brm_from_file.cpp) +target_link_libraries(mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11) +install(TARGETS mcs-load-brm-from-file DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) + +add_executable(mcs-shmem-locks shmem_locks.cpp) +target_link_libraries(mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11) +install(TARGETS mcs-shmem-locks DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) \ No newline at end of file diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 11e72f0ef..b1e3e9890 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -61,17 +61,10 @@ #include "configcpp.h" #endif -#define EXTENTMAP_DLLEXPORT #include "extentmap.h" -#undef EXTENTMAP_DLLEXPORT #define EM_MAX_SEQNUM 2000000000 #define MAX_IO_RETRIES 10 -#define EM_MAGIC_V1 0x76f78b1c -#define EM_MAGIC_V2 0x76f78b1d -#define EM_MAGIC_V3 0x76f78b1e -#define EM_MAGIC_V4 0x76f78b1f -#define EM_MAGIC_V5 0x76f78b20 #ifndef NDEBUG #define ASSERT(x) \ @@ -122,6 +115,12 @@ EMCasualPartition_struct::EMCasualPartition_struct() isValid = CP_INVALID; } +EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum, + const char status) + : sequenceNum(seqNum), isValid(status), loVal(lo), hiVal(hi) +{ +} + EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum) { loVal = lo; @@ -172,6 +171,22 @@ EMEntry::EMEntry() status = 0; } +EMEntry::EMEntry(const InlineLBIDRange& range, int fileID, uint32_t blockOffset, HWM_t hwm, + PartitionNumberT partitionNum, uint16_t segmentNum, DBRootT dbRoot, uint16_t colWid, + int16_t status, EMPartition_t partition) + : range(range) + , fileID(fileID) + , blockOffset(blockOffset) + , HWM(hwm) + , partitionNum(partitionNum) + , segmentNum(segmentNum) + , dbRoot(dbRoot) + , colWid(colWid) + , status(status) + , partition(partition) +{ +} + EMEntry::EMEntry(const EMEntry& e) { range.start = e.range.start; @@ -1682,17 +1697,17 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) fEMRBTreeShminfo->currentSize = (emNumElements * EM_RB_TREE_NODE_SIZE) + EM_RB_TREE_EMPTY_SIZE; -#ifdef DUMP_EXTENT_MAP cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl; - for (const auto& lbidEMEntryPair : *fExtentMapRBTRee) + // for (const auto& lbidEMEntryPair : *fExtentMapRBTRee) + for (auto& lbidEMEntryPair : *fExtentMapRBTree) { const EMEntry& emEntry = lbidEMEntryPair.second; - cout << emEntry.start << '\t' << emEntry.size << '\t' << emEntry.fileID << '\t' << emEntry.blockOffset - << '\t' << emEntry.HWM << '\t' << emEntry.partitionNum << '\t' << emEntry.segmentNum << '\t' - << emEntry.dbRoot << '\t' << emEntry.status << '\t' << emEntry.partition.cprange.hiVal << '\t' - << emEntry.partition.cprange.loVal << '\t' << emEntry.partition.cprange.sequenceNum << '\t' - << (int)(emEntry.partition.cprange.isValid) << endl; + cout << emEntry.range.start << '\t' << emEntry.range.size << '\t' << emEntry.fileID << '\t' + << emEntry.blockOffset << '\t' << emEntry.HWM << '\t' << emEntry.partitionNum << '\t' + << emEntry.segmentNum << '\t' << emEntry.dbRoot << '\t' << emEntry.status << '\t' + << emEntry.partition.cprange.hiVal << '\t' << emEntry.partition.cprange.loVal << '\t' + << emEntry.partition.cprange.sequenceNum << '\t' << (int)(emEntry.partition.cprange.isValid) << endl; } cout << "Free list entries:" << endl; @@ -1700,8 +1715,6 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) for (uint32_t i = 0; i < flNumElements; i++) cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl; - -#endif } void ExtentMap::load(const string& filename, bool fixFL) @@ -2745,7 +2758,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co highestSegNum = emEntry.segmentNum; highestOffset = emEntry.blockOffset; } // found extentmap entry for specified OID - } // Loop through extent map entries + } // Loop through extent map entries DBRootVec dbRootVec(getAllDbRoots()); // 2. for empty DBRoot track hi seg# in user specified part# @@ -2832,7 +2845,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co } } } // loop over em idents - } // current dbroot == target dbroot + } // current dbroot == target dbroot else { // 4. Track hi seg for hwm+1 partition @@ -2853,8 +2866,8 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co partHighSeg = emEntry.segmentNum; } } // current dbroot != target dbroot - } // loop over dbroots - } // (lastExtentIndex >= 0) + } // loop over dbroots + } // (lastExtentIndex >= 0) //-------------------------------------------------------------------------- // Third Step: Select partition and segment number for new extent @@ -3038,7 +3051,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co newBlockOffset = lastExtent->blockOffset; } } - } // lastExtentIndex >= 0 + } // lastExtentIndex >= 0 else // Empty DBRoot; use part# that the user specifies { if (bHighEmptySegNumSet) @@ -3245,8 +3258,7 @@ void ExtentMap::logAndSetEMIndexReadOnly(const std::string& funcName) { fPExtMapIndexImpl_->makeReadOnly(); ostringstream os; - os << "ExtentMap::" << funcName << ": " - << "Can not update EM Index. EM Index shmem segment is set to" + os << "ExtentMap::" << funcName << ": " << "Can not update EM Index. EM Index shmem segment is set to" << " readonly. Please restart Columnstore."; log(os.str(), logging::LOG_TYPE_CRITICAL); @@ -4123,7 +4135,7 @@ void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) emIt = deleteExtent(emIt); } } // em iterarors loop - } // em info map loop + } // em info map loop } else { @@ -4193,7 +4205,7 @@ void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) } } } // em iterarors loop - } // em info map loop + } // em info map loop } } //------------------------------------------------------------------------------ @@ -4613,11 +4625,11 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v& if (emDbRoot.status == EXTENTUNAVAILABLE) { ostringstream oss; - oss << "ExtentMap::getDbRootHWMInfo(): " - << "OID " << OID << " has HWM extent that is UNAVAILABLE for " - << "DBRoot" << emDbRoot.dbRoot << "; part#: " << emDbRoot.partitionNum - << ", seg#: " << emDbRoot.segmentNum << ", fbo: " << emDbRoot.fbo - << ", localHWM: " << emDbRoot.localHWM << ", lbid: " << emDbRoot.startLbid << endl; + oss << "ExtentMap::getDbRootHWMInfo(): " << "OID " << OID + << " has HWM extent that is UNAVAILABLE for " << "DBRoot" << emDbRoot.dbRoot + << "; part#: " << emDbRoot.partitionNum << ", seg#: " << emDbRoot.segmentNum + << ", fbo: " << emDbRoot.fbo << ", localHWM: " << emDbRoot.localHWM + << ", lbid: " << emDbRoot.startLbid << endl; log(oss.str(), logging::LOG_TYPE_CRITICAL); throw runtime_error(oss.str()); } diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index ab7c3f4ee..cdfa738a4 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -79,6 +79,12 @@ class IDBDataFile; namespace BRM { +#define EM_MAGIC_V1 0x76f78b1c +#define EM_MAGIC_V2 0x76f78b1d +#define EM_MAGIC_V3 0x76f78b1e +#define EM_MAGIC_V4 0x76f78b1f +#define EM_MAGIC_V5 0x76f78b20 + using PartitionNumberT = uint32_t; using DBRootT = uint16_t; using SegmentT = uint16_t; @@ -158,6 +164,8 @@ struct EMCasualPartition_struct EXPORT EMCasualPartition_struct(); EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum, + const char status); EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em); }; @@ -182,6 +190,8 @@ struct EMEntry int16_t status; // extent avail for query or not, or out of service EMPartition_t partition; EXPORT EMEntry(); + EMEntry(const InlineLBIDRange& range, int fileID, uint32_t bOffset, HWM_t hwm, PartitionNumberT pNum, + uint16_t sNum, DBRootT dRoot, uint16_t cWid, int16_t s, EMPartition_t partition); EXPORT EMEntry(const EMEntry&); EXPORT EMEntry& operator=(const EMEntry&); EXPORT bool operator<(const EMEntry&) const; diff --git a/versioning/BRM/load_brm_from_file.cpp b/versioning/BRM/load_brm_from_file.cpp index bc7c05d9a..f70baafd7 100644 --- a/versioning/BRM/load_brm_from_file.cpp +++ b/versioning/BRM/load_brm_from_file.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 InfiniDB, Inc. +/* Copyright (C) 2024 MariaDB Corporation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -20,67 +20,81 @@ * If you re-compile extentmap.cpp to dump the extent map as it loads, you'll get a csv file on stdout. * Save this to a file and edit it as needed (remove the cruft at the top & bottom for sure). Then use * this tool to create a binary BRM_saves_em file. - * - * compile with - * g++ -g -Wall -o load_brm_from_file -I$HOME/genii/export/include -I/usr/include/libxml2 - * load_brm_from_file.cpp - * */ + #include #include #include #include #include #include -//#define NDEBUG #include #include using namespace std; -#include -using namespace boost; - +#include "CLI11.hpp" #include "extentmap.h" -using namespace BRM; -#define EM_MAGIC_V4 0x76f78b1f +static const char* BIN_NAME = "mcs-load-brm-from-file"; -namespace BRM +template +T parseField(std::stringstream& ss, const char delimiter) { -EMEntry::EMEntry() -{ - fileID = 0; - blockOffset = 0; - HWM = 0; - partitionNum = 0; - segmentNum = 0; - dbRoot = 0; - colWid = 0; - status = 0; + std::string field; + std::getline(ss, field, delimiter); + return std::stoll(field); } -EMCasualPartition_struct::EMCasualPartition_struct() + +BRM::EMEntry parseLine(const std::string& line, char delimiter = '|') { - lo_val = numeric_limits::min(); - hi_val = numeric_limits::max(); - sequenceNum = 0; - isValid = CP_INVALID; + std::stringstream ss(line); + std::string field; + + auto rangeStart = parseField(ss, delimiter); + auto rangeSize = parseField(ss, delimiter); + BRM::InlineLBIDRange range{rangeStart, rangeSize}; + + auto fileID = parseField(ss, delimiter); + auto blockOffset = parseField(ss, delimiter); + auto hwm = parseField(ss, delimiter); + auto partitionNum = parseField(ss, delimiter); + auto segmentNum = parseField(ss, delimiter); + auto dbRoot = parseField(ss, delimiter); + auto colWid = parseField(ss, delimiter); + auto status = parseField(ss, delimiter); + + auto hiVal = parseField(ss, delimiter); + auto loVal = parseField(ss, delimiter); + auto sequenceNum = parseField(ss, delimiter); + auto isValid = parseField(ss, delimiter); + auto partition = BRM::EMCasualPartition_t{loVal, hiVal, sequenceNum, isValid}; + + return BRM::EMEntry{range, fileID, blockOffset, hwm, partitionNum, + segmentNum, dbRoot, colWid, status, {partition}}; } -} // namespace BRM int main(int argc, char** argv) { - int e; + CLI::App app{BIN_NAME}; + app.description( + "A tool to build Extent Map image file from its text representation. A text representation can be obtained using 'editem -i'" + "display the lock state."); + std::string srcFilename; + std::string dstFilename; + bool debug = false; - int loadSize[3]; + app.add_option("-i,--input-filename", srcFilename, + "Extent Map as its text representation.") + ->required(); + app.add_option("-o,--output-filename", dstFilename, + "Extent Map output image file, default as input-filename.out") + ->default_val(""); + app.add_option("-d,--debug", debug, "Print extra output")->default_val(false); - if (argc < 2) - { - cerr << "filename arg needed" << endl; - return 1; - } + CLI11_PARSE(app, argc, argv); - ifstream in(argv[1]); - e = errno; + ifstream in(srcFilename); + int e = errno; if (!in) { @@ -90,81 +104,43 @@ int main(int argc, char** argv) // Brute force count the number of lines int numEMEntries = 0; - - string line; - - getline(in, line); - - while (!in.eof()) { - numEMEntries++; + string line; getline(in, line); + while (!in.eof()) + { + numEMEntries++; + getline(in, line); + } } - // start at the beginning again... - in.clear(); - in.seekg(0, ios_base::beg); + std::cout << "Number of entries: " << numEMEntries << std::endl; - idbassert(in.good()); - idbassert(in.tellg() == static_cast(0)); - - string outname(argv[1]); - outname += ".out"; - - ofstream out(outname.c_str()); - e = errno; - - if (!out) + if (dstFilename.empty()) { - cerr << "file write error: " << strerror(e) << endl; + dstFilename = srcFilename + ".out"; + } + ofstream outFile(dstFilename); + + BRM::InlineLBIDRange maxLBIDinUse{0, 0}; + + std::ifstream infile(srcFilename); + if (!infile.is_open()) + { + std::cerr << "Can not open file " << srcFilename << std::endl; return 1; } - loadSize[0] = EM_MAGIC_V4; + int loadSize[3]; + loadSize[0] = EM_MAGIC_V5; loadSize[1] = numEMEntries; loadSize[2] = 1; // one free list entry - out.write((char*)&loadSize, (3 * sizeof(int))); + outFile.write((char*)&loadSize, (3 * sizeof(int))); - InlineLBIDRange fl; - fl.start = 0; - // the max lbid is 2^54-1, the size is in units of 1k - fl.size = numeric_limits::max(); - - InlineLBIDRange maxLBIDinUse; - maxLBIDinUse.start = 0; - maxLBIDinUse.size = 0; - - getline(in, line); - - while (!in.eof()) + string line; + while (std::getline(infile, line)) { - EMEntry em; - int64_t v; - tokenizer<> tok(line); - tokenizer<>::iterator beg = tok.begin(); -#if 0 - emSrc[i].range.start - << '\t' << emSrc[i].range.size - << '\t' << emSrc[i].fileID - << '\t' << emSrc[i].blockOffset - << '\t' << emSrc[i].HWM - << '\t' << emSrc[i].partitionNum - << '\t' << emSrc[i].segmentNum - << '\t' << emSrc[i].dbRoot - << '\t' << emSrc[i].colWid - << '\t' << emSrc[i].status - << '\t' << emSrc[i].partition.cprange.hi_val - << '\t' << emSrc[i].partition.cprange.lo_val - << '\t' << emSrc[i].partition.cprange.sequenceNum - << '\t' << (int)(emSrc[i].partition.cprange.isValid) -#endif - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.range.start = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.range.size = v; + BRM::EMEntry em = parseLine(line); if (em.range.start > maxLBIDinUse.start) { @@ -172,66 +148,26 @@ int main(int argc, char** argv) maxLBIDinUse.size = em.range.size; } - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.fileID = v; + if (debug) + { + std::cout << em.range.start << '\t' << em.range.size << '\t' << em.fileID << '\t' << em.blockOffset + << '\t' << em.HWM << '\t' << em.partitionNum << '\t' << em.segmentNum << '\t' << em.dbRoot + << '\t' << em.colWid << '\t' << em.status << '\t' << em.partition.cprange.hiVal << '\t' + << em.partition.cprange.loVal << '\t' << em.partition.cprange.sequenceNum << '\t' + << (short int)(em.partition.cprange.isValid) << std::endl; + } - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.blockOffset = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.HWM = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partitionNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.segmentNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.dbRoot = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.colWid = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.status = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.hi_val = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.lo_val = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.sequenceNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.isValid = v; - - out.write((char*)&em, sizeof(em)); - - getline(in, line); + outFile.write((char*)&em, sizeof(em)); } + infile.close(); - fl.start = maxLBIDinUse.start + maxLBIDinUse.size * 1024; - fl.size -= fl.start / 1024; + auto flStart = maxLBIDinUse.start + maxLBIDinUse.size * 1024; + assert(flStart / 1024 <= numeric_limits::max()); + uint32_t flEnd = numeric_limits::max() - flStart / 1024; + BRM::InlineLBIDRange fl{flStart, flEnd}; - out.write((char*)&fl, sizeof(fl)); - - out.close(); - in.close(); + outFile.write((char*)&fl, sizeof(fl)); + outFile.close(); return 0; } diff --git a/versioning/BRM/lock_grabber.cpp b/versioning/BRM/lock_grabber.cpp deleted file mode 100644 index 7c6884bb8..000000000 --- a/versioning/BRM/lock_grabber.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016-2022 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* Takes two params, - * first, which lock use - * second, which side to use (read or write) - * third, lock or unlock it - */ - -#include -#include -#include -#include - -using namespace std; -using namespace rwlock; - -char* name; - -void usage() -{ - std::cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock" << std::endl; - size_t lockId = 0; - for (auto& lockName : RWLockNames) - { - std::cout << " " << lockId++ << "=" << lockName << " "; - } - std::cout << std::endl - << " which_side_to_use: r|w (read or write)" << std::endl - << " lock_or_unlock: l|u (lock or unlock)" << std::endl; - exit(1); -} - -int main(int argc, char** argv) -{ - uint32_t which_lock; // 1-5 - uint32_t which_side; // 0 or 1 - uint32_t lock_unlock; // 0 or 1 - RWLock* rwlock; - - name = argv[0]; - - if (argc != 4) - usage(); - - if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1) - usage(); - - try - { - which_lock = std::stoi(argv[1]); - } - catch (std::exception const& e) - { - std::cerr << "Cannot convert the lock id: " << e.what() << std::endl; - usage(); - } - - if (which_lock >= RWLockNames.size()) - usage(); - - size_t minLockId = (which_lock > 0) ? which_lock : 1; - size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1; - - if (argv[2][0] == 'r') - which_side = 0; - else if (argv[2][0] == 'w') - which_side = 1; - else - usage(); - - if (argv[3][0] == 'l') - lock_unlock = 0; - else if (argv[3][0] == 'u') - lock_unlock = 1; - else - usage(); - - for (size_t i = minLockId; i <= maxLockId; ++i) - { - rwlock = new RWLock(0x10000 * which_lock); - - if (which_side == 0) - { - if (lock_unlock == 0) - rwlock->read_lock(); - else - rwlock->read_unlock(); - } - else if (lock_unlock == 0) - { - rwlock->write_lock(); - } - else - { - rwlock->write_unlock(); - } - - delete rwlock; - } - - return 0; -} diff --git a/versioning/BRM/lock_state.cpp b/versioning/BRM/lock_state.cpp deleted file mode 100644 index 4b5cef63e..000000000 --- a/versioning/BRM/lock_state.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016-2022 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* Takes two params, - * first, which lock use - * second, which side to use (read or write) - * third, lock or unlock it - */ - -#include -#include -#include -#include - -using namespace std; -using namespace rwlock; - -char* name; - -void usage() -{ - std::cout << "Usage " << name << " which_lock_to_use:" << std::endl; - size_t lockId = 0; - for (auto& lockName : RWLockNames) - { - std::cout << " " << lockId++ << "=" << lockName << std::endl; - } - exit(1); -} - -int main(int argc, char** argv) -{ - uint32_t which_lock; // 0-6 - RWLock* rwlock; - LockState state; - - name = argv[0]; - - if (argc != 2) - usage(); - - if (strlen(argv[1]) != 1) - usage(); - - try - { - which_lock = std::stoi(argv[1]); - } - catch (std::exception const& e) - { - std::cerr << "Cannot convert the lock id: " << e.what() << std::endl; - usage(); - } - - if (which_lock >= RWLockNames.size()) - usage(); - - size_t minLockId = (which_lock > 0) ? which_lock : 1; - size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1; - - for (size_t i = minLockId; i <= maxLockId; ++i) - { - rwlock = new RWLock(0x10000 * i); - state = rwlock->getLockState(); - - cout << RWLockNames[i] << " RWLock" << std::endl - << " readers = " << state.reading << std::endl - << " writers = " << state.writing << std::endl - << " readers waiting = " << state.readerswaiting << std::endl - << " writers waiting = " << state.writerswaiting << std::endl - << " mutex locked = " << (int)state.mutexLocked << std::endl; - delete rwlock; - } - - return 0; -} diff --git a/versioning/BRM/shmem_locks.cpp b/versioning/BRM/shmem_locks.cpp new file mode 100644 index 000000000..917e64b81 --- /dev/null +++ b/versioning/BRM/shmem_locks.cpp @@ -0,0 +1,128 @@ +/* Copyright (C) 2016-2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#include +#include + +#include "CLI11.hpp" + +using namespace std; +using namespace rwlock; + +static const char* BIN_NAME = "mcs-load-brm-from-file"; + +std::string getShmemLocksList() +{ + std::ostringstream oss; + size_t lockId = 0; + oss << std::endl; + for (auto& lockName : RWLockNames) + { + oss << " " << lockId++ << "=" << lockName << std::endl; + } + return oss.str(); +} + +int viewLock(uint8_t lockId) +{ + size_t minLockId = (lockId > 0) ? lockId : 1; + size_t maxLockId = (lockId > 0) ? lockId : RWLockNames.size() - 1; + + for (size_t i = minLockId; i <= maxLockId; ++i) + { + auto rwlock = RWLock(0x10000 * i); + auto state = rwlock.getLockState(); + + cout << RWLockNames[i] << " RWLock" << std::endl + << " readers = " << state.reading << std::endl + << " writers = " << state.writing << std::endl + << " readers waiting = " << state.readerswaiting << std::endl + << " writers waiting = " << state.writerswaiting << std::endl + << " mutex locked = " << (int)state.mutexLocked << std::endl; + } + return 0; +} + +int lockOp(size_t minLockId, size_t maxLockId, bool lock, bool read) +{ + for (size_t i = minLockId; i <= maxLockId; ++i) + { + auto rwlock = RWLock(0x10000 * i); + + if (read) + { + if (lock) + rwlock.read_lock(); + else + rwlock.read_unlock(); + } + else if (lock) + { + rwlock.write_lock(); + } + else + { + rwlock.write_unlock(); + } + } + return 0; +} + +int main(int argc, char** argv) +{ + CLI::App app{BIN_NAME}; + app.description( + "A tool to operate or view shmem locks. If neither read nor write operation is specified, the tool will " + "display the lock state."); + uint8_t lockId; + bool debug = false; + bool read = false; + bool write = false; + bool lock = false; + bool unlock = false; + + app.add_option("-i, --lock-id", lockId, "Shmem lock numerical id: " + getShmemLocksList()) + ->expected(0, RWLockNames.size()) + ->required(); + app.add_flag("-r, --read-lock", read, "Use read lock.")->default_val(false); + app.add_flag("-w, --write-lock", write, "Use write lock..")->default_val(false)->excludes("-r"); + app.add_flag("-l, --lock", lock, "Lock the corresponding shmem lock.")->default_val(false); + app.add_flag("-u, --unlock", unlock, "Unlock the corresponding shmem write lock.") + ->default_val(false) + ->excludes("-l"); + app.add_flag("-d,--debug", debug, "Print extra output.")->default_val(false); + + CLI11_PARSE(app, argc, argv); + + if (!read && !write) + { + return viewLock(lockId); + } + + if (lock || unlock) + { + size_t minLockId = (lockId > 0) ? lockId : 1; + size_t maxLockId = (lockId > 0) ? lockId : RWLockNames.size() - 1; + return lockOp(minLockId, maxLockId, lock, read); + } + + return 0; +} + diff --git a/versioning/BRM/tablelockserver.cpp b/versioning/BRM/tablelockserver.cpp index 4d58c59c8..3ec2ebc50 100644 --- a/versioning/BRM/tablelockserver.cpp +++ b/versioning/BRM/tablelockserver.cpp @@ -110,7 +110,7 @@ void TableLockServer::load() if (!in) { ostringstream os; - os << "TableLockServer::load(): could not open the save file" << filename; + os << "TableLockServer::load(): either this is the first cluster start or could not open the save file" << filename; log(os.str(), logging::LOG_TYPE_DEBUG); return; } From 232c11d7f6cc63992727667887436f1243c6b4c1 Mon Sep 17 00:00:00 2001 From: Allen Herrera Date: Fri, 10 Jan 2025 16:43:42 -0500 Subject: [PATCH 35/65] update columnstore_review to v1.4.13 --- extra/columnstore_review.sh | 416 +++++++++++++++++++++++++++++++++--- 1 file changed, 385 insertions(+), 31 deletions(-) diff --git a/extra/columnstore_review.sh b/extra/columnstore_review.sh index ae249be0e..5c87983d9 100644 --- a/extra/columnstore_review.sh +++ b/extra/columnstore_review.sh @@ -1,15 +1,28 @@ #!/bin/bash # columnstore_review.sh # script by Edward Stoever for MariaDB support -VERSION=1.4.5 +# Contributors: Allen Herrera +# Patrizio Tamorri +VERSION=1.4.13 function prepare_for_run() { unset ERR - OUTDIR=/tmp/columnstore_review + if [ -n "$USER_PROVIDED_OUTPUT_PATH" ] && [ ! -d "$USER_PROVIDED_OUTPUT_PATH" ]; then + printf "The directory $USER_PROVIDED_OUTPUT_PATH does not exist.\n\n" + exit 1 + fi + + if [ -n "$USER_PROVIDED_OUTPUT_PATH" ]; then + OUTDIR=$USER_PROVIDED_OUTPUT_PATH/columnstore_review + TARDIR=$USER_PROVIDED_OUTPUT_PATH + else + OUTDIR=/tmp/columnstore_review + TARDIR=/tmp + fi mkdir -p $OUTDIR WARNFILE=$OUTDIR/cs_warnings.out if [ $EM_CHECK ]; then - EMOUTDIR=/tmp/columnstore_review/em; mkdir -p $EMOUTDIR + EMOUTDIR=$OUTDIR/em; mkdir -p $EMOUTDIR OUTPUTFILE=$EMOUTDIR/$(hostname)_cs_em_check.txt else OUTPUTFILE=$OUTDIR/$(hostname)_cs_review.txt @@ -44,6 +57,7 @@ function exists_mariadbd_running() { } function exists_columnstore_running() { + if [[ "$(ps -ef | grep -E "(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode)" | grep -v "grep"|wc -l)" == "0" ]]; then echo 'There are no Mariadb-Columnstore processes running.' >> $WARNFILE; else @@ -942,7 +956,15 @@ function dump_log () { } function collect_logs() { - LOGSOUTDIR=/tmp/columnstore_review/logs_$(date +"%m-%d-%H-%M-%S")/$(hostname) + + if [ -n "$USER_PROVIDED_OUTPUT_PATH" ]; then + TARPATH="$USER_PROVIDED_OUTPUT_PATH" + LOGSOUTDIR="$USER_PROVIDED_OUTPUT_PATH/columnstore_review/logs_$(date +"%m-%d-%H-%M-%S")/$(hostname)" + else + TARPATH=/tmp + LOGSOUTDIR=/tmp/columnstore_review/logs_$(date +"%m-%d-%H-%M-%S")/$(hostname) + fi + mkdir -p $LOGSOUTDIR || ech0 'Cannot create temporary directory for logs.'; mkdir -p $LOGSOUTDIR/system mkdir -p $LOGSOUTDIR/mariadb @@ -968,6 +990,7 @@ function collect_logs() { dump_log "mcs-loadbrm" $LOGSOUTDIR/columnstore/ dump_log "mcs-primproc" $LOGSOUTDIR/columnstore/ dump_log "mcs-workernode@1" $LOGSOUTDIR/columnstore/ + dump_log "mcs-workernode@2" $LOGSOUTDIR/columnstore/ dump_log "mcs-writeengineserver" $LOGSOUTDIR/columnstore/ dump_log "mcs-controllernode" $LOGSOUTDIR/columnstore/ @@ -984,7 +1007,9 @@ function collect_logs() { if [ -f "/proc/sys/kernel/threads-max" ]; then cp /proc/sys/kernel/threads-max $LOGSOUTDIR/system/kernal-threads-max; fi; if [ -f "/proc/sys/kernel/pid_max" ]; then cp /proc/sys/kernel/pid_max $LOGSOUTDIR/system/kernal-pid_max; fi; if [ -f "/proc/sys/vm/max_map_count" ]; then cp /proc/sys/vm/max_map_count $LOGSOUTDIR/system/kernal-max_map_count; fi; - if [ -f "/var/log/messages" ]; then cp /var/log/messages* $LOGSOUTDIR/system; fi; + # if [ -f "/var/log/messages" ]; then cp /var/log/messages* $LOGSOUTDIR/system; fi; # TOO MUCH COLLECTED... + find /var/log -name "messages*" -mtime -5 -type f -exec cp {} $LOGSOUTDIR/system \; 2>/dev/null + if [ -f "/var/log/syslog" ]; then find /var/log/syslog -name syslog -type f -exec tail -10000 {} > $LOGSOUTDIR/system/syslog \;; fi; if [ -f "/var/log/daemon.log" ]; then find /var/log/daemon.log -name daemon.log -type f -exec tail -10000 {} > $LOGSOUTDIR/system/daemon.log \;; fi; if command -v ulimit >/dev/null 2>&1; then @@ -998,7 +1023,13 @@ function collect_logs() { ls -1 columnstore/*.log 2>/dev/null | cpio -pd $LOGSOUTDIR/ 2>/dev/null ls -1 columnstore/*z 2>/dev/null | cpio -pd $LOGSOUTDIR/ 2>/dev/null find columnstore/archive columnstore/install columnstore/trace -mtime -30 | cpio -pd $LOGSOUTDIR/ 2>/dev/null - find columnstore/cpimport -mtime -1 | cpio -pd $LOGSOUTDIR/ 2>/dev/null + # find columnstore/cpimport -mtime -1 | cpio -pd $LOGSOUTDIR/ 2>/dev/null # COLLECTS TOO MUCH + find columnstore/cpimport -name "*.err" -size +0 -mtime -2 | cpio -pd $LOGSOUTDIR/ 2>/dev/null + + #collect ports Status + unset SUPPRESS_CLOSED_PORTS + check_ports > $LOGSOUTDIR/columnstore/$(hostname)_ports_check.txt 2>/dev/null + if [ $CAN_CONNECT ]; then mariadb -ABNe "show global variables" > $LOGSOUTDIR/mariadb/$(hostname)_global_variables.txt 2>/dev/null @@ -1007,18 +1038,18 @@ function collect_logs() { my_print_defaults --mysqld > $LOGSOUTDIR/mariadb/$(hostname)_my_print_defaults.txt 2>/dev/null if [ -f $OUTPUTFILE ]; then cp $OUTPUTFILE $LOGSOUTDIR/; fi cd $LOGSOUTDIR/.. - tar -czf /tmp/$COMPRESSFILE ./* + tar -czf $TARPATH/$COMPRESSFILE ./* cd - 1>/dev/null print_color "### COLLECTED LOGS FOR SUPPORT TICKET ###\n" ech0 "Attach the following tar file to your support ticket." if [ $THISISCLUSTER ]; then ech0 "Please collect logs with this script from each node in your cluster." fi - FILE_SIZE=$(stat -c %s /tmp/$COMPRESSFILE) + FILE_SIZE=$(stat -c %s $TARPATH/$COMPRESSFILE) if (( $FILE_SIZE > 52428800 )); then - print0 "The file /tmp/$COMPRESSFILE is larger than 50MB.\nPlease use MariaDB Large file upload at https://mariadb.com/upload/\nInform us about the upload in the support ticket.\n" + print0 "The file $TARPATH/$COMPRESSFILE is larger than 50MB.\nPlease use MariaDB Large file upload at https://mariadb.com/upload/\nInform us about the upload in the support ticket.\n" fi - print0 "\nCreated: /tmp/$COMPRESSFILE\n" + print0 "\nCreated: $TARPATH/$COMPRESSFILE\n" ech0 } @@ -1204,6 +1235,12 @@ fi } function backup_dbrm() { +if [ -n "$USER_PROVIDED_OUTPUT_PATH" ]; then + TARPATH="$USER_PROVIDED_OUTPUT_PATH" +else + TARPATH=/tmp +fi + STORAGE_TYPE=$(grep service /etc/columnstore/storagemanager.cnf | grep -v "^\#" | grep "\=" | awk -F= '{print $2}' | xargs) if [ "$(echo $STORAGE_TYPE | awk '{print tolower($0)}')" == "s3" ]; then print0 "This is node uses S3 storage for Columnstore. Exiting.\n\n"; return; fi @@ -1229,10 +1266,10 @@ fi fi set_data1dir cd $DATA1DIR/systemFiles - tar -czf /tmp/$COMPRESSFILE ./dbrm + tar -czf $TARPATH/$COMPRESSFILE ./dbrm cd - 1>/dev/null print_color "### DBRM EXTENT MAP BACKUP ###\n" - ech0 "Files in dbrm directory backed up to compressed archive /tmp/$COMPRESSFILE." + ech0 "Files in dbrm directory backed up to compressed archive $TARPATH/$COMPRESSFILE" ech0 "Files in /tmp can be deleted on reboot. It is recommended to move the archive to a safe location." ech0 } @@ -2059,6 +2096,250 @@ function ensure_owner_privs_of_tmp_dir() { } +# CHECK PORTS FUNCTIONS BY Patrizio Tamorri +function check_ports(){ + # Check if nmap is installed + if ! command -v nmap &> /dev/null; then + printf "nmap is not installed.\n\n" + return + fi + + # Define the ports to check + ports="8600,8601,8602,8603,8604,8605,8606,8607,8608,8609,8610,8611,8612,8613,8614,8615,8616,8617,8618,8619,8620,8630,8700,8800,3306,8999" + + # Get the local node from the file + my_node=$(cat /var/lib/columnstore/local/module) + + # Get the hostname and local IP address of the machine + hostname=$(hostname) + local_ip=$(hostname -I | awk '{print $1}') + + # Extract IPs from Columnstore.xml, handling special characters like \r, \n, and \t + ips=$(grep -A 1 "_WriteEngineServer" /etc/columnstore/Columnstore.xml \ + | sed "/${my_node}_WriteEngineServer/,+0d" \ + | grep "" \ + | tr -d '\r\n\t' \ + | sed -e 's///g' -e 's/<\/IPAddr>//g' -e 's/^[ \t]*//' -e 's/[ \t]*$//' \ + | sort -u) + + # Extract IPAddr:Port pairs from the XML file, removing \r, \n, and \t + local_ports=$(grep -E "|" /etc/columnstore/Columnstore.xml \ + | tr -d '\r\n\t' \ + | sed -e 's/<\/\?IPAddr>//g' -e 's/<\/\?Port>//g' \ + | awk 'NR%2{printf "%s:", $0; next;} 1') + + pass=true + + # Function to check if a port is available to use + check_port_nmap_available_to_use() { + ip=$1 + port=$2 + + # Use nmap to check the port status + result=$(nmap -T4 -p $port $ip | grep "$port" | awk '{print $2}') + + if [ "$result" = "open" ]; then + echo "$ip:$port - Port is open: SUCCESS" + elif [ "$result" = "closed" ]; then + if [ ! ${SUPPRESS_CLOSED_PORTS} ]; then echo "$ip:$port - Port is closed and not firewalled"; fi + elif [ "$result" = "filtered" ]; then + echo "$ip:$port - Port is filtered (firewalled or blocked): ERROR" + pass=false + else + echo "$ip:$port - Unknown port status: ERROR" + pass=false + fi + } + + # Function to check if a port must be open + check_port_nmap_must_be_opened() { + ip=$1 + port=$2 + + # Use nmap to check the port status + result=$(nmap -T4 -p $port $ip | grep "$port" | awk '{print $2}') + + if [ "$result" = "open" ]; then + echo "$ip:$port - Port is open: SUCCESS" + elif [ "$result" = "closed" ]; then + echo "$ip:$port - Port is closed and not firewalled: ERROR" + pass=false + elif [ "$result" = "filtered" ]; then + echo "$ip:$port - Port is filtered (firewalled or blocked): ERROR" + pass=false + else + echo "$ip:$port - Unknown port status: ERROR" + pass=false + fi + } + + # Loop through each IP and check the ports + for ipadd in $ips; do + echo "Checking ports on $ipadd..." + + # Replace ipadd with 127.0.0.1 if it matches the local IP or hostname + if [[ "$ipadd" == "$local_ip" || "$ipadd" == "$hostname" ]]; then + ipadd="127.0.0.1" + fi + + for port in ${ports//,/ }; do + ip_port="$ipadd:$port" + if [[ " ${local_ports[@]} " =~ " $ip_port " ]]; then + check_port_nmap_must_be_opened $ipadd $port + else + check_port_nmap_available_to_use $ipadd $port + fi + done + done + + # Final status report + if [ "$pass" = true ]; then + printf "All nodes passed the port test.\n\n" + else + printf "One or more nodes failed the port test. Please investigate.\n\n" + fi + + +} + +function clear_rollback() { + unset ERR + CLEAR_ROLLBACK_MESSAGE="It is recommended that you clear rollback files only when instructed to do so by Mariadb Support.\nType c to clear rollback files.\nType any other key to exit.\n" + STORAGE_TYPE=$(grep service /etc/columnstore/storagemanager.cnf | grep -v "^\#" | grep "\=" | awk -F= '{print $2}' | awk '{print tolower($0)}' | xargs) + DATA1DIR=$(mcsGetConfig SystemConfig DBRoot1 2>/dev/null) || DATA1DIR=/var/lib/columnstore/data1 + BRMSAV=$(cat $DATA1DIR/systemFiles/dbrm/BRM_saves_current | xargs) +if [[ ! "$(ps -ef | grep -E "(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode)" | grep -v "grep"|wc -l)" == "0" ]]; then + TEMP_COLOR=lred; print_color "Columnstore processes are running.\nYou may clear rollback fragments only when Columnstore processes are stopped."; unset TEMP_COLOR + print0 "\nExiting.\n\n"; exit 0 +fi + +if [ "$STORAGE_TYPE" == "localstorage" ]; then + COUNTFILES=$(find $DATA1DIR/systemFiles \( -name "${BRMSAV}_vss" -o -name "${BRMSAV}_vbbm" \) -size +0 | wc -l) + if [ "$COUNTFILES" == "0" ]; then + TEMP_COLOR=lred; print_color "Rollback files are empty."; unset TEMP_COLOR + print0 "\nExiting.\n\n"; exit 0 + fi + + print0 "$CLEAR_ROLLBACK_MESSAGE" + + read -s -n 1 RESPONSE + if [ "$RESPONSE" == "c" ]; then + ech0; ech0 + BRM_SAVES_BACKUP_FILE=$(hostname)_$(date +"%Y-%m-%d-%H-%M-%S")_BRM_saves.tar + cd $DATA1DIR/systemFiles/dbrm/ + print0 "BRM_saves_current: ${BRMSAV}\n\nBacking up these files:\n" + find . \( -name "${BRMSAV}_vss" -o -name "${BRMSAV}_vbbm" \) -exec tar -rvf /tmp/$BRM_SAVES_BACKUP_FILE {} \; + ech0 + find . \( -name "${BRMSAV}_vss" -o -name "${BRMSAV}_vbbm" \) -size +0 -exec truncate -s0 {} \; || ERR=true + COUNTFILES=$(find $DATA1DIR/systemFiles \( -name "${BRMSAV}_vss" -o -name "${BRMSAV}_vbbm" \) -size +0 | wc -l) + if [ $ERR ] || [ "$COUNTFILES" != "0" ]; then + ech0 "Something went wrong. Check the size of files ${BRMSAV}_vss and ${BRMSAV}_vbbm. Each file should be zero bytes in size." + ls -lrt $DATA1DIR/systemFiles/dbrm + else + TEMP_COLOR=lcyan; print_color "BRM_saves files backed up to /tmp/$BRM_SAVES_BACKUP_FILE.\nFiles cleared successfully.\n\n"; unset TEMP_COLOR + fi + else + print0 "\nNothing done.\n\n" + fi + +fi + +if [ "$STORAGE_TYPE" == "s3" ]; then + DBRM_TMP_DIR=/tmp/dbrm-before-clearing-$(date +"%Y-%m-%d-%H-%M-%S") || ERR=true + print0 "$CLEAR_ROLLBACK_MESSAGE" + read -s -n 1 RESPONSE + if [ "$RESPONSE" == "c" ]; then + ## REF: https://mariadbcorp.atlassian.net/wiki/spaces/Support/pages/1600094249/Stuck+load_brm+failed+rollback+of+a+transaction + cd /var/lib/columnstore/storagemanager/metadata/data1/systemFiles/dbrm/ || ERR=true + mkdir -p $DBRM_TMP_DIR || ERR=true + find . | cpio -pd $DBRM_TMP_DIR || ERR=true + # Clear vss and vbbm files + rm -f BRM_saves_vss.meta BRM_saves_vbbm.meta || ERR=true + touch BRM_saves_vss.meta || ERR=true; chown mysql:mysql BRM_saves_vss.meta || ERR=true + touch BRM_saves_vbbm.meta || ERR=true; chown mysql:mysql BRM_saves_vbbm.meta || ERR=true + rm -rf /var/lib/columnstore/storagemanager/cache/data1/* || ERR=true + mkdir /var/lib/columnstore/storagemanager/cache/data1/downloading || ERR=true + chown mysql:mysql -R /var/lib/columnstore/storagemanager/cache || ERR=true + + if [ $ERR ]; then + ech0 "Something went wrong." + else + TEMP_COLOR=lcyan; print_color "BRM_saves files backed up to $DBRM_TMP_DIR.\nFiles cleared successfully.\n\n"; unset TEMP_COLOR + fi + + else + print0 "\nNothing done.\n\n" + fi +fi +} + +function kill_columnstore(){ +COUNT_ANY_STRAGGLERS=$(ps -ef | grep -E '(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode|load_brm)' | grep -v "grep" | wc -l) +PM1=$(mcsGetConfig pm1_WriteEngineServer IPAddr) +PM2=$(mcsGetConfig pm2_WriteEngineServer IPAddr) +if [ ! "$PM1" == "127.0.0.1" ] && [ ! -z $PM2 ]; then + THISISCLUSTER=true +fi + +if [ "$COUNT_ANY_STRAGGLERS" == "0" ]; then + TEMP_COLOR=lred; print_color "Columnstore processes are not running.\n"; unset TEMP_COLOR + clearShm + TEMP_COLOR=lcyan; print_color "Columnstore shared memory cleared.\n"; unset TEMP_COLOR + print0 "\nExiting.\n\n"; exit 0 +fi + +if [ $THISISCLUSTER ] && [ "$COUNT_ANY_STRAGGLERS" != "0" ]; then + TEMP_COLOR=lred; print_color "WARNING: This is a columnstore cluster and it is best to use cmapi commands to stop columnstore processes.\n"; unset TEMP_COLOR +fi + + +TEMP_COLOR=lcyan; print_color "Press c to stop all columnstore processes on this node.\n"; unset TEMP_COLOR + +read -s -n 1 RESPONSE + if [ "$RESPONSE" == "c" ]; then + if [ "$COUNT_ANY_STRAGGLERS" != "0" ]; then + ech0 "Attempting to gracefully stop mcs-ddlproc." + systemctl stop mcs-ddlproc; + ech0 "Attempting to gracefully stop mcs-dmlproc." + systemctl stop mcs-dmlproc; + systemctl stop mcs-exemgr 2>/dev/null; # if cs 6.4 and prior + ech0 "Attempting to gracefully stop mcs-controllernode." + systemctl stop mcs-controllernode; + ech0 "Attempting to gracefully stop mcs-storagemanager." + systemctl stop mcs-storagemanager; + ech0 "Attempting to gracefully stop mcs-primproc." + systemctl stop mcs-primproc; + ech0 "Attempting to gracefully stop mcs-writeengineserver." + systemctl stop mcs-writeengineserver; + ech0 "Attempting to gracefully stop mcs-workernode@1." + systemctl stop mcs-workernode@1; + ech0 "Attempting to gracefully stop mcs-workernode@2." + systemctl stop mcs-workernode@2; + fi + + COUNT_ANY_STRAGGLERS=$(ps -ef | grep -E '(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode|load_brm)' | grep -v "grep" | wc -l) + if [ "$COUNT_ANY_STRAGGLERS" != "0" ]; then + ech0 "Remaining processes:" + ps -ef | grep -E '(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode|load_brm)' | grep -v "grep" + ech0 "Killing them..." + ps -ef | grep -E '(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode|load_brm)' | grep -v "grep" | awk '{print $2}' | xargs kill -9 + fi + + COUNT_ANY_STRAGGLERS=$(ps -ef | grep -E '(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode|load_brm)' | grep -v "grep" | wc -l) + + if [ "$COUNT_ANY_STRAGGLERS" == "0" ]; then + ech0 "No columnstore processes running." + clearShm + TEMP_COLOR=lcyan; print_color "Columnstore shared memory cleared.\n"; unset TEMP_COLOR + else + ech0 "After two attempts to kill all Columnstore processes, this is still running:" + ps -ef | grep -E '(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode|load_brm)' | grep -v "grep" + fi +else + print0 "\nNothing done.\n\n" +fi +} + function display_outputfile_message() { echo "The output of this script is saved in the file $OUTPUTFILE"; echo; } @@ -2071,26 +2352,31 @@ function display_help_message() { If database is up, this script will connect as root@localhost via socket. Switches: - --help # display this message - --version # only show the header with version information - --logs # create a compressed archive of logs for MariaDB Support Ticket - --backupdbrm # takes a compressed backup of extent map files in dbrm directory - --testschema # creates a test schema, tables, imports, queries, drops schema - --testschemakeep # creates a test schema, tables, imports, queries, does not drop - --ldlischema # using ldli, creates test schema, tables, imports, queries, drops schema - --ldlischemakeep # using ldli, creates test schema, tables, imports, queries, does not drop - --emptydirs # searches $COLUMNSTOREDIR for empty directories - --notmysqldirs # searches $COLUMNSTOREDIR for directories not owned by mysql - --emcheck # Checks the extent map for orphaned and missing files - --s3check # Checks the extent map against S3 storage - --pscs # Adds the pscs command. pscs lists running columnstore processes - --schemasync # Fix out-of-sync columnstore tables (CAL0009) - --tmpdir # Ensure owner of temporary dir after reboot (MCOL-4866 & MCOL-5242) + --help # display this message + --version # only show the header with version information + --logs # create a compressed archive of logs for MariaDB Support Ticket + --path # define the path for where to save files/tarballs and outputs of this script + --backupdbrm # takes a compressed backup of extent map files in dbrm directory + --testschema # creates a test schema, tables, imports, queries, drops schema + --testschemakeep # creates a test schema, tables, imports, queries, does not drop + --ldlischema # using ldli, creates test schema, tables, imports, queries, drops schema + --ldlischemakeep # using ldli, creates test schema, tables, imports, queries, does not drop + --emptydirs # searches $COLUMNSTOREDIR for empty directories + --notmysqldirs # searches $COLUMNSTOREDIR for directories not owned by mysql + --emcheck # Checks the extent map for orphaned and missing files + --s3check # Checks the extent map against S3 storage + --pscs # Adds the pscs command. pscs lists running columnstore processes + --schemasync # Fix out-of-sync columnstore tables (CAL0009) + --tmpdir # Ensure owner of temporary dir after reboot (MCOL-4866 & MCOL-5242) + --checkports # Checks if ports needed by Columnstore are opened + --eustack # Dumps the stack of Columnstore processes + --clearrollback # Clear any rollback fragments from dbrm files + --killcolumnstore # Stop columnstore processes gracefully, then kill remaining processes Color output switches: - --color=none # print headers without color - --color=red # print headers in color - # Options: [none,red,blue,green,yellow,magenta,cyan] prefix color with "l" for light\n" + --color=none # print headers without color + --color=red # print headers in color + # Options: [none,red,blue,green,yellow,magenta,cyan] prefix color with "l" for light\n" ech0 } @@ -2145,6 +2431,53 @@ fi printf "$1" >> $OUTPUTFILE } +function get_eu_stack() { + if ! command -v eu-stack &> /dev/null; then + printf "\n[!] eu-stack not found. Please install eu-stack\n\n" + ech0 "example: " + ech0 " yum install elfutils -y" + ech0 " apt-get install elfutils" + ech0 + exit 1; + fi + + # Confirm CS online + if [[ "$(ps -ef | grep -E "(PrimProc|ExeMgr|DMLProc|DDLProc|WriteEngineServer|StorageManager|controllernode|workernode)" | grep -v "grep"|wc -l)" == "0" ]]; then + printf "Columnstore processes are not running. EU Stack will not be collected.\n\n" + exit 1; + fi + + eu=$(which eu-stack) + EU_FOLDER="$(hostname)_$(date +"%Y-%m-%d-%H-%M-%S")_eu_stack" + if [ ! -d "$OUTDIR/$EU_FOLDER" ]; then mkdir -p "$OUTDIR/$EU_FOLDER"; fi + + $eu -p $(pidof PrimProc) > "$OUTDIR/$EU_FOLDER/eu-PrimProc.txt" ; + $eu -p $(pidof DMLProc) > "$OUTDIR/$EU_FOLDER/eu-DMLProc.txt" ; + $eu -p $(pidof DDLProc) > "$OUTDIR/$EU_FOLDER/eu-DDLProc.txt" ; + $eu -p $(pidof mariadbd) > "$OUTDIR/$EU_FOLDER/eu-mariadbd.txt" ; + $eu -p $(pidof WriteEngineServer) > "$OUTDIR/$EU_FOLDER/eu-WriteEngineServer.txt" ; + $eu -p $(pidof controllernode) > "$OUTDIR/$EU_FOLDER/eu-controllernode.txt" ; + $eu -p $(pidof workernode) > "$OUTDIR/$EU_FOLDER/eu-workernode.txt" ; + cd $OUTDIR + tar -czf "$OUTDIR/$EU_FOLDER.tar.gz" $EU_FOLDER/* + + if [ -f "$OUTDIR/$EU_FOLDER.tar.gz" ]; then + print_color "### EU STACK COMPLETE ###\n" + else + print0 "EU Stack files not found.\n" + exit 1; + fi + + # cleanup + mv "$OUTDIR/$EU_FOLDER.tar.gz" $TARDIR + if [ -f "$TARDIR/$EU_FOLDER.tar.gz" ]; then + print0 "Created: $TARDIR/$EU_FOLDER.tar.gz \n\n" + else + print0 "EU Stack files not found.\n" + exit 1; + fi +} + COLOR=default for params in "$@"; do unset VALID; @@ -2165,6 +2498,7 @@ for params in "$@"; do if [ "$params" == '--help' ]; then HELP=true; VALID=true; fi if [ "$params" == '--version' ]; then if [ ! $SKIP_REPORT ]; then DISPLAY_VERSION=true; fi; VALID=true; fi if [ "$params" == '--logs' ]; then if [ ! $SKIP_REPORT ]; then COLLECT_LOGS=true; fi; VALID=true; fi + if [[ "$params" == "--path"* ]]; then USER_PROVIDED_OUTPUT_PATH=$(echo "$params" | awk -F= '{print $2}'); VALID=true; fi if [ "$params" == '--backupdbrm' ]; then BACKUP_DBRM=true; SKIP_REPORT=true; unset COLLECT_LOGS; VALID=true; fi if [ "$params" == '--testschema' ]; then TEST_SCHEMA=true; SKIP_REPORT=true; unset COLLECT_LOGS; VALID=true; fi if [ "$params" == '--testschemakeep' ]; then TEST_SCHEMA_KEEP=true; SKIP_REPORT=true; unset COLLECT_LOGS; VALID=true; fi @@ -2177,13 +2511,17 @@ for params in "$@"; do if [ "$params" == '--pscs' ]; then PSCS_ALIAS=true; SKIP_REPORT=true; unset COLLECT_LOGS; VALID=true; fi if [ "$params" == '--schemasync' ]; then SCHEMA_SYNC=true; SKIP_REPORT=true; unset COLLECT_LOGS; VALID=true; fi if [ "$params" == '--tmpdir' ]; then FIX_TMP_DIR=true; SKIP_REPORT=true; unset COLLECT_LOGS; VALID=true; fi + if [ "$params" == '--clearrollback' ]; then CLEARROLLBACK=true; SKIP_REPORT=true; unset COLLECT_LOGS; VALID=true; fi + if [ "$params" == '--checkports' ]; then SKIP_REPORT=true; CHECKPORTS=true;VALID=true; fi + if [ "$params" == '--killcolumnstore' ]; then KILLCS=true; SKIP_REPORT=true; unset COLLECT_LOGS; VALID=true; fi + if [ "$params" == '--eustack' ]; then SKIP_REPORT=true; COLLECT_EU_STACK=true;VALID=true; fi if [ ! $VALID ]; then INVALID_INPUT=$params; fi done prepare_for_run exists_client_able_to_connect_with_socket if [ $DISPLAY_VERSION ]; then exit 0; fi -if [ $INVALID_INPUT ]; then TEMP_COLOR=lred; print_color "Invalid parameter: ";ech0 $INVALID_INPUT; ech0; unset TEMP_COLOR; fi +if [ $INVALID_INPUT ]; then TEMP_COLOR=lred; print_color "Invalid parameter: ";ech0 $INVALID_INPUT; ech0; unset TEMP_COLOR; exit 1; fi if [ $HELP ]||[ $INVALID_INPUT ]; then display_help_message exit 0 @@ -2267,6 +2605,7 @@ report_cs_table_locks report_columnstore_query_count report_calpontsys_exists report_columnstore_tables +SUPPRESS_CLOSED_PORTS=true; check_ports TEMP_COLOR=lblue; print_color "===================== LOGS =====================\n"; unset TEMP_COLOR report_host_datetime report_last_10_error_log_error @@ -2329,3 +2668,18 @@ if [ $FIX_TMP_DIR ]; then ensure_owner_privs_of_tmp_dir fi +if [ $CLEARROLLBACK ]; then + clear_rollback +fi + +if [ $CHECKPORTS ]; then + check_ports +fi + +if [ $KILLCS ]; then + kill_columnstore +fi + +if [ $COLLECT_EU_STACK ]; then + get_eu_stack +fi From f99c24b47da7a951e56b54db6993cfaec50b614a Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 13 Jan 2025 16:05:23 +0000 Subject: [PATCH 36/65] MCOL-5880: get rid of CLI11 dep in favour of boost::program_options --- CMakeLists.txt | 1 - cmake/CLI11.cmake | 15 ------ versioning/BRM/CMakeLists.txt | 4 +- versioning/BRM/load_brm_from_file.cpp | 43 +++++++++++----- versioning/BRM/shmem_locks.cpp | 73 +++++++++++++++++++-------- 5 files changed, 84 insertions(+), 52 deletions(-) delete mode 100644 cmake/CLI11.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e26483967..0f8cd0737 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,7 +136,6 @@ SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE(columnstore_version) INCLUDE(misc) INCLUDE(boost) -INCLUDE(CLI11) INCLUDE(thrift) INCLUDE(arrow) diff --git a/cmake/CLI11.cmake b/cmake/CLI11.cmake deleted file mode 100644 index d86e9f99c..000000000 --- a/cmake/CLI11.cmake +++ /dev/null @@ -1,15 +0,0 @@ -set(EXTERNAL_INCLUDE_DIR "${CMAKE_BINARY_DIR}/external/include") -file(MAKE_DIRECTORY "${EXTERNAL_INCLUDE_DIR}") - -set(C11CLI_URL "https://github.com/CLIUtils/CLI11/releases/download/v2.4.2/CLI11.hpp") -set(C11CLI_HEADER "${EXTERNAL_INCLUDE_DIR}/CLI11.hpp") -set(CLI11_INCLUDE_DIR "${EXTERNAL_INCLUDE_DIR}") - -file(DOWNLOAD - ${C11CLI_URL} - ${C11CLI_HEADER} - SHOW_PROGRESS STATUS download_status -) - -add_library(CLI11 INTERFACE) -target_include_directories(CLI11 INTERFACE ${CLI11_INCLUDE_DIR}) \ No newline at end of file diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 203360bad..ac165d79e 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -122,9 +122,9 @@ target_link_libraries(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENG install(TARGETS mcs-load-em DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) add_executable(mcs-load-brm-from-file load_brm_from_file.cpp) -target_link_libraries(mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11) +target_link_libraries(mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} boost_program_options) install(TARGETS mcs-load-brm-from-file DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) add_executable(mcs-shmem-locks shmem_locks.cpp) -target_link_libraries(mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11) +target_link_libraries(mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} boost_program_options) install(TARGETS mcs-shmem-locks DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) \ No newline at end of file diff --git a/versioning/BRM/load_brm_from_file.cpp b/versioning/BRM/load_brm_from_file.cpp index f70baafd7..1ec44f1d1 100644 --- a/versioning/BRM/load_brm_from_file.cpp +++ b/versioning/BRM/load_brm_from_file.cpp @@ -30,13 +30,14 @@ #include #include #include + +#include +namespace po = boost::program_options; + using namespace std; -#include "CLI11.hpp" #include "extentmap.h" -static const char* BIN_NAME = "mcs-load-brm-from-file"; - template T parseField(std::stringstream& ss, const char delimiter) { @@ -75,23 +76,37 @@ BRM::EMEntry parseLine(const std::string& line, char delimiter = '|') int main(int argc, char** argv) { - CLI::App app{BIN_NAME}; - app.description( - "A tool to build Extent Map image file from its text representation. A text representation can be obtained using 'editem -i'" + po::options_description desc( + "A tool to build Extent Map image file from its text representation. A text representation can be " + "obtained using 'editem -i'" "display the lock state."); + std::string srcFilename; std::string dstFilename; bool debug = false; - app.add_option("-i,--input-filename", srcFilename, - "Extent Map as its text representation.") - ->required(); - app.add_option("-o,--output-filename", dstFilename, - "Extent Map output image file, default as input-filename.out") - ->default_val(""); - app.add_option("-d,--debug", debug, "Print extra output")->default_val(false); + // clang-format off + desc.add_options() + ("help", "produce help message") + ("input-filename,i", + po::value(&srcFilename)->required(), + "Extent Map as its text representation.") + ("output-filename,o", + po::value(&dstFilename)->default_value(""), + "Extent Map output image file, default as input-filename.out") + ("debug,d", po::bool_switch(&debug)->default_value(false), "Print extra output"); + // clang-format on - CLI11_PARSE(app, argc, argv); + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + + if (argc == 1 || vm.count("help")) + { + cout << desc << "\n"; + return 1; + } + + po::notify(vm); ifstream in(srcFilename); int e = errno; diff --git a/versioning/BRM/shmem_locks.cpp b/versioning/BRM/shmem_locks.cpp index 917e64b81..f8a455597 100644 --- a/versioning/BRM/shmem_locks.cpp +++ b/versioning/BRM/shmem_locks.cpp @@ -21,12 +21,11 @@ #include #include -#include "CLI11.hpp" - using namespace std; using namespace rwlock; -static const char* BIN_NAME = "mcs-load-brm-from-file"; +#include +namespace po = boost::program_options; std::string getShmemLocksList() { @@ -85,31 +84,66 @@ int lockOp(size_t minLockId, size_t maxLockId, bool lock, bool read) return 0; } +void conflicting_options(const boost::program_options::variables_map& vm, const std::string& opt1, + const std::string& opt2) +{ + if (vm.count(opt1) && !vm[opt1].defaulted() && vm.count(opt2) && !vm[opt2].defaulted()) + { + throw std::logic_error(std::string("Conflicting options '") + opt1 + "' and '" + opt2 + "'."); + } +} + +template +void check_value(const boost::program_options::variables_map& vm, const std::string& opt1, T lower_bound, + T upper_bound) +{ + auto value = vm[opt1].as(); + if (value < lower_bound || value >= upper_bound) + { + throw std::logic_error(std::string("Option '") + opt1 + "' is out of range.: " + std::to_string(value)); + } +} + int main(int argc, char** argv) { - CLI::App app{BIN_NAME}; - app.description( - "A tool to operate or view shmem locks. If neither read nor write operation is specified, the tool will " - "display the lock state."); - uint8_t lockId; + int lockId; bool debug = false; bool read = false; bool write = false; bool lock = false; bool unlock = false; - app.add_option("-i, --lock-id", lockId, "Shmem lock numerical id: " + getShmemLocksList()) - ->expected(0, RWLockNames.size()) - ->required(); - app.add_flag("-r, --read-lock", read, "Use read lock.")->default_val(false); - app.add_flag("-w, --write-lock", write, "Use write lock..")->default_val(false)->excludes("-r"); - app.add_flag("-l, --lock", lock, "Lock the corresponding shmem lock.")->default_val(false); - app.add_flag("-u, --unlock", unlock, "Unlock the corresponding shmem write lock.") - ->default_val(false) - ->excludes("-l"); - app.add_flag("-d,--debug", debug, "Print extra output.")->default_val(false); + po::options_description desc( + "A tool to operate or view shmem locks. If neither read nor write operation is specified, the tool " + "will " + "display the lock state."); - CLI11_PARSE(app, argc, argv); + std::string lockid_description = std::string("Shmem lock numerical id: ") + getShmemLocksList(); + + // clang-format off + desc.add_options()("help", "produce help message") + ("lock-id,i", po::value(&lockId)->required(), lockid_description.c_str()) + ("read-lock,r", po::bool_switch(&read)->default_value(false), "Use read lock.") + ("write-lock,w", po::bool_switch(&write)->default_value(false), "Use write lock.") + ("lock,l", po::bool_switch(&lock)->default_value(false), "Lock the corresponding shmem lock.") + ("unlock,u", po::bool_switch(&unlock)->default_value(false), "Unlock the corresponding shmem write lock.") + ("debug,d", po::bool_switch(&debug)->default_value(false), "Print extra output."); + // clang-format on + + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + + if (argc == 1 || vm.count("help")) + { + cout << desc << "\n"; + return 1; + } + + conflicting_options(vm, "lock", "unlock"); + conflicting_options(vm, "read-lock", "write-lock"); + check_value(vm, "lock-id", 0, RWLockNames.size()); + + po::notify(vm); if (!read && !write) { @@ -125,4 +159,3 @@ int main(int argc, char** argv) return 0; } - From 7dcc6a251a9dd7dd8f8d523567a87a825736c3db Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 16 Jan 2025 09:02:20 +1100 Subject: [PATCH 37/65] MCOL-5881 set/getThreadName use FreeBSD API (#3383) Taken from FreeBSD ports, this uses the FreeBSD APIs rather than the Linux specific prctl to change and retreive the thread names. Co-authored-by: Bernard Spil --- utils/common/threadnaming.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/utils/common/threadnaming.cpp b/utils/common/threadnaming.cpp index 3183bc80f..722d17b36 100644 --- a/utils/common/threadnaming.cpp +++ b/utils/common/threadnaming.cpp @@ -22,13 +22,21 @@ namespace utils { void setThreadName(const char* threadName) { +#ifdef __FreeBSD__ + pthread_set_name_np(pthread_self(), threadName); +#else prctl(PR_SET_NAME, threadName, 0, 0, 0); +#endif } std::string getThreadName() { char buf[32]; +#ifdef __FreeBSD__ + pthread_get_name_np(pthread_self(), buf, sizeof(buf)); +#else prctl(PR_GET_NAME, buf, 0, 0, 0); +#endif return std::string(buf); } } // namespace utils From 8b9db66ddd0381706d22e4fabd731bf39103cad9 Mon Sep 17 00:00:00 2001 From: Serguey Zefiov Date: Mon, 10 Feb 2025 20:18:06 +0000 Subject: [PATCH 38/65] fix(MCOL-5889): Improper handle of DOUBLE result type with DECIMAL arguments Sometimes server assigns DOUBLE type for arithmetic operations over DECIMAL arguments. In this rare case width of result was incorrectly adjusted and it triggered an assertion. Now width of result gets adjusted only if result type is also DECIMAL. --- dbcon/mysql/ha_mcs_execplan.cpp | 3 ++- .../MCOL-5889-double-is-not-decimal.result | 25 +++++++++++++++++++ .../MCOL-5889-double-is-not-decimal.test | 24 ++++++++++++++++++ tests/scripts/fullmtr.sh | 6 +++++ 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 mysql-test/columnstore/bugfixes/MCOL-5889-double-is-not-decimal.result create mode 100644 mysql-test/columnstore/bugfixes/MCOL-5889-double-is-not-decimal.test diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 6ec3990ce..a597155e6 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4025,7 +4025,8 @@ ReturnedColumn* buildArithmeticColumnBody(Item_func* item, gp_walk_info& gwi, bo int32_t leftColWidth = leftColType.colWidth; int32_t rightColWidth = rightColType.colWidth; - if (leftColWidth == datatypes::MAXDECIMALWIDTH || rightColWidth == datatypes::MAXDECIMALWIDTH) + if ((leftColWidth == datatypes::MAXDECIMALWIDTH || rightColWidth == datatypes::MAXDECIMALWIDTH) + && datatypes::isDecimal(mysqlType.colDataType)) { mysqlType.colWidth = datatypes::MAXDECIMALWIDTH; diff --git a/mysql-test/columnstore/bugfixes/MCOL-5889-double-is-not-decimal.result b/mysql-test/columnstore/bugfixes/MCOL-5889-double-is-not-decimal.result new file mode 100644 index 000000000..b8a31665c --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-5889-double-is-not-decimal.result @@ -0,0 +1,25 @@ +DROP DATABASE IF EXISTS MCOL5889; +CREATE DATABASE MCOL5889; +USE MCOL5889; +CREATE TABLE t1 (f1 DECIMAL, f2 BIGINT, f3 DOUBLE) ENGINE=columnstore; +CREATE TABLE t2 (f1 DECIMAL, f2 INT, f3 DOUBLE) ENGINE=columnstore; +INSERT INTO t1 VALUES (1, 2, 3), (2, 3, 4), (3, 4, 5); +INSERT INTO t2 VALUES (4, 5, 6), (5, 6, 7), (6, 7, 8); +SELECT f1, f2, f3, f2 * f3 FROM +( +SELECT f1, f2, AVG(f3) f3 FROM +( +SELECT f1, f2, f3 FROM t1 +UNION ALL +SELECT f1, f2, f3 FROM t2 +) U +GROUP BY f1 +) V; +f1 f2 f3 f2 * f3 +1 2 3 6 +2 3 4 12 +3 4 5 20 +4 5 6 30 +5 6 7 42 +6 7 8 56 +DROP DATABASE MCOL5889; diff --git a/mysql-test/columnstore/bugfixes/MCOL-5889-double-is-not-decimal.test b/mysql-test/columnstore/bugfixes/MCOL-5889-double-is-not-decimal.test new file mode 100644 index 000000000..2786ac009 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-5889-double-is-not-decimal.test @@ -0,0 +1,24 @@ +--disable_warnings +DROP DATABASE IF EXISTS MCOL5889; +--enable_warnings +CREATE DATABASE MCOL5889; +USE MCOL5889; + +CREATE TABLE t1 (f1 DECIMAL, f2 BIGINT, f3 DOUBLE) ENGINE=columnstore; +CREATE TABLE t2 (f1 DECIMAL, f2 INT, f3 DOUBLE) ENGINE=columnstore; +INSERT INTO t1 VALUES (1, 2, 3), (2, 3, 4), (3, 4, 5); +INSERT INTO t2 VALUES (4, 5, 6), (5, 6, 7), (6, 7, 8); + +--sorted_result +SELECT f1, f2, f3, f2 * f3 FROM +( + SELECT f1, f2, AVG(f3) f3 FROM + ( + SELECT f1, f2, f3 FROM t1 + UNION ALL + SELECT f1, f2, f3 FROM t2 + ) U + GROUP BY f1 +) V; + +DROP DATABASE MCOL5889; diff --git a/tests/scripts/fullmtr.sh b/tests/scripts/fullmtr.sh index 8ed3f2133..470a03c9c 100644 --- a/tests/scripts/fullmtr.sh +++ b/tests/scripts/fullmtr.sh @@ -40,6 +40,12 @@ if (( $# == 2 )); then exit 1 fi +if (( $# == 1 )); then + run_suite $1 + exit 1 +fi + + run_suite basic run_suite bugfixes run_suite setup From aeb14af63e0fae2e79ef1d17f01c27fe8a409cd4 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 27 Jan 2025 13:28:06 +0000 Subject: [PATCH 39/65] move upgrade tests to the last phase --- .drone.jsonnet | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index f23b16f65..0334bcbd3 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -352,7 +352,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') }, upgrade(version):: { name: 'upgrade-test from ' + version, - depends_on: ['smoke'], + depends_on: ['regressionlog'], image: 'docker', failure: 'ignore', volumes: [pipeline._volumes.docker], @@ -880,14 +880,14 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') [pipeline.cmapitest] + [pipeline.cmapilog] + [pipeline.publish('cmapilog')] + - [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + - (if (std.length(mdb_server_versions) == 0) then [] else [pipeline.upgradelog] + [pipeline.publish('upgradelog')]) + (if (platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish('mtr')] + [pipeline.mtrlog] + [pipeline.publish('mtrlog')]) + (if (event == 'cron' && platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.publish('mtr latest', 'latest')] else []) + [pipeline.prepare_regression] + [pipeline.regression(regression_tests[i], [if (i == 0) then 'prepare regression' else regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + [pipeline.publish('regressionlog')] + + [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + + (if (std.length(mdb_server_versions) == 0) then [] else [pipeline.upgradelog] + [pipeline.publish('upgradelog')]) + (if (event == 'cron') then [pipeline.publish('regressionlog latest', 'latest')] else []), volumes: [pipeline._volumes.mdb { temp: {} }, pipeline._volumes.docker { host: { path: '/var/run/docker.sock' } }], From c0fade0c555de2bc621d20893d6eb86c8c9a2e13 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 27 Jan 2025 16:53:14 +0000 Subject: [PATCH 40/65] chore(ci): add more verbose for upgrade --- core_dumps/upgrade_setup_deb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core_dumps/upgrade_setup_deb.sh b/core_dumps/upgrade_setup_deb.sh index 644a4b422..82c7b131b 100755 --- a/core_dumps/upgrade_setup_deb.sh +++ b/core_dumps/upgrade_setup_deb.sh @@ -20,7 +20,7 @@ wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O chmod +x mariadb_es_repo_setup bash -c "./mariadb_es_repo_setup --token=${UPGRADE_TOKEN} --apply --mariadb-server-version=${VERSION} --skip-maxscale --skip-tools" apt update --yes -apt install --yes mariadb-server mariadb-client mariadb-plugin-columnstore +apt install --yes -oDebug::RunScripts=1 mariadb-server mariadb-client mariadb-plugin-columnstore systemctl start mariadb systemctl start mariadb-columnstore @@ -49,7 +49,7 @@ bash -c "./setup-repo.sh" # the -o options are used to make choise of keep your currently-installed version without interactive prompt -apt-get --yes --with-new-pkgs -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade +apt-get --yes --with-new-pkgs -oDebug::RunScripts=1 -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade UPGRADED_VERSION=$(mariadb -e "select @@version;") From 6001db44ab738fca32be543851f2101f8df91c75 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Thu, 20 Feb 2025 16:33:28 +0000 Subject: [PATCH 41/65] fix(BRM): MCOL-5879 DBRM::clearShm runs crit sections w/o sync mechanism (#3390) --- dbcon/mysql/ha_mcs_partition.cpp | 6 +++--- dbcon/mysql/is_columnstore_extents.cpp | 18 ++---------------- dbcon/mysql/is_columnstore_files.cpp | 2 +- versioning/BRM/dbrm.h | 8 ++++---- versioning/BRM/extentmap.h | 13 +++++++++++++ versioning/BRM/mastersegmenttable.h | 6 ++++++ 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index bfb677718..0a4538e74 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -330,7 +330,7 @@ void partitionByValue_common(UDF_ARGS* args, // inp string functionName) // input { // identify partitions by the range - BRM::DBRM::refreshShm(); + BRM::DBRM::refreshShmWithLock(); DBRM em; vector entries; vector::iterator iter; @@ -575,7 +575,7 @@ extern "C" const char* calshowpartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { - BRM::DBRM::refreshShm(); + BRM::DBRM::refreshShmWithLock(); DBRM em; vector entries; vector::iterator iter; @@ -1170,7 +1170,7 @@ extern "C" const char* calshowpartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { - BRM::DBRM::refreshShm(); + BRM::DBRM::refreshShmWithLock(); DBRM em; vector entries; vector::iterator iter; diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index c12793cbf..b8f0cacf0 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -196,27 +196,13 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th return 0; } -struct refresher -{ - BRM::DBRM* guarded; - refresher() - { - guarded = new BRM::DBRM(); - } - ~refresher() - { - delete guarded; - BRM::DBRM::refreshShm(); - } -}; static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) { BRM::OID_t cond_oid = 0; TABLE* table = tables->table; - BRM::DBRM* emp; - refresher shmRefresher; - emp = shmRefresher.guarded; + BRM::DBRM::refreshShmWithLock(); + BRM::DBRM* emp = new BRM::DBRM(); if (!emp || !emp->isDBRMReady()) { diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index 5c3ea2d12..11f64e1d2 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -215,7 +215,7 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) { - BRM::DBRM::refreshShm(); + BRM::DBRM::refreshShmWithLock(); BRM::DBRM* emp = new BRM::DBRM(); BRM::OID_t cond_oid = 0; TABLE* table = tables->table; diff --git a/versioning/BRM/dbrm.h b/versioning/BRM/dbrm.h index eb5dc557e..ed2298ca6 100644 --- a/versioning/BRM/dbrm.h +++ b/versioning/BRM/dbrm.h @@ -101,11 +101,11 @@ class DBRM EXPORT DBRM(bool noBRMFcns = false); EXPORT ~DBRM(); - EXPORT static void refreshShm() + static void refreshShmWithLock() { - MasterSegmentTableImpl::refreshShm(); - ExtentMapRBTreeImpl::refreshShm(); - FreeListImpl::refreshShm(); + MasterSegmentTableImpl::refreshShmWithLock(); + ExtentMapRBTreeImpl::refreshShmWithLock(); + FreeListImpl::refreshShmWithLock(); } // @bug 1055+ - Added functions below for multiple files per OID enhancement. diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index cdfa738a4..96e5f4b40 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -265,6 +265,12 @@ class ExtentMapRBTreeImpl static ExtentMapRBTreeImpl* makeExtentMapRBTreeImpl(unsigned key, off_t size, bool readOnly = false); + static void refreshShmWithLock() + { + boost::mutex::scoped_lock lk(fInstanceMutex); + return refreshShm(); + } + static void refreshShm() { if (fInstance) @@ -317,6 +323,13 @@ class FreeListImpl ~FreeListImpl(){}; static FreeListImpl* makeFreeListImpl(unsigned key, off_t size, bool readOnly = false); + + static void refreshShmWithLock() + { + boost::mutex::scoped_lock lk(fInstanceMutex); + return refreshShm(); + } + static void refreshShm() { if (fInstance) diff --git a/versioning/BRM/mastersegmenttable.h b/versioning/BRM/mastersegmenttable.h index 2644a81b8..06bc1d9ae 100644 --- a/versioning/BRM/mastersegmenttable.h +++ b/versioning/BRM/mastersegmenttable.h @@ -61,6 +61,12 @@ class MasterSegmentTableImpl ~MasterSegmentTableImpl(){}; static MasterSegmentTableImpl* makeMasterSegmentTableImpl(int key, int size); + static void refreshShmWithLock() + { + boost::mutex::scoped_lock lk(fInstanceMutex); + return refreshShm(); + } + static void refreshShm() { if (fInstance) From 5556d818f80d17e8c307cce4e5d7861514ea0bae Mon Sep 17 00:00:00 2001 From: Aleksei Antipovskii Date: Thu, 12 Sep 2024 12:27:02 +0200 Subject: [PATCH 42/65] chore(codestyle): mark virtual methods as override --- datatypes/mcs_data_condition.h | 2 +- datatypes/mcs_datatype.h | 57 +- datatypes/mcs_decimal.h | 21 +- dbcon/ddlpackage/ddlpkg.h | 397 +++++------- dbcon/ddlpackage/sqlparser.h | 10 +- dbcon/ddlpackageproc/createtableprocessor.h | 2 +- dbcon/ddlpackageproc/ddlindexpopulator.h | 5 +- dbcon/ddlpackageproc/ddlpackageprocessor.h | 8 +- dbcon/ddlpackageproc/dropindexprocessor.h | 2 +- dbcon/ddlpackageproc/droppartitionprocessor.h | 2 +- dbcon/ddlpackageproc/droptableprocessor.h | 4 +- dbcon/ddlpackageproc/markpartitionprocessor.h | 2 +- .../restorepartitionprocessor.h | 2 +- dbcon/dmlpackage/calpontdmlpackage.h | 4 +- dbcon/dmlpackage/commanddmlpackage.h | 12 +- dbcon/dmlpackage/deletedmlpackage.h | 12 +- dbcon/dmlpackage/dmlcolumn.h | 12 +- dbcon/dmlpackage/dmlpkg.h | 106 ++-- dbcon/dmlpackage/dmltable.h | 6 +- dbcon/dmlpackage/insertdmlpackage.h | 12 +- dbcon/dmlpackage/row.h | 6 +- dbcon/dmlpackage/updatedmlpackage.h | 12 +- dbcon/dmlpackage/vendordmlstatement.h | 13 +- dbcon/dmlpackageproc/autoincrementdata.h | 3 +- dbcon/execplan/aggregatecolumn.h | 52 +- dbcon/execplan/arithmeticcolumn.h | 60 +- dbcon/execplan/arithmeticoperator.h | 46 +- dbcon/execplan/blocksize.h | 2 +- dbcon/execplan/calpontselectexecutionplan.cpp | 80 ++- dbcon/execplan/calpontselectexecutionplan.h | 47 +- dbcon/execplan/calpontsystemcatalog.h | 13 +- dbcon/execplan/clientrotator.h | 6 +- dbcon/execplan/constantcolumn.h | 102 +-- dbcon/execplan/constantfilter.h | 26 +- dbcon/execplan/existsfilter.h | 18 +- dbcon/execplan/expressionparser.h | 4 +- dbcon/execplan/filter.h | 26 +- dbcon/execplan/functioncolumn.h | 69 +- dbcon/execplan/groupconcatcolumn.h | 18 +- dbcon/execplan/intervalcolumn.h | 16 +- dbcon/execplan/jsonarrayaggcolumn.h | 18 +- dbcon/execplan/logicoperator.h | 22 +- dbcon/execplan/mcsanalyzetableexecutionplan.h | 10 +- dbcon/execplan/objectreader.h | 8 +- dbcon/execplan/operator.h | 26 +- dbcon/execplan/outerjoinonfilter.h | 19 +- dbcon/execplan/parsetree.h | 48 +- dbcon/execplan/predicateoperator.h | 20 +- dbcon/execplan/pseudocolumn.h | 18 +- dbcon/execplan/returnedcolumn.h | 24 +- dbcon/execplan/rowcolumn.h | 34 +- dbcon/execplan/selectfilter.h | 20 +- dbcon/execplan/sessionmanager.h | 5 +- dbcon/execplan/sessionmonitor.h | 4 +- dbcon/execplan/simplecolumn.h | 50 +- dbcon/execplan/simplecolumn_decimal.h | 28 +- dbcon/execplan/simplecolumn_int.h | 36 +- dbcon/execplan/simplecolumn_uint.h | 30 +- dbcon/execplan/simplefilter.h | 33 +- dbcon/execplan/simplescalarfilter.h | 20 +- dbcon/execplan/treenode.h | 37 +- dbcon/execplan/treenodeimpl.h | 23 +- dbcon/execplan/udafcolumn.h | 18 +- dbcon/execplan/wf_frame.h | 22 +- dbcon/execplan/windowfunctioncolumn.h | 44 +- dbcon/joblist/bandeddl.h | 8 +- dbcon/joblist/bucketdl.h | 2 +- dbcon/joblist/columncommand-jl.h | 15 +- dbcon/joblist/crossenginestep.h | 73 ++- dbcon/joblist/datalist.h | 24 +- dbcon/joblist/datalistimpl.h | 30 +- dbcon/joblist/dictstep-jl.h | 21 +- dbcon/joblist/diskjoinstep.h | 10 +- dbcon/joblist/distributedenginecomm.h | 4 +- dbcon/joblist/elementtype.h | 32 +- dbcon/joblist/errorinfo.h | 3 +- dbcon/joblist/expressionstep.h | 22 +- dbcon/joblist/fifo.h | 38 +- dbcon/joblist/filtercommand-jl.h | 18 +- dbcon/joblist/groupconcat.h | 53 +- dbcon/joblist/jlf_common.h | 19 +- dbcon/joblist/joblist.h | 11 +- dbcon/joblist/jobstep.h | 33 +- dbcon/joblist/jsonarrayagg.h | 63 +- dbcon/joblist/largedatalist.h | 4 +- dbcon/joblist/largehashjoin.h | 4 +- dbcon/joblist/limitedorderby.h | 8 +- dbcon/joblist/passthrucommand-jl.h | 21 +- dbcon/joblist/primitivemsg.h | 11 +- dbcon/joblist/primitivestep.h | 227 ++++--- dbcon/joblist/pseudocc-jl.h | 10 +- dbcon/joblist/resourcedistributor.h | 6 +- dbcon/joblist/resourcemanager.h | 16 +- dbcon/joblist/rtscommand-jl.h | 19 +- dbcon/joblist/subquerystep.h | 41 +- dbcon/joblist/subquerytransformer.h | 7 +- dbcon/joblist/threadsafequeue.h | 23 +- dbcon/joblist/timestamp.h | 12 +- dbcon/joblist/tupleaggregatestep.h | 29 +- dbcon/joblist/tupleannexstep.h | 29 +- dbcon/joblist/tupleconstantstep.h | 55 +- dbcon/joblist/tuplehashjoin.h | 50 +- dbcon/joblist/tuplehavingstep.h | 29 +- dbcon/joblist/tupleunion.h | 36 +- dbcon/joblist/unique32generator.h | 8 +- dbcon/joblist/virtualtable.h | 4 +- dbcon/joblist/windowfunctionstep.h | 29 +- dbcon/mysql/ha_mcs.h | 30 +- dbcon/mysql/ha_mcs_datatype.h | 19 +- dbcon/mysql/ha_mcs_impl_if.h | 16 +- dbcon/mysql/ha_mcs_pushdown.h | 6 +- dbcon/mysql/ha_subquery.h | 30 +- dbcon/mysql/ha_view.h | 6 +- dbcon/mysql/sm.h | 27 +- ddlproc/ddlprocessor.h | 6 +- dmlproc/batchinsertprocessor.h | 3 +- primitives/blockcache/blockrequestprocessor.h | 2 +- primitives/primproc/activestatementcounter.h | 7 +- primitives/primproc/batchprimitiveprocessor.h | 4 +- primitives/primproc/bppseeder.h | 6 +- primitives/primproc/bppsendthread.h | 22 +- primitives/primproc/columncommand.h | 27 +- primitives/primproc/dictstep.h | 30 +- primitives/primproc/filtercommand.h | 36 +- primitives/primproc/passthrucommand.h | 22 +- primitives/primproc/primitiveserver.cpp | 96 ++- primitives/primproc/primitiveserver.h | 8 +- .../primproc/primitiveserverthreadpools.h | 3 +- primitives/primproc/primproc.h | 4 +- primitives/primproc/pseudocc.h | 14 +- primitives/primproc/rssmonfcn.h | 4 +- primitives/primproc/rtscommand.h | 26 +- primitives/primproc/serviceexemgr.h | 592 +++++++++--------- primitives/primproc/umsocketselector.h | 15 +- storage-manager/src/AppendTask.cpp | 4 +- storage-manager/src/AppendTask.h | 8 +- storage-manager/src/Cache.h | 4 +- storage-manager/src/CloudStorage.h | 7 +- storage-manager/src/Config.h | 10 +- storage-manager/src/CopyTask.cpp | 6 +- storage-manager/src/CopyTask.h | 8 +- storage-manager/src/Downloader.cpp | 31 +- storage-manager/src/Downloader.h | 12 +- storage-manager/src/IOCoordinator.h | 4 +- storage-manager/src/ListDirectoryTask.h | 7 +- storage-manager/src/LocalStorage.h | 19 +- storage-manager/src/MetadataFile.h | 6 +- storage-manager/src/OpenTask.cpp | 11 +- storage-manager/src/OpenTask.h | 8 +- storage-manager/src/Ownership.h | 2 +- storage-manager/src/PingTask.cpp | 8 +- storage-manager/src/PingTask.h | 8 +- storage-manager/src/PosixTask.cpp | 13 +- storage-manager/src/PosixTask.h | 9 +- storage-manager/src/PrefixCache.h | 5 +- storage-manager/src/ProcessTask.h | 7 +- storage-manager/src/ReadTask.h | 8 +- storage-manager/src/Replicator.h | 4 +- storage-manager/src/S3Storage.h | 12 +- storage-manager/src/StatTask.cpp | 5 +- storage-manager/src/StatTask.h | 8 +- storage-manager/src/SyncTask.cpp | 6 +- storage-manager/src/SyncTask.h | 8 +- storage-manager/src/Synchronizer.h | 10 +- storage-manager/src/ThreadPool.h | 2 +- storage-manager/src/TruncateTask.h | 8 +- storage-manager/src/UnlinkTask.h | 8 +- storage-manager/src/Utilities.h | 14 +- storage-manager/src/WriteTask.h | 8 +- storage-manager/src/main.cpp | 32 +- utils/cacheutils/cacheutils.cpp | 11 +- utils/cloudio/SMDataFile.h | 24 +- utils/cloudio/SMFileFactory.h | 2 +- utils/cloudio/SMFileSystem.h | 24 +- utils/cloudio/sm_exceptions.h | 2 +- utils/common/any.hpp | 36 +- utils/funcexp/func_mod.cpp | 2 +- utils/funcexp/funcexpwrapper.cpp | 18 +- utils/funcexp/funcexpwrapper.h | 8 +- utils/funcexp/funchelpers.h | 58 +- utils/funcexp/functor.h | 20 +- utils/funcexp/functor_all.h | 256 ++++---- utils/funcexp/functor_bool.h | 116 ++-- utils/funcexp/functor_dtm.h | 260 ++++---- utils/funcexp/functor_export.h | 16 +- utils/funcexp/functor_int.h | 368 +++++------ utils/funcexp/functor_json.h | 265 ++++---- utils/funcexp/functor_real.h | 380 +++++------ utils/funcexp/functor_str.h | 553 +++++++--------- utils/funcexp/jsonhelpers.h | 16 +- utils/funcexp/sql_crypt.h | 12 +- utils/funcexp/timeextract.h | 1 - utils/funcexp/utf8/checked.h | 18 +- utils/funcexp/utf8/unchecked.h | 11 +- utils/idbdatafile/BufferedFile.h | 25 +- utils/idbdatafile/BufferedFileFactory.h | 2 +- utils/idbdatafile/IDBDataFile.h | 6 +- utils/idbdatafile/IDBFileSystem.h | 2 +- utils/idbdatafile/PosixFileSystem.h | 2 +- utils/idbdatafile/UnbufferedFile.h | 24 +- utils/idbdatafile/UnbufferedFileFactory.h | 2 +- utils/messageqcpp/bytestream.h | 38 +- utils/messageqcpp/compressed_iss.h | 16 +- utils/messageqcpp/inetstreamsocket.cpp | 36 +- utils/messageqcpp/inetstreamsocket.h | 58 +- utils/messageqcpp/messagequeue.h | 13 +- utils/messageqcpp/samenodepseudosocket.h | 52 +- utils/messageqcpp/socketclosed.h | 13 +- utils/querytele/QueryTeleService.h | 144 ++--- utils/querytele/querystepparms.h | 4 +- utils/querytele/querytele_types.h | 13 +- utils/querytele/queryteleclient.h | 2 +- utils/querytele/queryteleprotoimpl.cpp | 6 +- utils/querytele/queryteleprotoimpl.h | 4 +- utils/querytele/queryteleserverparms.h | 4 +- utils/querytele/telestats.h | 8 +- utils/regr/corr.h | 14 +- utils/regr/covar_pop.h | 14 +- utils/regr/covar_samp.h | 14 +- utils/regr/moda.h | 110 ++-- utils/regr/regr_avgx.h | 14 +- utils/regr/regr_avgy.h | 14 +- utils/regr/regr_count.h | 14 +- utils/regr/regr_intercept.h | 14 +- utils/regr/regr_r2.h | 14 +- utils/regr/regr_slope.h | 14 +- utils/regr/regr_sxx.h | 14 +- utils/regr/regr_sxy.h | 14 +- utils/regr/regr_syy.h | 14 +- utils/rowgroup/rowaggregation.h | 45 +- utils/rowgroup/rowgroup.h | 51 +- utils/rwlock/rwlock.h | 22 +- utils/rwlock/rwlock_local.h | 11 +- utils/udfsdk/allnull.h | 16 +- utils/udfsdk/avg_mode.h | 30 +- utils/udfsdk/avgx.h | 14 +- utils/udfsdk/distinct_count.h | 14 +- utils/udfsdk/mcsv1_udaf.h | 26 +- utils/udfsdk/median.h | 28 +- utils/udfsdk/ssq.h | 14 +- utils/udfsdk/udfsdk.h | 91 ++- utils/windowfunction/framebound.cpp | 9 +- utils/windowfunction/framebound.h | 5 +- utils/windowfunction/frameboundrange.cpp | 4 +- utils/windowfunction/frameboundrange.h | 41 +- utils/windowfunction/frameboundrow.cpp | 5 +- utils/windowfunction/frameboundrow.h | 31 +- utils/windowfunction/idborderby.cpp | 61 +- utils/windowfunction/idborderby.h | 86 ++- utils/windowfunction/wf_count.cpp | 12 +- utils/windowfunction/wf_count.h | 7 +- utils/windowfunction/wf_lead_lag.cpp | 8 +- utils/windowfunction/wf_lead_lag.h | 9 +- utils/windowfunction/wf_min_max.cpp | 10 +- utils/windowfunction/wf_min_max.h | 7 +- utils/windowfunction/wf_nth_value.cpp | 8 +- utils/windowfunction/wf_nth_value.h | 9 +- utils/windowfunction/wf_ntile.cpp | 16 +- utils/windowfunction/wf_ntile.h | 9 +- utils/windowfunction/wf_percentile.cpp | 11 +- utils/windowfunction/wf_percentile.h | 9 +- utils/windowfunction/wf_ranking.cpp | 12 +- utils/windowfunction/wf_ranking.h | 7 +- utils/windowfunction/wf_row_number.cpp | 16 +- utils/windowfunction/wf_row_number.h | 7 +- utils/windowfunction/wf_stats.cpp | 14 +- utils/windowfunction/wf_stats.h | 7 +- utils/windowfunction/wf_sum_avg.h | 7 +- utils/windowfunction/wf_udaf.h | 15 +- utils/windowfunction/windowframe.h | 3 +- utils/windowfunction/windowfunctiontype.h | 7 +- versioning/BRM/brmtypes.h | 30 +- versioning/BRM/extentmap.cpp | 19 +- versioning/BRM/extentmap.h | 25 +- writeengine/bulk/we_bulkload.h | 6 +- writeengine/bulk/we_colbufcompressed.h | 10 +- writeengine/bulk/we_colbufmgr.h | 4 +- writeengine/bulk/we_colextinf.h | 28 +- writeengine/bulk/we_colopbulk.h | 17 +- writeengine/bulk/we_columnautoinc.h | 8 +- writeengine/bulk/we_columninfo.h | 5 +- writeengine/bulk/we_columninfocompressed.h | 16 +- writeengine/bulk/we_tableinfo.h | 2 +- writeengine/bulk/we_tempxmlgendata.h | 4 +- writeengine/shared/we_brm.h | 7 +- .../shared/we_bulkrollbackfilecompressed.h | 20 +- .../we_bulkrollbackfilecompressedhdfs.h | 20 +- writeengine/shared/we_bulkrollbackmgr.h | 2 +- writeengine/shared/we_cache.cpp | 62 +- writeengine/shared/we_cache.h | 24 +- writeengine/shared/we_chunkmanager.h | 4 +- writeengine/shared/we_config.h | 12 +- writeengine/shared/we_convertor.cpp | 8 +- writeengine/shared/we_dbfileop.h | 2 +- writeengine/splitter/we_brmupdater.h | 8 +- writeengine/splitter/we_cmdargs.h | 4 +- writeengine/wrapper/we_colop.h | 16 +- writeengine/wrapper/we_colopcompress.h | 42 +- writeengine/wrapper/we_dctnrycompress.h | 35 +- writeengine/wrapper/we_tablemetadata.h | 5 +- writeengine/wrapper/writeengine.h | 21 +- writeengine/xml/we_xmlgendata.h | 8 +- writeengine/xml/we_xmljob.h | 6 +- 303 files changed, 4091 insertions(+), 4894 deletions(-) diff --git a/datatypes/mcs_data_condition.h b/datatypes/mcs_data_condition.h index c1313a174..731143ead 100644 --- a/datatypes/mcs_data_condition.h +++ b/datatypes/mcs_data_condition.h @@ -58,7 +58,7 @@ class DataCondition return mError; } - // Adjust a sigened integer of any size to the range [-absMaxVal , +absMaxVal] + // Adjust a signed integer of any size to the range [-absMaxVal , +absMaxVal] template void adjustSIntXRange(T& val, T absMaxVal) { diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index a7e869433..6e16c1f96 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -125,7 +125,7 @@ struct WidthToSIntegralType<16> : _WidthToSIntegralType<16, int128_t> { }; -void decimalPrecisionAndScale(const utils::NullString& value, int &precision, int &scale); +void decimalPrecisionAndScale(const utils::NullString& value, int& precision, int& scale); // XXX: It is assumed here that ALL TYPES have width, scale and precision. // XXX: And then some of them have the type tag itself. @@ -147,8 +147,8 @@ class TypeAttributesStd /** @brief Convenience method to get int128 from a std::string. */ - int128_t decimal128FromString(const std::string& value, bool* saturate = 0) const; - int128_t decimal128FromString(const utils::NullString& value, bool* saturate = 0) const; + int128_t decimal128FromString(const std::string& value, bool* saturate = nullptr) const; + int128_t decimal128FromString(const utils::NullString& value, bool* saturate = nullptr) const; /** @brief The method sets the legacy scale and precision of a wide decimal @@ -507,7 +507,7 @@ class SessionParam long m_timeZone; public: - SessionParam(long timeZone) : m_timeZone(timeZone) + explicit SessionParam(long timeZone) : m_timeZone(timeZone) { } long timeZone() const @@ -576,7 +576,7 @@ class SimpleValue class SimpleValueSInt64 : public SimpleValue { public: - SimpleValueSInt64(int64_t value) : SimpleValue(value, 0, 0) + explicit SimpleValueSInt64(int64_t value) : SimpleValue(value, 0, 0) { } }; @@ -584,7 +584,7 @@ class SimpleValueSInt64 : public SimpleValue class SimpleValueUInt64 : public SimpleValue { public: - SimpleValueUInt64(uint64_t value) : SimpleValue(static_cast(value), 0, 0) + explicit SimpleValueUInt64(uint64_t value) : SimpleValue(static_cast(value), 0, 0) { } }; @@ -592,7 +592,7 @@ class SimpleValueUInt64 : public SimpleValue class SimpleValueSInt128 : public SimpleValue { public: - SimpleValueSInt128(int128_t value) : SimpleValue(0, value, 0) + explicit SimpleValueSInt128(int128_t value) : SimpleValue(0, value, 0) { } }; @@ -740,7 +740,7 @@ class MinMaxPartitionInfo : public MinMaxInfo public: MinMaxPartitionInfo() : m_status(0){}; - MinMaxPartitionInfo(const BRM::EMEntry& entry); + explicit MinMaxPartitionInfo(const BRM::EMEntry& entry); void set_invalid() { m_status |= CPINVALID; @@ -859,8 +859,8 @@ class DatabaseQualifiedColumnName std::string m_column; public: - DatabaseQualifiedColumnName(const std::string& db, const std::string& table, const std::string& column) - : m_db(db), m_table(table), m_column(column) + DatabaseQualifiedColumnName(std::string db, std::string table, std::string column) + : m_db(std::move(db)), m_table(std::move(table)), m_column(std::move(column)) { } const std::string& db() const @@ -880,9 +880,8 @@ class DatabaseQualifiedColumnName class StoreField { public: - virtual ~StoreField() - { - } + virtual ~StoreField() = default; + virtual int32_t colWidth() const = 0; virtual int32_t precision() const = 0; virtual int32_t scale() const = 0; @@ -910,9 +909,8 @@ class StoreField class WriteBatchField { public: - virtual ~WriteBatchField() - { - } + virtual ~WriteBatchField() = default; + virtual size_t ColWriteBatchDate(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; virtual size_t ColWriteBatchDatetime(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; virtual size_t ColWriteBatchTime(const unsigned char* buf, bool nullVal, ColBatchWriter& ci) = 0; @@ -957,9 +955,8 @@ class TypeHandler public: static const TypeHandler* find(SystemCatalog::ColDataType typeCode, const TypeAttributesStd& attr); static const TypeHandler* find_by_ddltype(const ddlpackage::ColumnType& ct); - virtual ~TypeHandler() - { - } + virtual ~TypeHandler() = default; + virtual const string& name() const = 0; virtual const string print(const TypeAttributesStd& attr) const { @@ -985,11 +982,11 @@ class TypeHandler const SimpleColumnParam& prm) const = 0; virtual SimpleValue getMinValueSimple() const { - return SimpleValue(std::numeric_limits::min(), std::numeric_limits::min(), 0); + return {std::numeric_limits::min(), std::numeric_limits::min(), 0}; } virtual SimpleValue getMaxValueSimple() const { - return SimpleValue(std::numeric_limits::max(), std::numeric_limits::max(), 0); + return {std::numeric_limits::max(), std::numeric_limits::max(), 0}; } virtual SimpleValue toSimpleValue(const SessionParam& sp, const TypeAttributesStd& attr, const char* str, round_style_t& rf) const = 0; @@ -1052,18 +1049,18 @@ class TypeHandlerBit : public TypeHandler const SimpleColumnParam& prm) const override { idbassert(0); - return NULL; + return nullptr; } SimpleValue toSimpleValue(const SessionParam& sp, const TypeAttributesStd& attr, const char* str, round_style_t& rf) const override { idbassert(0); - return SimpleValue(); + return {}; } boost::any getNullValueForType(const TypeAttributesStd& attr) const override { // TODO: How to communicate with write engine? - return boost::any(); + return {}; } boost::any convertFromString(const TypeAttributesStd& colType, const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; @@ -1808,11 +1805,11 @@ class TypeHandlerSDecimal128 : public TypeHandlerXDecimal } SimpleValue getMinValueSimple() const override { - return SimpleValue(std::numeric_limits::min(), datatypes::minInt128, 0); + return {std::numeric_limits::min(), datatypes::minInt128, 0}; } SimpleValue getMaxValueSimple() const override { - return SimpleValue(std::numeric_limits::max(), datatypes::maxInt128, 0); + return {std::numeric_limits::max(), datatypes::maxInt128, 0}; } MinMaxInfo widenMinMaxInfo(const TypeAttributesStd& attr, const MinMaxInfo& a, const MinMaxInfo& b) const override @@ -1914,7 +1911,7 @@ class TypeHandlerReal : public TypeHandler SimpleValue toSimpleValue(const SessionParam& sp, const TypeAttributesStd& attr, const char* str, round_style_t& rf) const override { - return SimpleValue(); // QQ: real types were not handled in IDB_format() + return {}; // QQ: real types were not handled in IDB_format() } std::string format(const SimpleValue& v, const TypeAttributesStd& attr) const override { @@ -2047,13 +2044,13 @@ class TypeHandlerSLongDouble : public TypeHandlerReal boost::any getNullValueForType(const TypeAttributesStd& attr) const override { // QQ: DDLPackageProcessor::getNullValueForType() did not handle LONGDOUBLE - return boost::any(); + return {}; } boost::any convertFromString(const TypeAttributesStd& colType, const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override { throw logging::QueryDataExcept("convertColumnData: unknown column data type.", logging::dataTypeErr); - return boost::any(); + return {}; } const uint8_t* getEmptyValueForType(const TypeAttributesStd& attr) const override { @@ -2255,7 +2252,7 @@ class TypeHandlerClob : public TypeHandlerStr } boost::any getNullValueForType(const TypeAttributesStd& attr) const override { - return boost::any(); // QQ + return {}; // QQ } boost::any convertFromString(const TypeAttributesStd& colType, const ConvertFromStringParam& prm, const std::string& str, bool& pushWarning) const override; diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 79988935f..6326bf03c 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -300,9 +300,9 @@ struct lldiv_t_128 inline lldiv_t_128 lldiv128(const int128_t& dividend, const int128_t& divisor) { if (UNLIKELY(divisor == 0) || UNLIKELY(dividend == 0)) - return lldiv_t_128(); + return {}; - return lldiv_t_128(dividend / divisor, dividend % divisor); + return {dividend / divisor, dividend % divisor}; } // TODO: derive it from TSInt64 eventually @@ -381,9 +381,8 @@ class TDecimal128 : public TSInt128 } public: - TDecimal128() - { - } + TDecimal128() = default; + explicit TDecimal128(const int128_t val) : TSInt128(val) { } @@ -541,11 +540,6 @@ class Decimal : public TDecimal128, public TDecimal64 return TSInt128(s128Value); } - inline TFloat128 toTFloat128() const - { - return TFloat128(s128Value); - } - inline double toDouble() const { int128_t scaleDivisor; @@ -554,7 +548,7 @@ class Decimal : public TDecimal128, public TDecimal64 return static_cast(tmpval); } - inline operator double() const + inline explicit operator double() const { return toDouble(); } @@ -567,7 +561,7 @@ class Decimal : public TDecimal128, public TDecimal64 return static_cast(tmpval); } - inline operator float() const + inline explicit operator float() const { return toFloat(); } @@ -580,7 +574,7 @@ class Decimal : public TDecimal128, public TDecimal64 return static_cast(tmpval); } - inline operator long double() const + inline explicit operator long double() const { return toLongDouble(); } @@ -1016,7 +1010,6 @@ struct NoOverflowCheck { void operator()(const int128_t& x, const int128_t& y) { - return; } }; diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index e1564c4cb..8d49466b9 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -349,11 +349,9 @@ enum DDL_SERIAL_TYPE */ struct SchemaObject { - virtual ~SchemaObject() - { - } + virtual ~SchemaObject() = default; - SchemaObject(std::string name) : fName(name) + explicit SchemaObject(std::string name) : fName(name) { } @@ -418,9 +416,7 @@ struct SqlStatement */ struct SqlStatementList { - SqlStatementList() - { - } + SqlStatementList() = default; SqlStatement* operator[](int i) const { @@ -452,17 +448,13 @@ struct QualifiedName /** @brief Serialize to ByteStream */ EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - QualifiedName() - { - } + QualifiedName() = default; - EXPORT QualifiedName(const char* name); + EXPORT explicit QualifiedName(const char* name); EXPORT QualifiedName(const char* name, const char* schema); EXPORT QualifiedName(const char* name, const char* schema, const char* catalog); - virtual ~QualifiedName() - { - } + virtual ~QualifiedName() = default; std::string fCatalog; std::string fName; @@ -479,7 +471,7 @@ struct TableDef : public SchemaObject /** @brief Serialize to ByteStream */ EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - TableDef() : fQualifiedName(0) + TableDef() : fQualifiedName(nullptr) { } @@ -495,7 +487,7 @@ struct TableDef : public SchemaObject { } - EXPORT virtual ~TableDef(); + EXPORT ~TableDef() override; QualifiedName* fQualifiedName; @@ -512,22 +504,22 @@ struct TableDef : public SchemaObject struct CreateTableStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ EXPORT CreateTableStatement(); /** @brief You can't have a CreateTableStatement without a table defintion */ - EXPORT CreateTableStatement(TableDef* tableDef); + EXPORT explicit CreateTableStatement(TableDef* tableDef); - EXPORT virtual ~CreateTableStatement(); + EXPORT ~CreateTableStatement() override; /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; std::string schemaName() const { @@ -561,13 +553,9 @@ struct AlterTableAction EXPORT virtual int serialize(messageqcpp::ByteStream& bs) = 0; /** @brief Ctor for deserialization */ - AlterTableAction() - { - } + AlterTableAction() = default; - virtual ~AlterTableAction() - { - } + virtual ~AlterTableAction() = default; /** @brief QualifiedName of the focal table for this statement. */ @@ -582,24 +570,24 @@ struct AlterTableAction struct AtaAddColumn : public AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaAddColumn() : fColumnDef(0) + AtaAddColumn() : fColumnDef(nullptr) { } /** @brief You can't add a column without specifying a column definition. */ - AtaAddColumn(ColumnDef* columnDef); + explicit AtaAddColumn(ColumnDef* columnDef); - virtual ~AtaAddColumn(); + ~AtaAddColumn() override; /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief The focal column definition. */ ColumnDef* fColumnDef; @@ -611,22 +599,20 @@ struct AtaAddColumn : public AlterTableAction struct AtaAddColumns : public AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaAddColumns() - { - } + AtaAddColumns() = default; - AtaAddColumns(TableElementList* tableElements); + explicit AtaAddColumns(TableElementList* tableElements); - virtual ~AtaAddColumns(); + ~AtaAddColumns() override; /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; ColumnDefList fColumns; }; @@ -637,22 +623,20 @@ struct AtaAddColumns : public AlterTableAction struct AtaDropColumns : public AlterTableAction { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaDropColumns() - { - } + AtaDropColumns() = default; - EXPORT AtaDropColumns(ColumnNameList* tableElements); + EXPORT explicit AtaDropColumns(ColumnNameList* tableElements); - EXPORT virtual ~AtaDropColumns(); + EXPORT ~AtaDropColumns() override; /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; ColumnNameList fColumns; }; @@ -662,22 +646,22 @@ struct AtaDropColumns : public AlterTableAction struct AtaAddTableConstraint : public AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaAddTableConstraint() : fTableConstraint(0) + AtaAddTableConstraint() : fTableConstraint(nullptr) { } - AtaAddTableConstraint(TableConstraintDef* tableConstraint); + explicit AtaAddTableConstraint(TableConstraintDef* tableConstraint); - virtual ~AtaAddTableConstraint(); + ~AtaAddTableConstraint() override; /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; TableConstraintDef* fTableConstraint; }; @@ -687,25 +671,21 @@ struct AtaAddTableConstraint : public AlterTableAction struct AtaDropColumn : public AlterTableAction { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaDropColumn() - { - } + AtaDropColumn() = default; /** @brief Ctor for parser construction */ EXPORT AtaDropColumn(std::string columnName, DDL_REFERENTIAL_ACTION dropBehavior); /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; - virtual ~AtaDropColumn() - { - } + ~AtaDropColumn() override = default; std::string fColumnName; DDL_REFERENTIAL_ACTION fDropBehavior; }; @@ -715,19 +695,19 @@ struct AtaDropColumn : public AlterTableAction struct AtaSetColumnDefault : AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; - AtaSetColumnDefault() : fDefaultValue(0) + AtaSetColumnDefault() : fDefaultValue(nullptr) { } /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~AtaSetColumnDefault(); + ~AtaSetColumnDefault() override; AtaSetColumnDefault(const char* colName, ColumnDefaultValue* defaultValue); @@ -739,25 +719,21 @@ struct AtaSetColumnDefault : AlterTableAction struct AtaDropColumnDefault : AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaDropColumnDefault() - { - } + AtaDropColumnDefault() = default; /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~AtaDropColumnDefault() - { - } + ~AtaDropColumnDefault() override = default; /** @brief Ctor for parser construction */ - AtaDropColumnDefault(const char* colName); + explicit AtaDropColumnDefault(const char* colName); std::string fColumnName; }; @@ -766,22 +742,18 @@ struct AtaDropColumnDefault : AlterTableAction struct AtaDropTableConstraint : AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaDropTableConstraint() - { - } + AtaDropTableConstraint() = default; /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~AtaDropTableConstraint() - { - } + ~AtaDropTableConstraint() override = default; AtaDropTableConstraint(const char* constraintName, DDL_REFERENTIAL_ACTION dropBehavior); @@ -793,21 +765,21 @@ struct AtaDropTableConstraint : AlterTableAction struct AtaRenameTable : public AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaRenameTable() : fQualifiedName(0) + AtaRenameTable() : fQualifiedName(nullptr) { } - AtaRenameTable(QualifiedName* qualifiedName); + explicit AtaRenameTable(QualifiedName* qualifiedName); /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~AtaRenameTable(); + ~AtaRenameTable() override; QualifiedName* fQualifiedName; }; @@ -816,21 +788,21 @@ struct AtaRenameTable : public AlterTableAction struct AtaTableComment : public AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ AtaTableComment() : fTableComment("") { } - AtaTableComment(const char* tableComment); + explicit AtaTableComment(const char* tableComment); /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~AtaTableComment(); + ~AtaTableComment() override; std::string fTableComment; }; @@ -839,13 +811,13 @@ struct AtaTableComment : public AlterTableAction struct AtaModifyColumnType : public AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaModifyColumnType() : fColumnType(0) + AtaModifyColumnType() : fColumnType(nullptr) { } @@ -854,12 +826,12 @@ struct AtaModifyColumnType : public AlterTableAction { } - AtaModifyColumnType(QualifiedName* qualifiedName); + explicit AtaModifyColumnType(QualifiedName* qualifiedName); /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~AtaModifyColumnType(); + ~AtaModifyColumnType() override; ColumnType* fColumnType; @@ -870,28 +842,28 @@ struct AtaModifyColumnType : public AlterTableAction struct AtaRenameColumn : public AlterTableAction { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - AtaRenameColumn() : fNewType(0), fDefaultValue(0) + AtaRenameColumn() : fNewType(nullptr), fDefaultValue(nullptr) { } - AtaRenameColumn(const char* name, const char* newName, ColumnType* newType, const char* comment = NULL) + AtaRenameColumn(const char* name, const char* newName, ColumnType* newType, const char* comment = nullptr) : fName(name), fNewName(newName), fNewType(newType) { if (comment) fComment = comment; - fDefaultValue = 0; + fDefaultValue = nullptr; } AtaRenameColumn(const char* name, const char* newName, ColumnType* newType, ColumnConstraintList* constraint_list, ColumnDefaultValue* defaultValue, - const char* comment = NULL) + const char* comment = nullptr) : fName(name), fNewName(newName), fNewType(newType), fDefaultValue(defaultValue) { if (constraint_list) @@ -906,12 +878,12 @@ struct AtaRenameColumn : public AlterTableAction fComment = comment; } - AtaRenameColumn(QualifiedName* qualifiedName); + explicit AtaRenameColumn(QualifiedName* qualifiedName); /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~AtaRenameColumn(); + ~AtaRenameColumn() override; std::string fName; ///< current column name std::string fNewName; ///< new column name @@ -935,7 +907,7 @@ struct ColumnType EXPORT int serialize(messageqcpp::ByteStream& bs); /** @brief For deserialization. */ - ColumnType() : fCharset(NULL), fCollate(NULL), fCharsetNum(0), fExplicitLength(false) + ColumnType() : fCharset(nullptr), fCollate(nullptr), fCharsetNum(0), fExplicitLength(false) { } @@ -951,9 +923,7 @@ struct ColumnType EXPORT ColumnType(int type); - virtual ~ColumnType() - { - } + virtual ~ColumnType() = default; /** @brief Type code from DDL_DATATYPES */ int fType; @@ -1003,19 +973,15 @@ struct ColumnConstraintDef : public SchemaObject /** @brief Serialize to ByteStream */ EXPORT virtual int serialize(messageqcpp::ByteStream& bs); - ColumnConstraintDef() - { - } + ColumnConstraintDef() = default; /** @brief Constructs as check constraint. */ - EXPORT ColumnConstraintDef(const char* check); + EXPORT explicit ColumnConstraintDef(const char* check); /** @brief Constructs as other constraint. */ - EXPORT ColumnConstraintDef(DDL_CONSTRAINTS type); + EXPORT explicit ColumnConstraintDef(DDL_CONSTRAINTS type); - virtual ~ColumnConstraintDef() - { - } + ~ColumnConstraintDef() override = default; /** @brief Whether deferrable. */ bool fDeferrable; @@ -1039,15 +1005,11 @@ struct ColumnDefaultValue /** @brief Serialize to ByteStream */ virtual int serialize(messageqcpp::ByteStream& bs); - ColumnDefaultValue() - { - } + ColumnDefaultValue() = default; - ColumnDefaultValue(const char* value); + explicit ColumnDefaultValue(const char* value); - virtual ~ColumnDefaultValue() - { - } + virtual ~ColumnDefaultValue() = default; /** @brief Is NULL the default value? */ bool fNull; @@ -1067,20 +1029,20 @@ struct ColumnDef : public SchemaObject EXPORT virtual int serialize(messageqcpp::ByteStream& bs); /** @brief For deserialization. */ - ColumnDef() : fType(0) + ColumnDef() : fType(nullptr) { } - EXPORT virtual ~ColumnDef(); + EXPORT ~ColumnDef() override; /** @brief Parser ctor. */ EXPORT ColumnDef(const char* name, ColumnType* type, ColumnConstraintList* constraint_list, - ColumnDefaultValue* defaultValue, const char* comment = NULL); + ColumnDefaultValue* defaultValue, const char* comment = nullptr); /** @brief ColumnDef ctor. * ctor */ ColumnDef(const char* name, ColumnType* type, ColumnConstraintList constraints, - ColumnDefaultValue* defaultValue = NULL, const char* comment = NULL) + ColumnDefaultValue* defaultValue = nullptr, const char* comment = nullptr) : SchemaObject(name), fType(type), fConstraints(constraints), fDefaultValue(defaultValue) { } @@ -1114,14 +1076,12 @@ struct TableConstraintDef : public SchemaObject TableConstraintDef(); - TableConstraintDef(DDL_CONSTRAINTS cType); + explicit TableConstraintDef(DDL_CONSTRAINTS cType); /** @brief Dump to stdout. */ virtual std::ostream& put(std::ostream& os) const; - virtual ~TableConstraintDef() - { - } + ~TableConstraintDef() override = default; // std::string fName; DDL_CONSTRAINTS fConstraintType; }; @@ -1131,28 +1091,26 @@ struct TableConstraintDef : public SchemaObject struct TableUniqueConstraintDef : public TableConstraintDef { /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() + DDL_SERIAL_TYPE getSerialType() override { return DDL_TABLE_UNIQUE_CONSTRAINT_DEF; } /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; TableUniqueConstraintDef() : TableConstraintDef(DDL_UNIQUE) { } - TableUniqueConstraintDef(ColumnNameList* columns); - virtual ~TableUniqueConstraintDef() - { - } + explicit TableUniqueConstraintDef(ColumnNameList* columns); + ~TableUniqueConstraintDef() override = default; /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; ColumnNameList fColumnNameList; }; @@ -1162,29 +1120,27 @@ struct TableUniqueConstraintDef : public TableConstraintDef struct TablePrimaryKeyConstraintDef : public TableConstraintDef { /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() + DDL_SERIAL_TYPE getSerialType() override { return DDL_TABLE_PRIMARY_CONSTRAINT_DEF; } /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; TablePrimaryKeyConstraintDef() : TableConstraintDef(DDL_PRIMARY_KEY) { } - EXPORT TablePrimaryKeyConstraintDef(ColumnNameList* columns); + EXPORT explicit TablePrimaryKeyConstraintDef(ColumnNameList* columns); - virtual ~TablePrimaryKeyConstraintDef() - { - } + ~TablePrimaryKeyConstraintDef() override = default; /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; ColumnNameList fColumnNameList; }; @@ -1194,9 +1150,7 @@ struct TablePrimaryKeyConstraintDef : public TableConstraintDef */ struct ReferentialAction { - virtual ~ReferentialAction() - { - } + virtual ~ReferentialAction() = default; /** @brief Deserialize from ByteStream */ virtual int unserialize(messageqcpp::ByteStream& bs); @@ -1213,18 +1167,19 @@ struct ReferentialAction struct TableReferencesConstraintDef : public TableConstraintDef { /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() + DDL_SERIAL_TYPE getSerialType() override { return DDL_TABLE_REFERENCES_CONSTRAINT_DEF; } /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; - TableReferencesConstraintDef() : TableConstraintDef(DDL_REFERENCES), fTableName(0), fRefAction(0) + TableReferencesConstraintDef() + : TableConstraintDef(DDL_REFERENCES), fTableName(nullptr), fRefAction(nullptr) { } @@ -1232,10 +1187,10 @@ struct TableReferencesConstraintDef : public TableConstraintDef ColumnNameList* foreignColumns, DDL_MATCH_TYPE matchType, ReferentialAction* refAction); - virtual ~TableReferencesConstraintDef(); + ~TableReferencesConstraintDef() override; /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; ColumnNameList fColumns; QualifiedName* fTableName; @@ -1249,29 +1204,27 @@ struct TableReferencesConstraintDef : public TableConstraintDef struct TableCheckConstraintDef : public TableConstraintDef { /** @brief Return DDL_SERIAL code */ - virtual DDL_SERIAL_TYPE getSerialType() + DDL_SERIAL_TYPE getSerialType() override { return DDL_TABLE_CHECK_CONSTRAINT_DEF; } /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; TableCheckConstraintDef() : TableConstraintDef(DDL_CHECK) { } - TableCheckConstraintDef(const char* check); + explicit TableCheckConstraintDef(const char* check); /** @brief Dump to stdout. */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~TableCheckConstraintDef() - { - } + ~TableCheckConstraintDef() override = default; std::string fCheck; }; @@ -1284,22 +1237,22 @@ struct TableCheckConstraintDef : public TableConstraintDef struct AlterTableStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; - AlterTableStatement() : fTableName(0) + AlterTableStatement() : fTableName(nullptr) { } EXPORT AlterTableStatement(QualifiedName* qName, AlterTableActionList* ataList); /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; /** @brief Delete members. */ - EXPORT virtual ~AlterTableStatement(); + EXPORT ~AlterTableStatement() override; std::string schemaName() const { @@ -1341,18 +1294,14 @@ struct ConstraintAttributes /** @brief Serialize to ByteStream */ virtual int serialize(messageqcpp::ByteStream& bs); - ConstraintAttributes() - { - } + ConstraintAttributes() = default; ConstraintAttributes(DDL_CONSTRAINT_ATTRIBUTES checkTime, bool deferrable) : fCheckTime(checkTime), fDeferrable(deferrable) { } - virtual ~ConstraintAttributes() - { - } + virtual ~ConstraintAttributes() = default; DDL_CONSTRAINT_ATTRIBUTES fCheckTime; bool fDeferrable; @@ -1366,19 +1315,19 @@ struct ConstraintAttributes struct CreateIndexStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; CreateIndexStatement(); CreateIndexStatement(QualifiedName* qualifiedName1, QualifiedName* qualifiedName2, ColumnNameList* columnNames, bool unique); /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~CreateIndexStatement(); + ~CreateIndexStatement() override; QualifiedName* fIndexName; QualifiedName* fTableName; @@ -1391,20 +1340,20 @@ struct CreateIndexStatement : public SqlStatement struct DropIndexStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - virtual int unserialize(messageqcpp::ByteStream& bs); + int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - virtual int serialize(messageqcpp::ByteStream& bs); + int serialize(messageqcpp::ByteStream& bs) override; - DropIndexStatement() : fIndexName(0) + DropIndexStatement() : fIndexName(nullptr) { } - DropIndexStatement(QualifiedName* qualifiedName); + explicit DropIndexStatement(QualifiedName* qualifiedName); /** @brief Dump to stdout. */ - std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; - virtual ~DropIndexStatement(); + ~DropIndexStatement() override; QualifiedName* fIndexName; }; @@ -1414,20 +1363,20 @@ struct DropIndexStatement : public SqlStatement struct DropTableStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; - DropTableStatement() : fTableName(0) + DropTableStatement() : fTableName(nullptr) { } EXPORT DropTableStatement(QualifiedName* qualifiedName, bool cascade); /** @brief Dump to stdout. */ - EXPORT std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; - virtual ~DropTableStatement() + ~DropTableStatement() override { delete fTableName; } @@ -1473,20 +1422,20 @@ struct DebugDDLStatement : public SqlStatement struct TruncTableStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; - TruncTableStatement() : fTableName(0) + TruncTableStatement() : fTableName(nullptr) { } - EXPORT TruncTableStatement(QualifiedName* qualifiedName); + EXPORT explicit TruncTableStatement(QualifiedName* qualifiedName); /** @brief Dump to stdout. */ - EXPORT std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; - virtual ~TruncTableStatement() + ~TruncTableStatement() override { delete fTableName; } @@ -1508,23 +1457,23 @@ struct TruncTableStatement : public SqlStatement struct MarkPartitionStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - MarkPartitionStatement() : fTableName(0) + MarkPartitionStatement() : fTableName(nullptr) { } /** @brief You can't have a CreateTableStatement without a table defintion */ - EXPORT MarkPartitionStatement(QualifiedName* qualifiedName); + EXPORT explicit MarkPartitionStatement(QualifiedName* qualifiedName); /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; - virtual ~MarkPartitionStatement() + ~MarkPartitionStatement() override { delete fTableName; } @@ -1539,22 +1488,22 @@ struct MarkPartitionStatement : public SqlStatement struct RestorePartitionStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - RestorePartitionStatement() : fTableName(0) + RestorePartitionStatement() : fTableName(nullptr) { } - EXPORT RestorePartitionStatement(QualifiedName* qualifiedName); + EXPORT explicit RestorePartitionStatement(QualifiedName* qualifiedName); /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; - virtual ~RestorePartitionStatement() + ~RestorePartitionStatement() override { delete fTableName; } @@ -1569,22 +1518,22 @@ struct RestorePartitionStatement : public SqlStatement struct DropPartitionStatement : public SqlStatement { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs) override; /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs) override; /** @brief Ctor for deserialization */ - DropPartitionStatement() : fTableName(0) + DropPartitionStatement() : fTableName(nullptr) { } - EXPORT DropPartitionStatement(QualifiedName* qualifiedName); + EXPORT explicit DropPartitionStatement(QualifiedName* qualifiedName); /** @brief Dump to stdout. */ - EXPORT virtual std::ostream& put(std::ostream& os) const; + EXPORT std::ostream& put(std::ostream& os) const override; - virtual ~DropPartitionStatement() + ~DropPartitionStatement() override { delete fTableName; } diff --git a/dbcon/ddlpackage/sqlparser.h b/dbcon/ddlpackage/sqlparser.h index 3a16ccb5c..e7e3dfaed 100644 --- a/dbcon/ddlpackage/sqlparser.h +++ b/dbcon/ddlpackage/sqlparser.h @@ -29,7 +29,7 @@ #include #include "collation.h" // CHARSET_INFO #include "ddlpkg.h" -#include "mariadb_my_sys.h" // myf, MYF() +#include "mariadb_my_sys.h" // myf, MYF() #define EXPORT @@ -86,12 +86,8 @@ struct pass_to_bison const CHARSET_INFO* default_table_charset; myf utf8_flag; - pass_to_bison(ParseTree* pt) : - fParseTree(pt) - , scanner(NULL) - , default_table_charset(NULL) - , utf8_flag(MYF(0)) - {}; + pass_to_bison(ParseTree* pt) + : fParseTree(pt), scanner(nullptr), default_table_charset(nullptr), utf8_flag(MYF(0)){}; }; class SqlParser diff --git a/dbcon/ddlpackageproc/createtableprocessor.h b/dbcon/ddlpackageproc/createtableprocessor.h index 48d14bb08..4ae32260e 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.h +++ b/dbcon/ddlpackageproc/createtableprocessor.h @@ -49,7 +49,7 @@ class CreateTableProcessor : public DDLPackageProcessor * * @param createTableStmt the CreateTableStatement */ - DDLResult processPackageInternal(ddlpackage::SqlStatement* sqlTableStmt); + DDLResult processPackageInternal(ddlpackage::SqlStatement* sqlTableStmt) override; }; } // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.h b/dbcon/ddlpackageproc/ddlindexpopulator.h index 1570d2221..52f79b318 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.h +++ b/dbcon/ddlpackageproc/ddlindexpopulator.h @@ -22,7 +22,7 @@ ***********************************************************************/ /** @file */ -#ifndef DDLINDEXPOPULATOR_H +#pragma once #include #include @@ -302,5 +302,4 @@ class DDLIndexPopulator }; }; -} // namespace ddlpackageprocessor -#endif // DDLPINDEXPOPULATOR_H +} // namespace ddlpackageprocessor \ No newline at end of file diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.h b/dbcon/ddlpackageproc/ddlpackageprocessor.h index 6d5171019..b1763f833 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.h +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.h @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include @@ -49,7 +49,7 @@ #define EXPORT -//#define IDB_DDL_DEBUG +// #define IDB_DDL_DEBUG namespace ddlpackageprocessor { #define SUMMARY_INFO(message) \ @@ -206,8 +206,8 @@ class DDLPackageProcessor struct NJLSysDataList { NJLSysDataVector sysDataVec; - EXPORT NJLSysDataList(){}; - EXPORT ~NJLSysDataList(); + EXPORT NJLSysDataList() = default; + EXPORT ~NJLSysDataList() = default; NJLSysDataVector::const_iterator begin() { return sysDataVec.begin(); diff --git a/dbcon/ddlpackageproc/dropindexprocessor.h b/dbcon/ddlpackageproc/dropindexprocessor.h index b92c569ba..ec73d77b2 100644 --- a/dbcon/ddlpackageproc/dropindexprocessor.h +++ b/dbcon/ddlpackageproc/dropindexprocessor.h @@ -38,7 +38,7 @@ class DropIndexProcessor : public DDLPackageProcessor * * @param dropIndexStmt the drop index statement */ - DDLResult processPackageInternal(ddlpackage::SqlStatement& dropIndexStmt); + DDLResult processPackageInternal(ddlpackage::SqlStatement* dropIndexStmt) override; protected: private: diff --git a/dbcon/ddlpackageproc/droppartitionprocessor.h b/dbcon/ddlpackageproc/droppartitionprocessor.h index 914df8f56..c3fa7edd9 100644 --- a/dbcon/ddlpackageproc/droppartitionprocessor.h +++ b/dbcon/ddlpackageproc/droppartitionprocessor.h @@ -46,7 +46,7 @@ class DropPartitionProcessor : public DDLPackageProcessor, FormatStatementString * * @param dropTableStmt the drop table statement */ - DDLResult processPackageInternal(ddlpackage::SqlStatement* dropPartitionStmt); + DDLResult processPackageInternal(ddlpackage::SqlStatement* dropPartitionStmt) override; }; } // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/droptableprocessor.h b/dbcon/ddlpackageproc/droptableprocessor.h index 1a86706e3..aa87d6f77 100644 --- a/dbcon/ddlpackageproc/droptableprocessor.h +++ b/dbcon/ddlpackageproc/droptableprocessor.h @@ -46,7 +46,7 @@ class DropTableProcessor : public DDLPackageProcessor * * @param dropTableStmt the drop table statement */ - DDLResult processPackageInternal(ddlpackage::SqlStatement* dropTableStmt); + DDLResult processPackageInternal(ddlpackage::SqlStatement* dropTableStmt) override; }; /** @brief specialization of a DDLPacakageProcessor @@ -66,7 +66,7 @@ class TruncTableProcessor : public DDLPackageProcessor * * @param truncTableStmt the truncate table statement */ - DDLResult processPackageInternal(ddlpackage::SqlStatement* truncTableStmt); + DDLResult processPackageInternal(ddlpackage::SqlStatement* truncTableStmt) override; }; } // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/markpartitionprocessor.h b/dbcon/ddlpackageproc/markpartitionprocessor.h index ed3454f94..5fedb70c4 100644 --- a/dbcon/ddlpackageproc/markpartitionprocessor.h +++ b/dbcon/ddlpackageproc/markpartitionprocessor.h @@ -46,7 +46,7 @@ class MarkPartitionProcessor : public DDLPackageProcessor, FormatStatementString * * @param createTableStmt the CreateTableStatement */ - DDLResult processPackageInternal(ddlpackage::SqlStatement* MarkPartitionStmt); + DDLResult processPackageInternal(ddlpackage::SqlStatement* MarkPartitionStmt) override; }; } // namespace ddlpackageprocessor diff --git a/dbcon/ddlpackageproc/restorepartitionprocessor.h b/dbcon/ddlpackageproc/restorepartitionprocessor.h index d2b6e74f9..cef47d9c4 100644 --- a/dbcon/ddlpackageproc/restorepartitionprocessor.h +++ b/dbcon/ddlpackageproc/restorepartitionprocessor.h @@ -46,7 +46,7 @@ class RestorePartitionProcessor : public DDLPackageProcessor, FormatStatementStr * * @param dropTableStmt the drop table statement */ - DDLResult processPackageInternal(ddlpackage::SqlStatement* RestorePartitionStmt); + DDLResult processPackageInternal(ddlpackage::SqlStatement* RestorePartitionStmt) override; }; } // namespace ddlpackageprocessor diff --git a/dbcon/dmlpackage/calpontdmlpackage.h b/dbcon/dmlpackage/calpontdmlpackage.h index 651c151a7..dbf5b9311 100644 --- a/dbcon/dmlpackage/calpontdmlpackage.h +++ b/dbcon/dmlpackage/calpontdmlpackage.h @@ -188,7 +188,7 @@ class CalpontDMLPackage { fTableName = tableName; - if (fTable != 0) + if (fTable != nullptr) fTable->set_TableName(tableName); } @@ -207,7 +207,7 @@ class CalpontDMLPackage { fSchemaName = schemaName; - if (fTable != 0) + if (fTable != nullptr) fTable->set_SchemaName(schemaName); } diff --git a/dbcon/dmlpackage/commanddmlpackage.h b/dbcon/dmlpackage/commanddmlpackage.h index bd2af3c5f..11bb7f413 100644 --- a/dbcon/dmlpackage/commanddmlpackage.h +++ b/dbcon/dmlpackage/commanddmlpackage.h @@ -47,33 +47,33 @@ class CommandDMLPackage : public CalpontDMLPackage /** @brief dtor */ - EXPORT virtual ~CommandDMLPackage(); + EXPORT ~CommandDMLPackage() override; /** @brief write a CommandDMLPackage to a ByteStream * * @param bytestream the ByteStream to write to */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + EXPORT int write(messageqcpp::ByteStream& bytestream) override; /** @brief read CommandDMLPackage from bytestream * * @param bytestream the ByteStream to read from */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + EXPORT int read(messageqcpp::ByteStream& bytestream) override; /** @brief do nothing * * @param buffer * @param columns the number of columns in the buffer * @param rows the number of rows in the buffer */ - inline int buildFromBuffer(std::string& buffer, int columns = 0, int rows = 0) + inline int buildFromBuffer(std::string& buffer, int columns = 0, int rows = 0) override { return 1; }; /** @brief build a CommandDMLPackage from a CommandSqlStatement */ - EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); + EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement) override; /** @brief build a InsertDMLPackage from MySQL buffer * @@ -81,7 +81,7 @@ class CommandDMLPackage : public CalpontDMLPackage * @param rows the number of rows in the buffer */ int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, - NullValuesBitset& nullValues) + NullValuesBitset& nullValues) override { return 1; }; diff --git a/dbcon/dmlpackage/deletedmlpackage.h b/dbcon/dmlpackage/deletedmlpackage.h index 3d2615592..e9e608e29 100644 --- a/dbcon/dmlpackage/deletedmlpackage.h +++ b/dbcon/dmlpackage/deletedmlpackage.h @@ -53,19 +53,19 @@ class DeleteDMLPackage : public CalpontDMLPackage /** @brief dtor */ - EXPORT virtual ~DeleteDMLPackage(); + EXPORT ~DeleteDMLPackage() override; /** @brief write a DeleteDMLPackage to a ByteStream * * @param bytestream the ByteStream to write to */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + EXPORT int write(messageqcpp::ByteStream& bytestream) override; /** @brief read a DeleteDMLPackage from a ByteStream * * @param bytestream the ByteStream to read from */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + EXPORT int read(messageqcpp::ByteStream& bytestream) override; /** @brief build a DeleteDMLPackage from a string buffer * @@ -73,20 +73,20 @@ class DeleteDMLPackage : public CalpontDMLPackage * @param columns the number of columns in the buffer * @param rows the number of rows in the buffer */ - EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); + EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows) override; /** @brief build a DeleteDMLPackage from a parsed DeleteSqlStatement * * @param sqlStatement the parsed DeleteSqlStatement */ - EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); + EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement) override; /** @brief build a InsertDMLPackage from MySQL buffer * * @param colNameList, tableValuesMap * @param rows the number of rows in the buffer */ EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, - int rows, NullValuesBitset& nullValues); + int rows, NullValuesBitset& nullValues) override; protected: private: diff --git a/dbcon/dmlpackage/dmlcolumn.h b/dbcon/dmlpackage/dmlcolumn.h index 13f2b4718..1859847b0 100644 --- a/dbcon/dmlpackage/dmlcolumn.h +++ b/dbcon/dmlpackage/dmlcolumn.h @@ -52,27 +52,27 @@ class DMLColumn : public DMLObject uint32_t funcScale = 0, bool isNULL = false); /** @brief new ctor - * + * */ - EXPORT DMLColumn(std::string name, utils::NullString& value, bool isFromCol = false, - uint32_t funcScale = 0, bool isNULL = false); + EXPORT DMLColumn(std::string name, utils::NullString& value, bool isFromCol = false, uint32_t funcScale = 0, + bool isNULL = false); /** @brief dtor */ - EXPORT ~DMLColumn(); + EXPORT ~DMLColumn() override; /** @brief read a DMLColumn from a ByteStream * * @param bytestream the ByteStream to read from */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + EXPORT int read(messageqcpp::ByteStream& bytestream) override; /** @brief write a DML column to a ByteStream * * @param bytestream the ByteStream to write to */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + EXPORT int write(messageqcpp::ByteStream& bytestream) override; /** @brief get the data for the column */ diff --git a/dbcon/dmlpackage/dmlpkg.h b/dbcon/dmlpackage/dmlpkg.h index c85bbb25f..0c4b548ef 100644 --- a/dbcon/dmlpackage/dmlpkg.h +++ b/dbcon/dmlpackage/dmlpkg.h @@ -23,13 +23,14 @@ /** @file */ #pragma once +#include #include #include #include #include #include #include -#include +#include #include "nullstring.h" namespace dmlpackage @@ -158,9 +159,7 @@ class SqlStatementList public: /** @brief ctor */ - SqlStatementList() - { - } + SqlStatementList() = default; /** @brief dtor */ @@ -218,19 +217,19 @@ class InsertSqlStatement : public SqlStatement /** @brief dtor */ - virtual ~InsertSqlStatement(); + ~InsertSqlStatement() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get a string representation of the query spec */ - virtual std::string getQueryString() const; + std::string getQueryString() const override; /** @brief get the statement type - DML_INSERT */ - inline virtual int getStatementType() const + inline int getStatementType() const override { return DML_INSERT; } @@ -259,25 +258,25 @@ class UpdateSqlStatement : public SqlStatement * @param whereClausePtr pointer to a WhereClause object - default 0 */ UpdateSqlStatement(TableName* tableNamePtr, ColumnAssignmentList* colAssignmentListPtr, - WhereClause* whereClausePtr = 0); + WhereClause* whereClausePtr = nullptr); /** @brief dtor */ - virtual ~UpdateSqlStatement(); + ~UpdateSqlStatement() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the * SET assignment_commalist opt_where_clause * statement */ - virtual std::string getQueryString() const; + std::string getQueryString() const override; /** @brief get the statement type - DML_UPDATE */ - inline virtual int getStatementType() const + inline int getStatementType() const override { return DML_UPDATE; } @@ -304,23 +303,23 @@ class DeleteSqlStatement : public SqlStatement * @param tableNamePtr pointer to a TableName object * @param whereClausePtr pointer to a WhereClause object - default = 0 */ - DeleteSqlStatement(TableName* tableNamePtr, WhereClause* whereClausePtr = 0); + explicit DeleteSqlStatement(TableName* tableNamePtr, WhereClause* whereClausePtr = nullptr); /** @brief dtor */ - virtual ~DeleteSqlStatement(); + ~DeleteSqlStatement() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the WHERE clause */ - virtual std::string getQueryString() const; + std::string getQueryString() const override; /** @brief get the statement type - DML_DELETE */ - inline virtual int getStatementType() const + inline int getStatementType() const override { return DML_DELETE; } @@ -346,22 +345,22 @@ class CommandSqlStatement : public SqlStatement * * @param command the COMMIT or ROLLBACK string */ - CommandSqlStatement(std::string command); + explicit CommandSqlStatement(std::string command); /** @brief get the statement type - DML_COMMAND */ - inline virtual int getStatementType() const + inline int getStatementType() const override { return DML_COMMAND; } /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the COMMIT or ROLLBACK string */ - virtual std::string getQueryString() const; + std::string getQueryString() const override; std::string fCommandText; }; @@ -380,7 +379,7 @@ class TableName * * @param name the table name */ - TableName(char* name); + explicit TableName(char* name); /** @brief ctor * @@ -406,11 +405,10 @@ class TableName class ColumnAssignment { public: - explicit ColumnAssignment(std::string const& column, std::string const& op = "=", - std::string const& expr = "") - : fColumn(column) - , fOperator(op) - , fScalarExpression(expr) + explicit ColumnAssignment(std::string column, std::string op = "=", std::string expr = "") + : fColumn(std::move(column)) + , fOperator(std::move(op)) + , fScalarExpression(std::move(expr)) , fFromCol(false) , fFuncScale(0) , fIsNull(false){}; @@ -449,13 +447,13 @@ class ValuesOrQuery * * @param valuesPtr pointer to a ValuesList object */ - ValuesOrQuery(ValuesList* valuesPtr); + explicit ValuesOrQuery(ValuesList* valuesPtr); /** @brief ctor * * @param querySpecPtr pointer to a QuerySpec object */ - ValuesOrQuery(QuerySpec* querySpecPtr); + explicit ValuesOrQuery(QuerySpec* querySpecPtr); /** @brief dtor */ @@ -491,7 +489,7 @@ class SelectFilter * * @param columnListPtr pointer to a ColumnNameList object */ - SelectFilter(ColumnNameList* columnListPtr); + explicit SelectFilter(ColumnNameList* columnListPtr); /** @brief dtor */ @@ -524,7 +522,7 @@ class FromClause * * @param tableNameList pointer to a TableNameList object */ - FromClause(TableNameList* tableNameList); + explicit FromClause(TableNameList* tableNameList); /** @brief dtor */ @@ -663,7 +661,7 @@ class Predicate * * @param predicateType the PREDICATE_TYPE */ - Predicate(PREDICATE_TYPE predicateType); + explicit Predicate(PREDICATE_TYPE predicateType); /** @brief dtor */ @@ -696,16 +694,16 @@ class ComparisonPredicate : public Predicate /** @brief dtor */ - ~ComparisonPredicate(); + ~ComparisonPredicate() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the COMPARISON * predicate */ - virtual std::string getPredicateString() const; + std::string getPredicateString() const override; std::string fLHScalarExpression; std::string fRHScalarExpression; @@ -731,16 +729,16 @@ class BetweenPredicate : public Predicate /** @brief dtor */ - ~BetweenPredicate(); + ~BetweenPredicate() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the BETWEEN * predicate */ - virtual std::string getPredicateString() const; + std::string getPredicateString() const override; std::string fLHScalarExpression; std::string fRH1ScalarExpression; @@ -766,16 +764,16 @@ class LikePredicate : public Predicate /** @brief dtor */ - ~LikePredicate(); + ~LikePredicate() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the LIKE * predicate */ - virtual std::string getPredicateString() const; + std::string getPredicateString() const override; std::string fLHScalarExpression; std::string fAtom; @@ -800,16 +798,16 @@ class NullTestPredicate : public Predicate /** @brief dtor */ - ~NullTestPredicate(); + ~NullTestPredicate() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the NULL test * predicate */ - std::string getPredicateString() const; + std::string getPredicateString() const override; std::string fColumnRef; @@ -834,16 +832,16 @@ class InPredicate : public Predicate /** @brief dtor */ - ~InPredicate(); + ~InPredicate() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the IN * predicate */ - virtual std::string getPredicateString() const; + std::string getPredicateString() const override; std::string fScalarExpression; std::string fOperator; @@ -867,16 +865,16 @@ class AllOrAnyPredicate : public Predicate /** @brief dtor */ - ~AllOrAnyPredicate(); + ~AllOrAnyPredicate() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the * ALL or ANY predicate */ - virtual std::string getPredicateString() const; + std::string getPredicateString() const override; std::string fScalarExpression; std::string fOperator; @@ -900,16 +898,16 @@ class ExistanceTestPredicate : public Predicate /** @brief dtor */ - ~ExistanceTestPredicate(); + ~ExistanceTestPredicate() override; /** @brief dump to stdout */ - virtual std::ostream& put(std::ostream& os) const; + std::ostream& put(std::ostream& os) const override; /** @brief get the string representation of the EXISTS * predicate */ - virtual std::string getPredicateString() const; + std::string getPredicateString() const override; QuerySpec* fSubQuerySpecPtr; }; diff --git a/dbcon/dmlpackage/dmltable.h b/dbcon/dmlpackage/dmltable.h index 0bb7ece3c..9b8592234 100644 --- a/dbcon/dmlpackage/dmltable.h +++ b/dbcon/dmlpackage/dmltable.h @@ -44,7 +44,7 @@ class DMLTable : public DMLObject /** @brief dtor */ - ~DMLTable(); + ~DMLTable() override; /** @brief get the schema name */ @@ -85,7 +85,7 @@ class DMLTable : public DMLObject * * @param bytestream the ByteStream to read from */ - int read(messageqcpp::ByteStream& bytestream); + int read(messageqcpp::ByteStream& bytestream) override; /** @brief read a DMLTable metadata from a ByteStream * @@ -103,7 +103,7 @@ class DMLTable : public DMLObject * * @param bytestream the ByteStream to write to */ - int write(messageqcpp::ByteStream& bytestream); + int write(messageqcpp::ByteStream& bytestream) override; protected: private: diff --git a/dbcon/dmlpackage/insertdmlpackage.h b/dbcon/dmlpackage/insertdmlpackage.h index f5216696d..fc52d49b2 100644 --- a/dbcon/dmlpackage/insertdmlpackage.h +++ b/dbcon/dmlpackage/insertdmlpackage.h @@ -53,19 +53,19 @@ class InsertDMLPackage : public CalpontDMLPackage /** @brief dtor */ - EXPORT virtual ~InsertDMLPackage(); + EXPORT ~InsertDMLPackage() override; /** @brief write a InsertDMLPackage to a ByteStream * * @param bytestream the ByteStream to write to */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + EXPORT int write(messageqcpp::ByteStream& bytestream) override; /** @brief read InsertDMLPackage from bytestream * * @param bytestream the ByteStream to read from */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + EXPORT int read(messageqcpp::ByteStream& bytestream) override; /** @brief read InsertDMLPackage metadata from bytestream * @@ -85,7 +85,7 @@ class InsertDMLPackage : public CalpontDMLPackage * @param columns the number of columns in the buffer * @param rows the number of rows in the buffer */ - EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); + EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows) override; /** @brief build a InsertDMLPackage from MySQL buffer * @@ -95,13 +95,13 @@ class InsertDMLPackage : public CalpontDMLPackage * @param rows number of rows to be touched */ EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, - int rows, NullValuesBitset& nullValues); + int rows, NullValuesBitset& nullValues) override; /** @brief build a InsertDMLPackage from a InsertSqlStatement * * @param sqlStmt the InsertSqlStatement */ - EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); + EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement) override; /** @brief Dump the InsertDMLPackage for debugging purposes */ diff --git a/dbcon/dmlpackage/row.h b/dbcon/dmlpackage/row.h index d115299d4..dbf94dda7 100644 --- a/dbcon/dmlpackage/row.h +++ b/dbcon/dmlpackage/row.h @@ -45,7 +45,7 @@ class Row : public DMLObject /** @brief dtor */ - EXPORT ~Row(); + EXPORT ~Row() override; /** @brief copy constructor */ @@ -55,13 +55,13 @@ class Row : public DMLObject * * @param bytestream the ByteStream to read from */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + EXPORT int read(messageqcpp::ByteStream& bytestream) override; /** @brief write a Row to a ByteStream * * @param bytestream the ByteStream to write to */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + EXPORT int write(messageqcpp::ByteStream& bytestream) override; /** @brief get the list of columns in the row */ diff --git a/dbcon/dmlpackage/updatedmlpackage.h b/dbcon/dmlpackage/updatedmlpackage.h index 5e5eacba6..ea4a3c9c4 100644 --- a/dbcon/dmlpackage/updatedmlpackage.h +++ b/dbcon/dmlpackage/updatedmlpackage.h @@ -53,19 +53,19 @@ class UpdateDMLPackage : public CalpontDMLPackage /** @brief dtor */ - EXPORT virtual ~UpdateDMLPackage(); + EXPORT ~UpdateDMLPackage() override; /** @brief write a UpdateDMLPackage to a ByteStream * * @param bytestream the ByteStream to write to */ - EXPORT int write(messageqcpp::ByteStream& bytestream); + EXPORT int write(messageqcpp::ByteStream& bytestream) override; /** @brief read a UpdateDMLPackage from a ByteStream * * @param bytestream the ByteStream to read from */ - EXPORT int read(messageqcpp::ByteStream& bytestream); + EXPORT int read(messageqcpp::ByteStream& bytestream) override; /** @brief build a UpdateDMLPackage from a string buffer * @@ -73,13 +73,13 @@ class UpdateDMLPackage : public CalpontDMLPackage * @param columns the number of columns in the buffer * @param rows the number of rows in the buffer */ - EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows); + EXPORT int buildFromBuffer(std::string& buffer, int columns, int rows) override; /** @brief build a UpdateDMLPackage from a parsed UpdateSqlStatement * * @param sqlStatement the parsed UpdateSqlStatement */ - EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement); + EXPORT int buildFromSqlStatement(SqlStatement& sqlStatement) override; /** @brief build a InsertDMLPackage from MySQL buffer * @@ -87,7 +87,7 @@ class UpdateDMLPackage : public CalpontDMLPackage * @param rows the number of rows in the buffer */ EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, - int rows, NullValuesBitset& nullValues); + int rows, NullValuesBitset& nullValues) override; void buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt); protected: diff --git a/dbcon/dmlpackage/vendordmlstatement.h b/dbcon/dmlpackage/vendordmlstatement.h index a9d4dcffc..ca8efe3b9 100644 --- a/dbcon/dmlpackage/vendordmlstatement.h +++ b/dbcon/dmlpackage/vendordmlstatement.h @@ -23,10 +23,11 @@ /** @file */ #pragma once #include +#include #include #include #include -#include +#include #include "dmlpkg.h" #define EXPORT @@ -57,7 +58,7 @@ class VendorDMLStatement EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns, ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID); - + /** @brief destructor */ EXPORT ~VendorDMLStatement(); @@ -73,7 +74,7 @@ class VendorDMLStatement */ inline void set_TableName(std::string value) { - fTableName = value; + fTableName = std::move(value); } /** @brief Get the schema name @@ -87,7 +88,7 @@ class VendorDMLStatement */ inline void set_SchemaName(std::string value) { - fSchema = value; + fSchema = std::move(value); } /** @brief Get the DML statVendorDMLStatement classement type @@ -115,7 +116,7 @@ class VendorDMLStatement */ inline void set_DMLStatement(std::string dmlStatement) { - fDMLStatement = dmlStatement; + fDMLStatement = std::move(dmlStatement); } /** @brief Get the number of rows @@ -157,7 +158,7 @@ class VendorDMLStatement */ inline void set_DataBuffer(std::string value) { - fDataBuffer = value; + fDataBuffer = std::move(value); } /** @brief Get the session ID */ diff --git a/dbcon/dmlpackageproc/autoincrementdata.h b/dbcon/dmlpackageproc/autoincrementdata.h index 7dcd2a200..51506cf4f 100644 --- a/dbcon/dmlpackageproc/autoincrementdata.h +++ b/dbcon/dmlpackageproc/autoincrementdata.h @@ -21,7 +21,7 @@ #pragma once -#include +#include #include #include @@ -48,4 +48,3 @@ class AutoincrementData OIDNextValue fOidNextValueMap; boost::mutex fOIDnextvalLock; }; - diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index 6ead28c20..a0607e814 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -94,7 +94,7 @@ class AggregateColumn : public ReturnedColumn /** * ctor */ - AggregateColumn(const uint32_t sessionID); + explicit AggregateColumn(const uint32_t sessionID); /** * ctor @@ -109,9 +109,7 @@ class AggregateColumn : public ReturnedColumn /** * Destructors */ - virtual ~AggregateColumn() - { - } + ~AggregateColumn() override = default; /** * Accessor Methods @@ -167,7 +165,7 @@ class AggregateColumn : public ReturnedColumn * * deep copy of this pointer and return the copy */ - inline virtual AggregateColumn* clone() const override + inline AggregateColumn* clone() const override { return new AggregateColumn(*this); } @@ -190,14 +188,14 @@ class AggregateColumn : public ReturnedColumn /** * ASC flag */ - inline virtual bool asc() const override + inline bool asc() const override { return fAsc; } /** * ASC flag */ - inline virtual void asc(const bool asc) override + inline void asc(const bool asc) override { fAsc = asc; } @@ -205,7 +203,7 @@ class AggregateColumn : public ReturnedColumn /** * fData: SQL representation of this object */ - virtual const std::string data() const override + const std::string data() const override { return fData; } @@ -221,24 +219,24 @@ class AggregateColumn : public ReturnedColumn /** * Overloaded stream operator */ - virtual const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; + const std::string toString() const override; + std::string toCppCode(IncludeSet& includes) const override; /** * Serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; + void serialize(messageqcpp::ByteStream&) const override; /** * Serialize interface */ - virtual void unserialize(messageqcpp::ByteStream&) override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -253,7 +251,7 @@ class AggregateColumn : public ReturnedColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -309,8 +307,8 @@ class AggregateColumn : public ReturnedColumn static AggOp agname2num(const std::string&); using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() override; - virtual bool hasWindowFunc() override + bool hasAggregate() override; + bool hasWindowFunc() override { return false; } @@ -357,7 +355,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override + const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override { bool localIsNull = false; evaluate(row, localIsNull); @@ -368,7 +366,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getIntVal(); @@ -377,7 +375,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override + uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getUintVal(); @@ -386,7 +384,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) override + float getFloatVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getFloatVal(); @@ -395,7 +393,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) override + double getDoubleVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDoubleVal(); @@ -404,7 +402,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override + long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getLongDoubleVal(); @@ -413,7 +411,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override + IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDecimalVal(); @@ -421,7 +419,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override + int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDateIntVal(); @@ -429,7 +427,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getTimeIntVal(); @@ -437,7 +435,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDatetimeIntVal(); @@ -445,7 +443,7 @@ class AggregateColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getTimestampIntVal(); diff --git a/dbcon/execplan/arithmeticcolumn.h b/dbcon/execplan/arithmeticcolumn.h index ffb899204..a2b8fa923 100644 --- a/dbcon/execplan/arithmeticcolumn.h +++ b/dbcon/execplan/arithmeticcolumn.h @@ -51,7 +51,7 @@ class ArithmeticColumn : public ReturnedColumn ArithmeticColumn(); ArithmeticColumn(const std::string& sql, const uint32_t sessionID = 0); ArithmeticColumn(const ArithmeticColumn& rhs, const uint32_t sessionID = 0); - virtual ~ArithmeticColumn(); + ~ArithmeticColumn() override; inline ParseTree* expression() const { @@ -102,7 +102,7 @@ class ArithmeticColumn : public ReturnedColumn /** * get SQL representation of this object */ - virtual const std::string data() const override + const std::string data() const override { return fData; } @@ -110,7 +110,7 @@ class ArithmeticColumn : public ReturnedColumn /** * set SQL representation of this object */ - virtual void data(const std::string data) override + void data(const std::string data) override { fData = data; } @@ -118,13 +118,13 @@ class ArithmeticColumn : public ReturnedColumn /** * virtual stream method */ - virtual const std::string toString() const override; + const std::string toString() const override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual ArithmeticColumn* clone() const override + inline ArithmeticColumn* clone() const override { return new ArithmeticColumn(*this); } @@ -132,15 +132,15 @@ class ArithmeticColumn : public ReturnedColumn /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -154,7 +154,7 @@ class ArithmeticColumn : public ReturnedColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -164,16 +164,16 @@ class ArithmeticColumn : public ReturnedColumn bool operator!=(const ArithmeticColumn& t) const; using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() override; - virtual bool hasWindowFunc() override; + bool hasAggregate() override; + bool hasWindowFunc() override; - virtual void setDerivedTable() override; - virtual void replaceRealCol(std::vector&) override; - virtual const std::vector& simpleColumnList() const override + void setDerivedTable() override; + void replaceRealCol(std::vector&) override; + const std::vector& simpleColumnList() const override { return fSimpleColumnList; } - virtual void setSimpleColumnList() override; + void setSimpleColumnList() override; /** * Return the table that the column arguments belong to. @@ -181,13 +181,13 @@ class ArithmeticColumn : public ReturnedColumn * @return tablename, if all arguments belong to one table * empty string "", if multiple tables are involved in this func */ - virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan) override; + bool singleTable(CalpontSystemCatalog::TableAliasName& tan) override; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: std::string fTableAlias; // table alias for this column - bool fAsc = false; // asc flag for order by column + bool fAsc = false; // asc flag for order by column std::string fData; /** build expression tree @@ -211,62 +211,62 @@ class ArithmeticColumn : public ReturnedColumn * F&E framework * ***********************************************************/ public: - virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override + const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getStrVal(row, isNull); } - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getIntVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getIntVal(row, isNull); } - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override + uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getUintVal(row, isNull); } - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) override + float getFloatVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getFloatVal(row, isNull); } - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) override + double getDoubleVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getDoubleVal(row, isNull); } - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override + long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getLongDoubleVal(row, isNull); } - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override + IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getDecimalVal(row, isNull); } - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override + int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getDateIntVal(row, isNull); } - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getDatetimeIntVal(row, isNull); } - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getTimestampIntVal(row, isNull); } - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getTimeIntVal(row, isNull); } - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) override + bool getBoolVal(rowgroup::Row& row, bool& isNull) override { return fExpression->getBoolVal(row, isNull); } diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 974ac35c9..f83b6c5da 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -46,16 +46,16 @@ class ArithmeticOperator : public Operator public: ArithmeticOperator(); - ArithmeticOperator(const std::string& operatorName); + explicit ArithmeticOperator(const std::string& operatorName); ArithmeticOperator(const ArithmeticOperator& rhs); - virtual ~ArithmeticOperator(); + ~ArithmeticOperator() override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual ArithmeticOperator* clone() const override + inline ArithmeticOperator* clone() const override { return new ArithmeticOperator(*this); } @@ -72,15 +72,15 @@ class ArithmeticOperator : public Operator /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -94,7 +94,7 @@ class ArithmeticOperator : public Operator * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -107,11 +107,11 @@ class ArithmeticOperator : public Operator * F&E framework * ***********************************************************/ using Operator::evaluate; - inline virtual void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override; + inline void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override; using Operator::getStrVal; - virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, - ParseTree* rop) override + const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, + ParseTree* rop) override { bool localIsNull = false; evaluate(row, localIsNull, lop, rop); @@ -119,38 +119,37 @@ class ArithmeticOperator : public Operator return localIsNull ? fResult.strVal.dropString() : TreeNode::getStrVal(fTimeZone); } using Operator::getIntVal; - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + int64_t getIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getIntVal(); } using Operator::getUintVal; - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + uint64_t getUintVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getUintVal(); } using Operator::getFloatVal; - virtual float getFloatVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + float getFloatVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getFloatVal(); } using Operator::getDoubleVal; - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + double getDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getDoubleVal(); } using Operator::getLongDoubleVal; - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, - ParseTree* rop) override + long double getLongDoubleVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getLongDoubleVal(); } using Operator::getDecimalVal; - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); @@ -169,32 +168,31 @@ class ArithmeticOperator : public Operator return TreeNode::getDecimalVal(); } using Operator::getDateIntVal; - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + int32_t getDateIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getDateIntVal(); } using Operator::getDatetimeIntVal; - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getDatetimeIntVal(); } using Operator::getTimestampIntVal; - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, - ParseTree* rop) override + int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getTimestampIntVal(); } using Operator::getTimeIntVal; - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getTimeIntVal(); } using Operator::getBoolVal; - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { evaluate(row, isNull, lop, rop); return TreeNode::getBoolVal(); @@ -209,7 +207,7 @@ class ArithmeticOperator : public Operator fDecimalOverflowCheck = check; } - inline virtual std::string toCppCode(IncludeSet& includes) const override + inline std::string toCppCode(IncludeSet& includes) const override { includes.insert("arithmeticoperator.h"); std::stringstream ss; diff --git a/dbcon/execplan/blocksize.h b/dbcon/execplan/blocksize.h index 2151369cd..aa4e35729 100644 --- a/dbcon/execplan/blocksize.h +++ b/dbcon/execplan/blocksize.h @@ -23,6 +23,6 @@ // # of bytes in a block const uint64_t BLOCK_SIZE = 8192; -// lobgical_block_rids is the # of rows 1-byter-column in a block +// logical_block_rids is the # of rows 1-byter-column in a block // its value is the same as block_size, but different unit const uint64_t LOGICAL_BLOCK_RIDS = BLOCK_SIZE; diff --git a/dbcon/execplan/calpontselectexecutionplan.cpp b/dbcon/execplan/calpontselectexecutionplan.cpp index 693f8367b..6845cf9fd 100644 --- a/dbcon/execplan/calpontselectexecutionplan.cpp +++ b/dbcon/execplan/calpontselectexecutionplan.cpp @@ -25,13 +25,13 @@ using namespace std; #include +#include #include "bytestream.h" using namespace messageqcpp; #include "calpontselectexecutionplan.h" #include "objectreader.h" -#include "filter.h" #include "returnedcolumn.h" #include "simplecolumn.h" #include "querystats.h" @@ -63,10 +63,10 @@ CalpontSelectExecutionPlan::ColumnMap CalpontSelectExecutionPlan::fColMap; /** * Constructors/Destructors */ -CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(const int location) +CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(int location) : fLocalQuery(GLOBAL_QUERY) - , fFilters(0) - , fHaving(0) + , fFilters(nullptr) + , fHaving(nullptr) , fLocation(location) , fDependent(false) , fTxnID(-1) @@ -98,17 +98,18 @@ CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(const int location) fUuid = QueryTeleClient::genUUID(); } -CalpontSelectExecutionPlan::CalpontSelectExecutionPlan( - const ReturnedColumnList& returnedCols, ParseTree* filters, const SelectList& subSelects, - const GroupByColumnList& groupByCols, ParseTree* having, const OrderByColumnList& orderByCols, - const string alias, const int location, const bool dependent, const bool withRollup) - : fReturnedCols(returnedCols) +CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(ReturnedColumnList returnedCols, ParseTree* filters, + SelectList subSelects, GroupByColumnList groupByCols, + ParseTree* having, OrderByColumnList orderByCols, + string alias, int location, bool dependent, + bool withRollup) + : fReturnedCols(std::move(returnedCols)) , fFilters(filters) - , fSubSelects(subSelects) - , fGroupByCols(groupByCols) + , fSubSelects(std::move(subSelects)) + , fGroupByCols(std::move(groupByCols)) , fHaving(having) - , fOrderByCols(orderByCols) - , fTableAlias(alias) + , fOrderByCols(std::move(orderByCols)) + , fTableAlias(std::move(alias)) , fLocation(location) , fDependent(dependent) , fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL) @@ -118,23 +119,18 @@ CalpontSelectExecutionPlan::CalpontSelectExecutionPlan( } CalpontSelectExecutionPlan::CalpontSelectExecutionPlan(string data) - : fData(data) - , fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL) - , fWithRollup(false) + : fData(std::move(data)), fPriority(querystats::DEFAULT_USER_PRIORITY_LEVEL), fWithRollup(false) { fUuid = QueryTeleClient::genUUID(); } CalpontSelectExecutionPlan::~CalpontSelectExecutionPlan() { - if (fFilters != NULL) - delete fFilters; + delete fFilters; + delete fHaving; - if (fHaving != NULL) - delete fHaving; - - fFilters = NULL; - fHaving = NULL; + fFilters = nullptr; + fHaving = nullptr; if (!fDynamicParseTreeVec.empty()) { @@ -145,11 +141,11 @@ CalpontSelectExecutionPlan::~CalpontSelectExecutionPlan() // 'delete fFilters;' above has already deleted objects pointed // to by parseTree->left()/right()/data(), so we set the // pointers to NULL here before calling 'delete parseTree;' - parseTree->left((ParseTree*)(NULL)); - parseTree->right((ParseTree*)(NULL)); - parseTree->data((TreeNode*)(NULL)); + parseTree->left((ParseTree*)(nullptr)); + parseTree->right((ParseTree*)(nullptr)); + parseTree->data((TreeNode*)(nullptr)); delete parseTree; - parseTree = NULL; + parseTree = nullptr; } } @@ -265,7 +261,7 @@ string CalpontSelectExecutionPlan::toString() const // Filters output << ">>Filters" << endl; - if (filters() != 0) + if (filters() != nullptr) filters()->walk(ParseTree::print, output); else output << "empty filter tree" << endl; @@ -282,7 +278,7 @@ string CalpontSelectExecutionPlan::toString() const } // Having - if (having() != 0) + if (having() != nullptr) { output << ">>Having" << endl; having()->walk(ParseTree::print, output); @@ -496,16 +492,16 @@ void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) fDerivedTableList.clear(); fSelectSubList.clear(); - if (fFilters != 0) + if (fFilters != nullptr) { delete fFilters; - fFilters = 0; + fFilters = nullptr; } - if (fHaving != 0) + if (fHaving != nullptr) { delete fHaving; - fHaving = 0; + fHaving = nullptr; } if (!fDynamicParseTreeVec.empty()) @@ -517,11 +513,11 @@ void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) // 'delete fFilters;' above has already deleted objects pointed // to by parseTree->left()/right()/data(), so we set the // pointers to NULL here before calling 'delete parseTree;' - parseTree->left((ParseTree*)(NULL)); - parseTree->right((ParseTree*)(NULL)); - parseTree->data((TreeNode*)(NULL)); + parseTree->left((ParseTree*)(nullptr)); + parseTree->right((ParseTree*)(nullptr)); + parseTree->data((TreeNode*)(nullptr)); delete parseTree; - parseTree = NULL; + parseTree = nullptr; } } @@ -699,12 +695,12 @@ bool CalpontSelectExecutionPlan::operator==(const CalpontSelectExecutionPlan& t) return false; // fFilters - if (fFilters != NULL && t.fFilters != NULL) + if (fFilters != nullptr && t.fFilters != nullptr) { if (*fFilters != *t.fFilters) return false; } - else if (fFilters != NULL || t.fFilters != NULL) + else if (fFilters != nullptr || t.fFilters != nullptr) return false; // fSubSelects @@ -725,12 +721,12 @@ bool CalpontSelectExecutionPlan::operator==(const CalpontSelectExecutionPlan& t) return false; // fHaving - if (fHaving != NULL && t.fHaving != NULL) + if (fHaving != nullptr && t.fHaving != nullptr) { if (*fHaving != *t.fHaving) return false; } - else if (fHaving != NULL || t.fHaving != NULL) + else if (fHaving != nullptr || t.fHaving != nullptr) return false; // fOrderByCols @@ -795,7 +791,7 @@ bool CalpontSelectExecutionPlan::operator==(const CalpontExecutionPlan* t) const ac = dynamic_cast(t); - if (ac == NULL) + if (ac == nullptr) return false; return *this == *ac; diff --git a/dbcon/execplan/calpontselectexecutionplan.h b/dbcon/execplan/calpontselectexecutionplan.h index 7ced677c8..0fa8cec96 100644 --- a/dbcon/execplan/calpontselectexecutionplan.h +++ b/dbcon/execplan/calpontselectexecutionplan.h @@ -146,19 +146,18 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan /** * Constructors */ - CalpontSelectExecutionPlan(const int location = MAIN); + explicit CalpontSelectExecutionPlan(int location = MAIN); - CalpontSelectExecutionPlan(const ReturnedColumnList& returnedCols, ParseTree* filters, - const SelectList& subSelects, const GroupByColumnList& groupByCols, - ParseTree* having, const OrderByColumnList& orderByCols, const std::string alias, - const int location, const bool dependent, const bool withRollup); + CalpontSelectExecutionPlan(ReturnedColumnList returnedCols, ParseTree* filters, SelectList subSelects, + GroupByColumnList groupByCols, ParseTree* having, OrderByColumnList orderByCols, + std::string alias, int location, bool dependent, bool withRollup); - CalpontSelectExecutionPlan(const std::string data); + explicit CalpontSelectExecutionPlan(std::string data); /** * Destructors */ - virtual ~CalpontSelectExecutionPlan(); + ~CalpontSelectExecutionPlan() override; /** * Access and mutator methods @@ -366,13 +365,13 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan /** sql representation of this select query * */ - const std::string data() const + const std::string& data() const { return fData; } - void data(const std::string data) + void data(std::string data) { - fData = data; + fData = std::move(data); } /** session id @@ -679,7 +678,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan { fDJSSmallSideLimit = l; } - uint64_t djsSmallSideLimit() + uint64_t djsSmallSideLimit() const { return fDJSSmallSideLimit; } @@ -688,7 +687,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan { fDJSLargeSideLimit = l; } - uint64_t djsLargeSideLimit() + uint64_t djsLargeSideLimit() const { return fDJSLargeSideLimit; } @@ -697,7 +696,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan { fDJSPartitionSize = l; } - uint64_t djsPartitionSize() + uint64_t djsPartitionSize() const { return fDJSPartitionSize; } @@ -715,7 +714,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan { fDJSForceRun = b; } - bool djsForceRun() + bool djsForceRun() const { return fDJSForceRun; } @@ -724,7 +723,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan { fMaxPmJoinResultCount = value; } - uint32_t maxPmJoinResultCount() + uint32_t maxPmJoinResultCount() const { return fMaxPmJoinResultCount; } @@ -733,7 +732,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan { fUMMemLimit = l; } - int64_t umMemLimit() + int64_t umMemLimit() const { return fUMMemLimit; } @@ -742,7 +741,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan { fIsDML = b; } - bool isDML() + bool isDML() const { return fIsDML; } @@ -762,15 +761,15 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan /** * @note Serialize() assumes that none of the vectors contain NULL pointers. */ - virtual void serialize(messageqcpp::ByteStream&) const; - virtual void unserialize(messageqcpp::ByteStream&); + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const CalpontExecutionPlan* t) const; + bool operator==(const CalpontExecutionPlan* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -784,7 +783,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const CalpontExecutionPlan* t) const; + bool operator!=(const CalpontExecutionPlan* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -863,7 +862,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan /** * A tree of Filter objects */ - ParseTree* fFilters = nullptr; + ParseTree* fFilters{nullptr}; /** * A list of CalpontExecutionPlan objects */ @@ -875,7 +874,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan /** * A tree of having clause condition associated with group by clause */ - ParseTree* fHaving; + ParseTree* fHaving{nullptr}; /** * A list of order by columns */ @@ -955,7 +954,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan // Derived table involved in the query. For derived table optimization std::vector fSubSelectList; - boost::uuids::uuid fUuid; + boost::uuids::uuid fUuid{}; /* Disk-based join vars */ uint64_t fDJSSmallSideLimit = 0; diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index a3fc1ed24..6dffbdde8 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -139,9 +139,8 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog { } DictOID(const DictOID& rhs) - : dictOID(rhs.dictOID), listOID(rhs.listOID), treeOID(rhs.treeOID), compressionType(rhs.compressionType) - { - } + + = default; OID dictOID; OID listOID; OID treeOID; @@ -176,7 +175,7 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog // If we used an unorderedmap, we might improve performance. // Maybe. NJLSysDataVector sysDataVec; - NJLSysDataList(){}; + NJLSysDataList() = default; ~NJLSysDataList(); NJLSysDataVector::const_iterator begin() const { @@ -439,9 +438,7 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog */ struct TableName { - TableName() - { - } + TableName() = default; TableName(std::string sch, std::string tb) : schema(sch), table(tb) { } @@ -873,7 +870,7 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog private: /** Constuctors */ - explicit CalpontSystemCatalog(const CalpontSystemCatalog& rhs); + CalpontSystemCatalog(const CalpontSystemCatalog& rhs); CalpontSystemCatalog& operator=(const CalpontSystemCatalog& rhs); diff --git a/dbcon/execplan/clientrotator.h b/dbcon/execplan/clientrotator.h index 273863c53..e2616247e 100644 --- a/dbcon/execplan/clientrotator.h +++ b/dbcon/execplan/clientrotator.h @@ -56,7 +56,7 @@ class ClientRotator } } - /** @brief connnect + /** @brief connect * * Try connecting to client based on session id. If no connection, * try connectList. @@ -79,7 +79,7 @@ class ClientRotator { fClient->shutdown(); delete fClient; - fClient = 0; + fClient = nullptr; } } @@ -140,7 +140,7 @@ class ClientRotator */ bool exeConnect(const std::string& clientName); - /** @brief connnect to list + /** @brief connect to list * * Try connecting to next client on list * until timeout lapses. Then throw exception. diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index e8fdf3812..3094f8b8d 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -67,22 +67,23 @@ class ConstantColumn : public ReturnedColumn /** * ctor */ - ConstantColumn(const std::string& sql, TYPE type = LITERAL); + explicit ConstantColumn(const std::string& sql, TYPE type = LITERAL); /** * ctor */ - ConstantColumn(const int64_t val, TYPE type = NUM); // deprecate + explicit ConstantColumn(const int64_t val, TYPE type = NUM); // deprecate /** * ctor */ - ConstantColumn(const uint64_t val, TYPE type = NUM, int8_t scale = 0, uint8_t precision = 0); // deprecate + explicit ConstantColumn(const uint64_t val, TYPE type = NUM, int8_t scale = 0, + uint8_t precision = 0); // deprecate // There are more ctors below... /** * dtor */ - virtual ~ConstantColumn(); + ~ConstantColumn() override; /* * Accessor Methods @@ -144,25 +145,25 @@ class ConstantColumn : public ReturnedColumn /** * accessor */ - virtual const std::string data() const override; + const std::string data() const override; /** * accessor */ - virtual void data(const std::string data) override + void data(const std::string data) override { fData = data; } /** * accessor */ - virtual const std::string toString() const override; + const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual ConstantColumn* clone() const override + inline ConstantColumn* clone() const override { return new ConstantColumn(*this); } @@ -173,18 +174,18 @@ class ConstantColumn : public ReturnedColumn /** * serialize */ - virtual void serialize(messageqcpp::ByteStream&) const override; + void serialize(messageqcpp::ByteStream&) const override; /** * unserialize */ - virtual void unserialize(messageqcpp::ByteStream&) override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -198,7 +199,7 @@ class ConstantColumn : public ReturnedColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -207,13 +208,13 @@ class ConstantColumn : public ReturnedColumn */ bool operator!=(const ConstantColumn& t) const; - virtual bool hasWindowFunc() override + bool hasWindowFunc() override { return false; } /** Constant column on the filte can always be moved into derived table */ - virtual void setDerivedTable() override + void setDerivedTable() override { fDerivedTable = std::string("*"); } @@ -267,7 +268,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) override + bool getBoolVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); return TreeNode::getBoolVal(); @@ -275,7 +276,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override + const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); return fResult.strVal; @@ -283,7 +284,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getIntVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); return fResult.intVal; @@ -291,7 +292,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override + uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); return fResult.uintVal; @@ -299,7 +300,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) override + float getFloatVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); return fResult.floatVal; @@ -307,7 +308,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) override + double getDoubleVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); return fResult.doubleVal; @@ -315,7 +316,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override + IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); return fResult.decimalVal; @@ -323,7 +324,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override + int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); @@ -338,7 +339,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); @@ -354,7 +355,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); @@ -370,7 +371,7 @@ class ConstantColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override { isNull = isNull || (fType == NULLDATA); @@ -409,7 +410,7 @@ class ConstantColumnNull : public ConstantColumn class ConstantColumnString : public ConstantColumn { public: - ConstantColumnString(const std::string& str) : ConstantColumn(str, ConstantColumn::LITERAL) + explicit ConstantColumnString(const std::string& str) : ConstantColumn(str, ConstantColumn::LITERAL) { } }; @@ -471,7 +472,6 @@ std::ostream& operator<<(std::ostream& output, const ConstantColumn& rhs); class RollupMarkColumn : public ReturnedColumn { public: - /** * ctor */ @@ -483,7 +483,7 @@ class RollupMarkColumn : public ReturnedColumn /** * dtor */ - virtual ~RollupMarkColumn(); + ~RollupMarkColumn() override; /** * accessor @@ -502,26 +502,26 @@ class RollupMarkColumn : public ReturnedColumn /** * accessor */ - virtual const std::string data() const override + const std::string data() const override { return ""; } /** * accessor */ - virtual void data(const std::string data) override + void data(const std::string data) override { idbassert(0); } /** * accessor */ - virtual const std::string toString() const override + const std::string toString() const override { return "RollupMarkColumn"; } - virtual std::string toCppCode(IncludeSet& includes) const override + std::string toCppCode(IncludeSet& includes) const override { idbassert(0); } @@ -529,7 +529,7 @@ class RollupMarkColumn : public ReturnedColumn * * deep copy of this pointer and return the copy */ - inline virtual RollupMarkColumn* clone() const override + inline RollupMarkColumn* clone() const override { return new RollupMarkColumn(); } @@ -540,18 +540,18 @@ class RollupMarkColumn : public ReturnedColumn /** * serialize */ - virtual void serialize(messageqcpp::ByteStream&) const override; + void serialize(messageqcpp::ByteStream&) const override; /** * unserialize */ - virtual void unserialize(messageqcpp::ByteStream&) override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override + bool operator==(const TreeNode* t) const override { return false; } @@ -571,7 +571,7 @@ class RollupMarkColumn : public ReturnedColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override + bool operator!=(const TreeNode* t) const override { return true; } @@ -586,13 +586,13 @@ class RollupMarkColumn : public ReturnedColumn return false; } - virtual bool hasWindowFunc() override + bool hasWindowFunc() override { return false; } /** Constant column on the filte can always be moved into derived table */ - virtual void setDerivedTable() override + void setDerivedTable() override { fDerivedTable = std::string("*"); } @@ -617,18 +617,18 @@ class RollupMarkColumn : public ReturnedColumn /** * F&E */ - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) override + bool getBoolVal(rowgroup::Row& row, bool& isNull) override { return true; } /** * F&E */ - virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override; + const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override; /** * F&E */ - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getIntVal(rowgroup::Row& row, bool& isNull) override { isNull = false; return 0x12340000UL; @@ -636,28 +636,28 @@ class RollupMarkColumn : public ReturnedColumn /** * F&E */ - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override + uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override { return getIntVal(row, isNull); } /** * F&E */ - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) override + float getFloatVal(rowgroup::Row& row, bool& isNull) override { return getIntVal(row, isNull); } /** * F&E */ - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) override + double getDoubleVal(rowgroup::Row& row, bool& isNull) override { return getIntVal(row, isNull); } /** * F&E */ - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override + IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override { isNull = false; return fResult.decimalVal; @@ -665,7 +665,7 @@ class RollupMarkColumn : public ReturnedColumn /** * F&E */ - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override + int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override { isNull = false; @@ -680,7 +680,7 @@ class RollupMarkColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override { isNull = false; @@ -696,7 +696,7 @@ class RollupMarkColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override { isNull = false; @@ -712,7 +712,7 @@ class RollupMarkColumn : public ReturnedColumn /** * F&E */ - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override { isNull = false; diff --git a/dbcon/execplan/constantfilter.h b/dbcon/execplan/constantfilter.h index 5a3c49063..0174604f0 100644 --- a/dbcon/execplan/constantfilter.h +++ b/dbcon/execplan/constantfilter.h @@ -64,7 +64,7 @@ class ConstantFilter : public Filter */ ConstantFilter(); ConstantFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs); - ConstantFilter(SimpleFilter* sf); + explicit ConstantFilter(SimpleFilter* sf); // for code generation purposes only ConstantFilter(const SOP& op, const FilterList& filterList, const SRCP& col, const std::string& functionName, const std::string& data); @@ -75,7 +75,7 @@ class ConstantFilter : public Filter /** * Destructors */ - virtual ~ConstantFilter(); + ~ConstantFilter() override; /** * Accessor Methods @@ -114,20 +114,20 @@ class ConstantFilter : public Filter } // virtual const std::string data() const; - virtual const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; + const std::string toString() const override; + std::string toCppCode(IncludeSet& includes) const override; /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -141,7 +141,7 @@ class ConstantFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -188,17 +188,17 @@ class ConstantFilter : public Filter FilterList fFilterList; /// vector of simple filters SRCP fCol; /// the common column std::string fFunctionName; /// function name - /*********************************************************** - * F&E framework * - ***********************************************************/ + /*********************************************************** + * F&E framework * + ***********************************************************/ public: ConstantFilter(const ConstantFilter& rhs); - inline virtual ConstantFilter* clone() const override + inline ConstantFilter* clone() const override { return new ConstantFilter(*this); } - inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) override; + inline bool getBoolVal(rowgroup::Row& row, bool& isNull) override; // get all simple columns involved in this column const std::vector& simpleColumnList(); diff --git a/dbcon/execplan/existsfilter.h b/dbcon/execplan/existsfilter.h index 89cb6e627..f807c2f5a 100644 --- a/dbcon/execplan/existsfilter.h +++ b/dbcon/execplan/existsfilter.h @@ -49,9 +49,9 @@ class ExistsFilter : public Filter * Constructors */ ExistsFilter(); - ExistsFilter(const SCSEP& sub, const bool existsFlag = false, const bool correlated = false); + explicit ExistsFilter(const SCSEP& sub, const bool existsFlag = false, const bool correlated = false); ExistsFilter(const ExistsFilter& rhs); - virtual ~ExistsFilter(); + ~ExistsFilter() override; /** * Accessor Methods @@ -87,19 +87,19 @@ class ExistsFilter : public Filter * Overloaded stream operator */ // virtual std::ostream& operator<< (std::ostream& output); - virtual const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; + const std::string toString() const override; + std::string toCppCode(IncludeSet& includes) const override; /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual ExistsFilter* clone() const override + inline ExistsFilter* clone() const override { return new ExistsFilter(*this); } @@ -109,7 +109,7 @@ class ExistsFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -123,7 +123,7 @@ class ExistsFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * diff --git a/dbcon/execplan/expressionparser.h b/dbcon/execplan/expressionparser.h index 5a9af0d16..611a28eb6 100644 --- a/dbcon/execplan/expressionparser.h +++ b/dbcon/execplan/expressionparser.h @@ -53,12 +53,12 @@ struct Token TreeNode* value; bool is_operator() const { - if (value == 0) + if (value == nullptr) return false; return (typeid(*value) == typeid(Operator)); } - Token() : value(0) + Token() : value(nullptr) { } Token(TreeNode* v) : value(v) diff --git a/dbcon/execplan/filter.h b/dbcon/execplan/filter.h index 7558971a4..cba6df5fb 100644 --- a/dbcon/execplan/filter.h +++ b/dbcon/execplan/filter.h @@ -60,14 +60,14 @@ class Filter : public TreeNode * Constructors */ Filter(); - Filter(const std::string& sql); + explicit Filter(const std::string& sql); // not needed yet // Filter(const Filter& rhs); /** * Destructors */ - virtual ~Filter(); + ~Filter() override; /** * Accessor Methods */ @@ -75,15 +75,15 @@ class Filter : public TreeNode /** * Operations */ - virtual const std::string toString() const override; + const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; - virtual const std::string data() const override + const std::string data() const override { return fData; } - virtual void data(const std::string data) override + void data(const std::string data) override { fData = data; } @@ -92,7 +92,7 @@ class Filter : public TreeNode * * deep copy of this pointer and return the copy */ - inline virtual Filter* clone() const override + inline Filter* clone() const override { return new Filter(*this); } @@ -100,15 +100,15 @@ class Filter : public TreeNode /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -122,7 +122,7 @@ class Filter : public TreeNode * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -133,9 +133,9 @@ class Filter : public TreeNode /** @brief test if this filter can be combined with the argument filter * This is for operation combine optimization - * @param f the filter that this fiter tries to combine with + * @param f the filter that this filter tries to combine with * @param op the operator that connects the two filters. if one or both of the - * two filters is constantFilter, need to make sure operator is consistant. + * two filters is constantFilter, need to make sure operator is consistent. * @return a filter(constantfilter) if successfully combined. otherwise * return NULL * For Oracle front end. Deprecated now. diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 57843828c..cf7a3519b 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -51,11 +51,11 @@ class FunctionColumn : public ReturnedColumn { public: FunctionColumn(); - FunctionColumn(std::string& funcName); + explicit FunctionColumn(std::string& funcName); FunctionColumn(const std::string& functionName, const std::string& funcParmsInString, const uint32_t sessionID = 0); FunctionColumn(const FunctionColumn& rhs, const uint32_t sessionID = 0); - virtual ~FunctionColumn(); + ~FunctionColumn() override; /** get function name * @@ -131,19 +131,19 @@ class FunctionColumn : public ReturnedColumn fTimeZone = timeZone; } - virtual const std::string data() const override; - virtual void data(const std::string data) override + const std::string data() const override; + void data(const std::string data) override { fData = data; } - virtual const std::string toString() const override; + const std::string toString() const override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual FunctionColumn* clone() const override + inline FunctionColumn* clone() const override { return new FunctionColumn(*this); } @@ -151,20 +151,20 @@ class FunctionColumn : public ReturnedColumn /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() override; - virtual bool hasWindowFunc() override; - virtual void setDerivedTable() override; - virtual void replaceRealCol(std::vector&) override; + bool hasAggregate() override; + bool hasWindowFunc() override; + void setDerivedTable() override; + void replaceRealCol(std::vector&) override; virtual const std::vector& simpleColumnList() const override { return fSimpleColumnList; } - virtual void setSimpleColumnList() override; + void setSimpleColumnList() override; /** * Return the tableAlias name of the table that the column arguments belong to. * @@ -172,9 +172,9 @@ class FunctionColumn : public ReturnedColumn * @return true, if all arguments belong to one table * false, if multiple tables are involved in the function */ - virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan) override; + bool singleTable(CalpontSystemCatalog::TableAliasName& tan) override; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: /** @@ -190,7 +190,7 @@ class FunctionColumn : public ReturnedColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -204,7 +204,7 @@ class FunctionColumn : public ReturnedColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -217,7 +217,7 @@ class FunctionColumn : public ReturnedColumn * F&E framework * ***********************************************************/ public: - virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override + const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); fResult.strVal.dropString(); @@ -228,32 +228,32 @@ class FunctionColumn : public ReturnedColumn } return fResult.strVal; } - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getIntVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getIntVal(row, fFunctionParms, isNull, fOperationType); } - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override + uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getUintVal(row, fFunctionParms, isNull, fOperationType); } - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) override + float getFloatVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getFloatVal(row, fFunctionParms, isNull, fOperationType); } - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) override + double getDoubleVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getDoubleVal(row, fFunctionParms, isNull, fOperationType); } - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override + long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getLongDoubleVal(row, fFunctionParms, isNull, fOperationType); } - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override + IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); IDB_Decimal decimal = fFunctor->getDecimalVal(row, fFunctionParms, isNull, fOperationType); @@ -290,36 +290,37 @@ class FunctionColumn : public ReturnedColumn if (fResultType.scale > decimal.scale) decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; else - decimal.value = (int64_t)( - decimal.value > 0 ? (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 - : (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); + decimal.value = + (int64_t)(decimal.value > 0 + ? (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 + : (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); } decimal.scale = fResultType.scale; decimal.precision = std::max(fResultType.precision, static_cast(decimal.precision)); return decimal; } - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) override + bool getBoolVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getBoolVal(row, fFunctionParms, isNull, fOperationType); } - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override + int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getDateIntVal(row, fFunctionParms, isNull, fOperationType); } - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); } - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getTimestampIntVal(row, fFunctionParms, isNull, fOperationType); } - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override { fOperationType.setTimeZone(fTimeZone); return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); @@ -332,8 +333,8 @@ class FunctionColumn : public ReturnedColumn private: funcexp::FunctionParm fFunctionParms; - funcexp::Func* fFunctor; /// functor to execute this function - funcexp::Func* fDynamicFunctor = NULL; // for rand encode decode + funcexp::Func* fFunctor; /// functor to execute this function + funcexp::Func* fDynamicFunctor = nullptr; // for rand encode decode bool fFixed = false; }; diff --git a/dbcon/execplan/groupconcatcolumn.h b/dbcon/execplan/groupconcatcolumn.h index af65ff08c..70bc6b660 100644 --- a/dbcon/execplan/groupconcatcolumn.h +++ b/dbcon/execplan/groupconcatcolumn.h @@ -52,25 +52,25 @@ class GroupConcatColumn : public AggregateColumn */ GroupConcatColumn(); - GroupConcatColumn(const uint32_t sessionID); + explicit GroupConcatColumn(const uint32_t sessionID); GroupConcatColumn(const GroupConcatColumn& rhs, const uint32_t sessionID = 0); /** * Destructors */ - virtual ~GroupConcatColumn(); + ~GroupConcatColumn() override; /** * Overloaded stream operator */ - virtual const std::string toString() const override; + const std::string toString() const override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - virtual GroupConcatColumn* clone() const override + GroupConcatColumn* clone() const override { return new GroupConcatColumn(*this); } @@ -98,8 +98,8 @@ class GroupConcatColumn : public AggregateColumn /** * Serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -107,7 +107,7 @@ class GroupConcatColumn : public AggregateColumn * @return true iff every member of t is a duplicate copy of every member of this; * false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -124,7 +124,7 @@ class GroupConcatColumn : public AggregateColumn * @return false iff every member of t is a duplicate copy of every member of this; * true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -135,7 +135,7 @@ class GroupConcatColumn : public AggregateColumn using AggregateColumn::operator!=; virtual bool operator!=(const GroupConcatColumn& t) const; - virtual string toCppCode(IncludeSet& includes) const override; + string toCppCode(IncludeSet& includes) const override; private: std::vector fOrderCols; diff --git a/dbcon/execplan/intervalcolumn.h b/dbcon/execplan/intervalcolumn.h index 7c8ffe376..3017de68a 100644 --- a/dbcon/execplan/intervalcolumn.h +++ b/dbcon/execplan/intervalcolumn.h @@ -79,9 +79,7 @@ class IntervalColumn : public ReturnedColumn IntervalColumn(); IntervalColumn(SRCP&, int); IntervalColumn(const IntervalColumn& rhs, const uint32_t sessionID = 0); - virtual ~IntervalColumn() - { - } + ~IntervalColumn() override = default; const SRCP& val() const { return fVal; @@ -99,22 +97,22 @@ class IntervalColumn : public ReturnedColumn fIntervalType = intervalType; } const std::string toString() const override; - inline virtual IntervalColumn* clone() const override + inline IntervalColumn* clone() const override { return new IntervalColumn(*this); } using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() override + bool hasAggregate() override { return false; } - virtual bool hasWindowFunc() override + bool hasWindowFunc() override { return false; } - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: /** @@ -124,12 +122,12 @@ class IntervalColumn : public ReturnedColumn int fIntervalType; // okay to be private for now. - virtual bool operator==(const TreeNode* t) const override + bool operator==(const TreeNode* t) const override { return false; } bool operator==(const IntervalColumn& t) const; - virtual bool operator!=(const TreeNode* t) const override + bool operator!=(const TreeNode* t) const override { return false; } diff --git a/dbcon/execplan/jsonarrayaggcolumn.h b/dbcon/execplan/jsonarrayaggcolumn.h index 980ca1549..edafd3d0b 100644 --- a/dbcon/execplan/jsonarrayaggcolumn.h +++ b/dbcon/execplan/jsonarrayaggcolumn.h @@ -47,25 +47,25 @@ class JsonArrayAggColumn : public AggregateColumn */ JsonArrayAggColumn(); - JsonArrayAggColumn(const uint32_t sessionID); + explicit JsonArrayAggColumn(const uint32_t sessionID); JsonArrayAggColumn(const JsonArrayAggColumn& rhs, const uint32_t sessionID = 0); /** * Destructors */ - virtual ~JsonArrayAggColumn(); + ~JsonArrayAggColumn() override; /** * Overloaded stream operator */ - virtual const std::string toString() const override; + const std::string toString() const override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - virtual JsonArrayAggColumn* clone() const override + JsonArrayAggColumn* clone() const override { return new JsonArrayAggColumn(*this); } @@ -93,8 +93,8 @@ class JsonArrayAggColumn : public AggregateColumn /** * Serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -102,7 +102,7 @@ class JsonArrayAggColumn : public AggregateColumn * @return true iff every member of t is a duplicate copy of every member of this; * false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -119,7 +119,7 @@ class JsonArrayAggColumn : public AggregateColumn * @return false iff every member of t is a duplicate copy of every member of this; * true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -130,7 +130,7 @@ class JsonArrayAggColumn : public AggregateColumn using AggregateColumn::operator!=; virtual bool operator!=(const JsonArrayAggColumn& t) const; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: std::vector fOrderCols; diff --git a/dbcon/execplan/logicoperator.h b/dbcon/execplan/logicoperator.h index 13a0126ff..3bc9d4cab 100644 --- a/dbcon/execplan/logicoperator.h +++ b/dbcon/execplan/logicoperator.h @@ -27,7 +27,7 @@ #include #include -//#include "expressionparser.h" +// #include "expressionparser.h" #include "operator.h" #include "parsetree.h" @@ -64,13 +64,13 @@ class LogicOperator : public Operator * Constructors */ LogicOperator(); - LogicOperator(const std::string& operatorName); + explicit LogicOperator(const std::string& operatorName); LogicOperator(const LogicOperator& rhs); /** * Destructors */ - virtual ~LogicOperator(); + ~LogicOperator() override; /** * Accessor Methods @@ -80,7 +80,7 @@ class LogicOperator : public Operator * * deep copy of this pointer and return the copy */ - inline virtual LogicOperator* clone() const override + inline LogicOperator* clone() const override { return new LogicOperator(*this); } @@ -88,15 +88,15 @@ class LogicOperator : public Operator /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -110,7 +110,7 @@ class LogicOperator : public Operator * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -123,7 +123,7 @@ class LogicOperator : public Operator // F&E framework using Operator::getBoolVal; - inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + inline bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { switch (fOp) { @@ -165,12 +165,12 @@ class LogicOperator : public Operator } using TreeNode::evaluate; - inline virtual void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override + inline void evaluate(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) override { fResult.boolVal = getBoolVal(row, isNull, lop, rop); } - inline virtual std::string toCppCode(IncludeSet& includes) const override + inline std::string toCppCode(IncludeSet& includes) const override { includes.insert("logicoperator.h"); std::stringstream ss; diff --git a/dbcon/execplan/mcsanalyzetableexecutionplan.h b/dbcon/execplan/mcsanalyzetableexecutionplan.h index 5dc6d8270..a33dfe504 100644 --- a/dbcon/execplan/mcsanalyzetableexecutionplan.h +++ b/dbcon/execplan/mcsanalyzetableexecutionplan.h @@ -54,7 +54,7 @@ class MCSAnalyzeTableExecutionPlan : public CalpontExecutionPlan { } - virtual ~MCSAnalyzeTableExecutionPlan() = default; + ~MCSAnalyzeTableExecutionPlan() override = default; const ReturnedColumnList& returnedCols() const { @@ -232,16 +232,16 @@ class MCSAnalyzeTableExecutionPlan : public CalpontExecutionPlan return ((fSessionID & 0x80000000) != 0); } - virtual void serialize(messageqcpp::ByteStream& bs) const; + void serialize(messageqcpp::ByteStream& bs) const override; - virtual void unserialize(messageqcpp::ByteStream& bs); + void unserialize(messageqcpp::ByteStream& bs) override; // TODO: Why do we need this? - virtual bool operator==(const CalpontExecutionPlan* t) const + bool operator==(const CalpontExecutionPlan* t) const override { return false; } - virtual bool operator!=(const CalpontExecutionPlan* t) const + bool operator!=(const CalpontExecutionPlan* t) const override { return false; } diff --git a/dbcon/execplan/objectreader.h b/dbcon/execplan/objectreader.h index 1bcd359c2..07c15fb3b 100644 --- a/dbcon/execplan/objectreader.h +++ b/dbcon/execplan/objectreader.h @@ -28,7 +28,7 @@ #include #include -#include +#include namespace messageqcpp { @@ -53,9 +53,9 @@ class ObjectReader class UnserializeException : public std::exception { public: - UnserializeException(std::string) throw(); - virtual ~UnserializeException() throw(); - virtual const char* what() const throw(); + explicit UnserializeException(std::string) noexcept; + ~UnserializeException() noexcept override; + const char* what() const noexcept override; private: std::string fWhat; diff --git a/dbcon/execplan/operator.h b/dbcon/execplan/operator.h index e1598fbe0..63b14be52 100644 --- a/dbcon/execplan/operator.h +++ b/dbcon/execplan/operator.h @@ -70,23 +70,23 @@ class Operator : public TreeNode { public: Operator(); - Operator(const std::string& operatorName); + explicit Operator(const std::string& operatorName); Operator(const Operator& rhs); - virtual ~Operator(); + ~Operator() override; - virtual const std::string toString() const override; - virtual const std::string data() const override + const std::string toString() const override; + const std::string data() const override { return fData; } - virtual void data(const std::string data) override; + void data(const std::string data) override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual Operator* clone() const override + inline Operator* clone() const override { return new Operator(*this); } @@ -101,15 +101,15 @@ class Operator : public TreeNode /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -123,7 +123,7 @@ class Operator : public TreeNode * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -139,7 +139,7 @@ class Operator : public TreeNode */ virtual void reverseOp(); - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; protected: std::string fData; @@ -231,11 +231,11 @@ class Operator : public TreeNode virtual void setOpType(Type& l, Type& r) { } - virtual void operationType(const Type& ot) override + void operationType(const Type& ot) override { fOperationType = ot; } - virtual const Type& operationType() const override + const Type& operationType() const override { return fOperationType; } diff --git a/dbcon/execplan/outerjoinonfilter.h b/dbcon/execplan/outerjoinonfilter.h index 1ba6ea3ae..9315aad22 100644 --- a/dbcon/execplan/outerjoinonfilter.h +++ b/dbcon/execplan/outerjoinonfilter.h @@ -49,9 +49,9 @@ class OuterJoinOnFilter : public Filter * Constructors */ OuterJoinOnFilter(); - OuterJoinOnFilter(const SPTP& pt); + explicit OuterJoinOnFilter(const SPTP& pt); OuterJoinOnFilter(const OuterJoinOnFilter& rhs); - virtual ~OuterJoinOnFilter(); + ~OuterJoinOnFilter() override; /** * Accessor Methods @@ -69,19 +69,19 @@ class OuterJoinOnFilter : public Filter * Overloaded stream operator */ // virtual std::ostream& operator<< (std::ostream& output); - virtual const std::string toString() const override; + const std::string toString() const override; /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual OuterJoinOnFilter* clone() const override + inline OuterJoinOnFilter* clone() const override { return new OuterJoinOnFilter(*this); } @@ -91,7 +91,7 @@ class OuterJoinOnFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -105,7 +105,7 @@ class OuterJoinOnFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -113,7 +113,8 @@ class OuterJoinOnFilter : public Filter * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ bool operator!=(const OuterJoinOnFilter& t) const; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; + private: // default okay? // OuterJoinOnFilter& operator=(const OuterJoinOnFilter& rhs); diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index c9c9f2b24..5c190fdc0 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -59,7 +59,7 @@ class ParseTree * Constructor / Destructor */ inline ParseTree(); - inline ParseTree(TreeNode* data); + inline explicit ParseTree(TreeNode* data); inline ParseTree(TreeNode* data, ParseTree* left, ParseTree* right); inline ParseTree(const ParseTree& rhs); inline virtual ~ParseTree(); @@ -210,9 +210,9 @@ class ParseTree inline static void deleter(ParseTree*& n) { delete n->fData; - n->fData = 0; + n->fData = nullptr; delete n; - n = 0; + n = nullptr; } inline void derivedTable(const std::string& derivedTable) @@ -238,7 +238,7 @@ class ParseTree { ParseTree* node; GoTo direction; - StackFrame(ParseTree* node_, GoTo direction_ = GoTo::Left) : node(node_), direction(direction_) + explicit StackFrame(ParseTree* node_, GoTo direction_ = GoTo::Left) : node(node_), direction(direction_) { } }; @@ -370,11 +370,11 @@ namespace execplan /** * Class Definition */ -inline ParseTree::ParseTree() : fData(0), fLeft(0), fRight(0), fDerivedTable("") +inline ParseTree::ParseTree() : fData(nullptr), fLeft(nullptr), fRight(nullptr), fDerivedTable("") { } -inline ParseTree::ParseTree(TreeNode* data) : fData(data), fLeft(0), fRight(0) +inline ParseTree::ParseTree(TreeNode* data) : fData(data), fLeft(nullptr), fRight(nullptr) { // bug5984. Need to validate data to be not null if (data) @@ -389,7 +389,7 @@ inline ParseTree::ParseTree(TreeNode* data, ParseTree* left, ParseTree* right) } inline ParseTree::ParseTree(const ParseTree& rhs) - : fData(0), fLeft(0), fRight(0), fDerivedTable(rhs.fDerivedTable) + : fData(nullptr), fLeft(nullptr), fRight(nullptr), fDerivedTable(rhs.fDerivedTable) { copyTree(rhs); } @@ -613,22 +613,22 @@ inline ParseTree& ParseTree::operator=(const ParseTree& rhs) inline void ParseTree::copyTree(const ParseTree& src) { - if (fLeft != NULL) + if (fLeft != nullptr) delete fLeft; - if (fRight != NULL) + if (fRight != nullptr) delete fRight; - fLeft = NULL; - fRight = NULL; + fLeft = nullptr; + fRight = nullptr; - if (src.left() != NULL) + if (src.left() != nullptr) { fLeft = new ParseTree(); fLeft->copyTree(*(src.left())); } - if (src.right() != NULL) + if (src.right() != nullptr) { fRight = new ParseTree(); fRight->copyTree(*(src.right())); @@ -636,29 +636,29 @@ inline void ParseTree::copyTree(const ParseTree& src) delete fData; - if (src.data() == NULL) - fData = NULL; + if (src.data() == nullptr) + fData = nullptr; else fData = src.data()->clone(); } inline void ParseTree::destroyTree(ParseTree* root) { - if (root == NULL) + if (root == nullptr) return; - if (root->left() != NULL) + if (root->left() != nullptr) { destroyTree(root->fLeft); } - if (root->right() != NULL) + if (root->right() != nullptr) { destroyTree(root->fRight); } delete root; - root = 0; + root = nullptr; } inline void ParseTree::draw(const ParseTree* n, std::ostream& dotFile) @@ -668,13 +668,11 @@ inline void ParseTree::draw(const ParseTree* n, std::ostream& dotFile) l = n->left(); r = n->right(); - if (l != 0) - dotFile << "n" << (void*)n << " -> " - << "n" << (void*)l << std::endl; + if (l != nullptr) + dotFile << "n" << (void*)n << " -> " << "n" << (void*)l << std::endl; - if (r != 0) - dotFile << "n" << (void*)n << " -> " - << "n" << (void*)r << std::endl; + if (r != nullptr) + dotFile << "n" << (void*)n << " -> " << "n" << (void*)r << std::endl; auto& node = *(n->data()); dotFile << "n" << (void*)n << " [label=\"" << n->data()->data() << " (" << n << ") " diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 8e3827e77..434c2c768 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -27,7 +27,7 @@ #include #include -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) #include #else #include @@ -52,15 +52,15 @@ class PredicateOperator : public Operator { public: PredicateOperator(); - PredicateOperator(const std::string& operatorName); + explicit PredicateOperator(const std::string& operatorName); PredicateOperator(const PredicateOperator& rhs); - virtual ~PredicateOperator(); + ~PredicateOperator() override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual PredicateOperator* clone() const override + inline PredicateOperator* clone() const override { return new PredicateOperator(*this); } @@ -68,15 +68,15 @@ class PredicateOperator : public Operator /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -90,7 +90,7 @@ class PredicateOperator : public Operator * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -107,10 +107,10 @@ class PredicateOperator : public Operator * F&E framework * ***********************************************************/ using Operator::getBoolVal; - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop) override; + bool getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop) override; void setOpType(Type& l, Type& r) override; - inline virtual std::string toCppCode(IncludeSet& includes) const override + inline std::string toCppCode(IncludeSet& includes) const override { includes.insert("predicateoperator.h"); std::stringstream ss; diff --git a/dbcon/execplan/pseudocolumn.h b/dbcon/execplan/pseudocolumn.h index 661d3269b..0f55e4844 100644 --- a/dbcon/execplan/pseudocolumn.h +++ b/dbcon/execplan/pseudocolumn.h @@ -62,7 +62,7 @@ class PseudoColumn : public SimpleColumn * Constructors */ PseudoColumn(); - PseudoColumn(const uint32_t pseudoType); + explicit PseudoColumn(const uint32_t pseudoType); PseudoColumn(const std::string& token, const uint32_t pseudoType, const uint32_t sessionID = 0); PseudoColumn(const std::string& schema, const std::string& table, const std::string& col, const uint32_t pseudoType, const uint32_t sessionID = 0); @@ -74,13 +74,13 @@ class PseudoColumn : public SimpleColumn /** * Destructor */ - virtual ~PseudoColumn(); + ~PseudoColumn() override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual PseudoColumn* clone() const override + inline PseudoColumn* clone() const override { return new PseudoColumn(*this); } @@ -106,17 +106,17 @@ class PseudoColumn : public SimpleColumn /** * The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; - virtual const std::string toString() const override; + const std::string toString() const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -130,7 +130,7 @@ class PseudoColumn : public SimpleColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -141,7 +141,7 @@ class PseudoColumn : public SimpleColumn static uint32_t pseudoNameToType(std::string& name); - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: /** diff --git a/dbcon/execplan/returnedcolumn.h b/dbcon/execplan/returnedcolumn.h index 2f77e7e1c..00031de8d 100644 --- a/dbcon/execplan/returnedcolumn.h +++ b/dbcon/execplan/returnedcolumn.h @@ -83,20 +83,20 @@ class ReturnedColumn : public TreeNode * Constructors */ ReturnedColumn(); - ReturnedColumn(const std::string& sql); - ReturnedColumn(const uint32_t sessionID, const bool returnAll = false); + explicit ReturnedColumn(const std::string& sql); + explicit ReturnedColumn(const uint32_t sessionID, const bool returnAll = false); ReturnedColumn(const ReturnedColumn& rhs, const uint32_t sessionID = 0); /** * Destructors */ - virtual ~ReturnedColumn(); + ~ReturnedColumn() override; /** * Accessor Methods */ - virtual const std::string data() const override; - virtual void data(const std::string data) override + const std::string data() const override; + void data(const std::string data) override { fData = data; } @@ -231,22 +231,22 @@ class ReturnedColumn : public TreeNode /** * Operations */ - virtual ReturnedColumn* clone() const override = 0; + ReturnedColumn* clone() const override = 0; /** * The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; - virtual const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; + const std::string toString() const override; + std::string toCppCode(IncludeSet& includes) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -261,7 +261,7 @@ class ReturnedColumn : public TreeNode * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * diff --git a/dbcon/execplan/rowcolumn.h b/dbcon/execplan/rowcolumn.h index 94fb666db..73e80b4b7 100644 --- a/dbcon/execplan/rowcolumn.h +++ b/dbcon/execplan/rowcolumn.h @@ -56,13 +56,13 @@ class RowColumn : public ReturnedColumn /** * Constructors */ - RowColumn(const uint32_t sessionID = 0); + explicit RowColumn(const uint32_t sessionID = 0); RowColumn(const RowColumn& rhs, const uint32_t sessionID = 0); - RowColumn(const std::vector& columnVec, const uint32_t sessionID = 0); + explicit RowColumn(const std::vector& columnVec, const uint32_t sessionID = 0); /** * Destructor */ - virtual ~RowColumn(); + ~RowColumn() override; /** * Accessor Methods @@ -80,7 +80,7 @@ class RowColumn : public ReturnedColumn * * deep copy of this pointer and return the copy */ - inline virtual RowColumn* clone() const override + inline RowColumn* clone() const override { return new RowColumn(*this); } @@ -95,20 +95,20 @@ class RowColumn : public ReturnedColumn // virtual void serialize(messageqcpp::ByteStream&) const; // virtual void unserialize(messageqcpp::ByteStream&); - virtual const std::string toString() const override; + const std::string toString() const override; /** * Serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -122,7 +122,7 @@ class RowColumn : public ReturnedColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -131,16 +131,16 @@ class RowColumn : public ReturnedColumn */ bool operator!=(const RowColumn& t) const; using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() override + bool hasAggregate() override { return false; } - virtual bool hasWindowFunc() override + bool hasWindowFunc() override { return false; } - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: /** @@ -156,23 +156,21 @@ class SubSelect : public ReturnedColumn SubSelect() : ReturnedColumn() { } - ~SubSelect() - { - } + ~SubSelect() override = default; SubSelect* clone() const override { return new SubSelect(); } using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() override + bool hasAggregate() override { return false; } - virtual bool hasWindowFunc() override + bool hasWindowFunc() override { return false; } - virtual const std::string toString() const override; + const std::string toString() const override; }; /** diff --git a/dbcon/execplan/selectfilter.h b/dbcon/execplan/selectfilter.h index d920e282c..712550333 100644 --- a/dbcon/execplan/selectfilter.h +++ b/dbcon/execplan/selectfilter.h @@ -63,7 +63,7 @@ class SelectFilter : public Filter /** * Destructors */ - virtual ~SelectFilter(); + ~SelectFilter() override; /** * Accessor Methods @@ -107,13 +107,13 @@ class SelectFilter : public Filter fCorrelated = correlated; } - virtual const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; - virtual inline const std::string data() const override + const std::string toString() const override; + std::string toCppCode(IncludeSet& includes) const override; + inline const std::string data() const override { return fData; } - virtual inline void data(const std::string data) override + inline void data(const std::string data) override { fData = data; } @@ -130,14 +130,14 @@ class SelectFilter : public Filter /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual SelectFilter* clone() const override + inline SelectFilter* clone() const override { return new SelectFilter(*this); } @@ -147,7 +147,7 @@ class SelectFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -161,7 +161,7 @@ class SelectFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * diff --git a/dbcon/execplan/sessionmanager.h b/dbcon/execplan/sessionmanager.h index b7e4c9685..0ea6a5b1a 100644 --- a/dbcon/execplan/sessionmanager.h +++ b/dbcon/execplan/sessionmanager.h @@ -67,7 +67,7 @@ namespace execplan * immediately, causing all subsequent references to fail. This only affects * 'leakcheck'. */ -//#define DESTROYSHMSEG +// #define DESTROYSHMSEG class SessionManager { @@ -94,7 +94,7 @@ class SessionManager * and no operation other than reset() should be performed on a * SessionManager instantiated with this. */ - SessionManager(bool nolock); + explicit SessionManager(bool nolock); /** @brief Destructor * @@ -211,4 +211,3 @@ class SessionManager }; } // namespace execplan - diff --git a/dbcon/execplan/sessionmonitor.h b/dbcon/execplan/sessionmonitor.h index 90a093367..1b1ba541e 100644 --- a/dbcon/execplan/sessionmonitor.h +++ b/dbcon/execplan/sessionmonitor.h @@ -36,7 +36,7 @@ #include "shmkeys.h" #include "brmtypes.h" -//#define SM_DEBUG +// #define SM_DEBUG namespace execplan { @@ -132,7 +132,7 @@ class SessionMonitor { txnCount = 0; verID = 0; - activeTxns = NULL; + activeTxns = nullptr; }; }; typedef struct SessionMonitorData_struct SessionMonitorData_t; diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index 836c927f5..241bb66e7 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -57,13 +57,15 @@ class SimpleColumn : public ReturnedColumn /** * Constructors */ - class ForTestPurposeWithoutOID{}; + class ForTestPurposeWithoutOID + { + }; SimpleColumn(); SimpleColumn(const std::string& token, ForTestPurposeWithoutOID); - SimpleColumn(const std::string& token, const uint32_t sessionID = 0); + explicit SimpleColumn(const std::string& token, const uint32_t sessionID = 0); SimpleColumn(const std::string& schema, const std::string& table, const std::string& col, const uint32_t sessionID = 0, const int lower_case_table_names = 0); @@ -75,7 +77,7 @@ class SimpleColumn : public ReturnedColumn /** * Destructor */ - virtual ~SimpleColumn(); + ~SimpleColumn() override; /** * Accessor Methods @@ -124,8 +126,8 @@ class SimpleColumn : public ReturnedColumn fOid = oid; } - virtual const std::string data() const override; - virtual void data(const std::string data) override + const std::string data() const override; + void data(const std::string data) override { fData = data; } @@ -179,7 +181,7 @@ class SimpleColumn : public ReturnedColumn * * deep copy of this pointer and return the copy */ - inline virtual SimpleColumn* clone() const override + inline SimpleColumn* clone() const override { return new SimpleColumn(*this); } @@ -191,17 +193,17 @@ class SimpleColumn : public ReturnedColumn /** * The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; - virtual const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; + const std::string toString() const override; + std::string toCppCode(IncludeSet& includes) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -217,7 +219,7 @@ class SimpleColumn : public ReturnedColumn * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -227,7 +229,7 @@ class SimpleColumn : public ReturnedColumn bool operator!=(const SimpleColumn& t) const; /** @brief check if this column is the same as the argument */ - virtual bool sameColumn(const ReturnedColumn* rc) const override; + bool sameColumn(const ReturnedColumn* rc) const override; /** @brief return column type of this column (could be of any engine type) */ const CalpontSystemCatalog::ColType& colType() const @@ -238,7 +240,7 @@ class SimpleColumn : public ReturnedColumn /** @brief set the column's OID from the syscat */ void setOID(); - virtual bool hasWindowFunc() override + bool hasWindowFunc() override { return false; } @@ -252,7 +254,7 @@ class SimpleColumn : public ReturnedColumn * @return true, if all arguments belong to one table * false, if multiple tables are involved in the function */ - virtual bool singleTable(CalpontSystemCatalog::TableAliasName& tan) override; + bool singleTable(CalpontSystemCatalog::TableAliasName& tan) override; protected: /** @@ -282,13 +284,13 @@ class SimpleColumn : public ReturnedColumn * F&E framework * ***********************************************************/ public: - virtual void evaluate(rowgroup::Row& row, bool& isNull) override; - virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) override + void evaluate(rowgroup::Row& row, bool& isNull) override; + bool getBoolVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getBoolVal(); } - virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override + const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override { bool localIsNull = false; evaluate(row, localIsNull); @@ -301,37 +303,37 @@ class SimpleColumn : public ReturnedColumn return TreeNode::getStrVal(fTimeZone); } - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getIntVal(); } - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override + uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getUintVal(); } - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) override + float getFloatVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getFloatVal(); } - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) override + double getDoubleVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDoubleVal(); } - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override + long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getLongDoubleVal(); } - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override + IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); diff --git a/dbcon/execplan/simplecolumn_decimal.h b/dbcon/execplan/simplecolumn_decimal.h index 8d4fd388b..2693962de 100644 --- a/dbcon/execplan/simplecolumn_decimal.h +++ b/dbcon/execplan/simplecolumn_decimal.h @@ -54,35 +54,33 @@ class SimpleColumn_Decimal : public SimpleColumn public: /** Constructors */ SimpleColumn_Decimal(); - SimpleColumn_Decimal(const std::string& token, const uint32_t sessionID = 0); + explicit SimpleColumn_Decimal(const std::string& token, const uint32_t sessionID = 0); SimpleColumn_Decimal(const std::string& schema, const std::string& table, const std::string& col, const bool isColumnStore, const uint32_t sessionID = 0); - SimpleColumn_Decimal(const SimpleColumn& rhs, const uint32_t sessionID = 0); + explicit SimpleColumn_Decimal(const SimpleColumn& rhs, const uint32_t sessionID = 0); /** Destructor */ - virtual ~SimpleColumn_Decimal() - { - } + ~SimpleColumn_Decimal() override = default; - inline virtual SimpleColumn_Decimal* clone() const override + inline SimpleColumn_Decimal* clone() const override { return new SimpleColumn_Decimal(*this); } /** Evaluate methods */ - virtual inline const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override; - virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) override; - virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull) override; - virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull) override; - virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override; - virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override; + inline const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override; + inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) override; + inline float getFloatVal(rowgroup::Row& row, bool& isNull) override; + inline double getDoubleVal(rowgroup::Row& row, bool& isNull) override; + inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override; + inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override; /** The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; uint64_t fNullVal; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: void setNullVal(); diff --git a/dbcon/execplan/simplecolumn_int.h b/dbcon/execplan/simplecolumn_int.h index 0533c3596..2ceb78ec5 100644 --- a/dbcon/execplan/simplecolumn_int.h +++ b/dbcon/execplan/simplecolumn_int.h @@ -53,36 +53,34 @@ class SimpleColumn_INT : public SimpleColumn public: /** Constructors */ SimpleColumn_INT(); - SimpleColumn_INT(const std::string& token, const uint32_t sessionID = 0); + explicit SimpleColumn_INT(const std::string& token, const uint32_t sessionID = 0); SimpleColumn_INT(const std::string& schema, const std::string& table, const std::string& col, const bool isColumnStore, const uint32_t sessionID = 0); - SimpleColumn_INT(const SimpleColumn& rhs, const uint32_t sessionID = 0); + explicit SimpleColumn_INT(const SimpleColumn& rhs, const uint32_t sessionID = 0); /** Destructor */ - virtual ~SimpleColumn_INT() - { - } + ~SimpleColumn_INT() override = default; - inline virtual SimpleColumn_INT* clone() const override + inline SimpleColumn_INT* clone() const override { return new SimpleColumn_INT(*this); } /** Evaluate methods */ - virtual inline const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override; - virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) override; - virtual inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override; - virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull) override; - virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull) override; - virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override; - virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override; + inline const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override; + inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) override; + inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override; + inline float getFloatVal(rowgroup::Row& row, bool& isNull) override; + inline double getDoubleVal(rowgroup::Row& row, bool& isNull) override; + inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override; + inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override; /** The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; uint64_t fNullVal; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: void setNullVal(); @@ -93,8 +91,8 @@ std::string SimpleColumn_INT::toCppCode(IncludeSet& includes) const { includes.insert("simplecolumn_int.h"); std::stringstream ss; - ss << "SimpleColumn_INT<" << len << ">(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", " << - std::quoted(fColumnName) << ", " << fisColumnStore << ", " << sessionID() << ")"; + ss << "SimpleColumn_INT<" << len << ">(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) + << ", " << std::quoted(fColumnName) << ", " << fisColumnStore << ", " << sessionID() << ")"; return ss.str(); } @@ -146,7 +144,7 @@ void SimpleColumn_INT::setNullVal() } template -inline const utils::NullString & SimpleColumn_INT::getStrVal(rowgroup::Row& row, bool& isNull) +inline const utils::NullString& SimpleColumn_INT::getStrVal(rowgroup::Row& row, bool& isNull) { if (row.equals(fNullVal, fInputIndex)) { diff --git a/dbcon/execplan/simplecolumn_uint.h b/dbcon/execplan/simplecolumn_uint.h index 2e5b67ab0..4fa6fb5a7 100644 --- a/dbcon/execplan/simplecolumn_uint.h +++ b/dbcon/execplan/simplecolumn_uint.h @@ -53,36 +53,34 @@ class SimpleColumn_UINT : public SimpleColumn public: /** Constructors */ SimpleColumn_UINT(); - SimpleColumn_UINT(const std::string& token, const uint32_t sessionID = 0); + explicit SimpleColumn_UINT(const std::string& token, const uint32_t sessionID = 0); SimpleColumn_UINT(const std::string& schema, const std::string& table, const std::string& col, const bool isColumnStore, const uint32_t sessionID = 0); - SimpleColumn_UINT(const SimpleColumn& rhs, const uint32_t sessionID = 0); + explicit SimpleColumn_UINT(const SimpleColumn& rhs, const uint32_t sessionID = 0); /** Destructor */ - virtual ~SimpleColumn_UINT() - { - } + ~SimpleColumn_UINT() override = default; - inline virtual SimpleColumn_UINT* clone() const override + inline SimpleColumn_UINT* clone() const override { return new SimpleColumn_UINT(*this); } /** Evaluate methods */ - virtual inline const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override; - virtual inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) override; - virtual inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override; - virtual inline float getFloatVal(rowgroup::Row& row, bool& isNull) override; - virtual inline double getDoubleVal(rowgroup::Row& row, bool& isNull) override; - virtual inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override; - virtual inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override; + inline const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override; + inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) override; + inline uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override; + inline float getFloatVal(rowgroup::Row& row, bool& isNull) override; + inline double getDoubleVal(rowgroup::Row& row, bool& isNull) override; + inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override; + inline IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override; /** The serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; uint64_t fNullVal; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: void setNullVal(); diff --git a/dbcon/execplan/simplefilter.h b/dbcon/execplan/simplefilter.h index bc478ad0a..a732e5762 100644 --- a/dbcon/execplan/simplefilter.h +++ b/dbcon/execplan/simplefilter.h @@ -65,17 +65,19 @@ class SimpleFilter : public Filter SEMI }; - struct ForTestPurposesWithoutColumnsOIDS{}; + struct ForTestPurposesWithoutColumnsOIDS + { + }; SimpleFilter(); - SimpleFilter(const std::string& sql); + explicit SimpleFilter(const std::string& sql); SimpleFilter(const std::string& sql, ForTestPurposesWithoutColumnsOIDS); SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const long timeZone = 0); SimpleFilter(const SimpleFilter& rhs); - virtual ~SimpleFilter(); + ~SimpleFilter() override; - inline virtual SimpleFilter* clone() const override + inline SimpleFilter* clone() const override { return new SimpleFilter(*this); } @@ -106,7 +108,7 @@ class SimpleFilter : public Filter } using Filter::data; - virtual const std::string data() const override; + const std::string data() const override; /** assign fLhs * @@ -129,20 +131,20 @@ class SimpleFilter : public Filter */ void rhs(ReturnedColumn* rhs); - virtual const std::string toString() const override; + const std::string toString() const override; /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true if every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -160,7 +162,6 @@ class SimpleFilter : public Filter bool operator<(const SimpleFilter& t) const; - bool semanticEq(const SimpleFilter& t) const; /** @brief Do a deep, strict (as opposed to semantic) equivalence test @@ -168,7 +169,7 @@ class SimpleFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return false if every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -221,7 +222,7 @@ class SimpleFilter : public Filter static std::string escapeString(const std::string& input); - virtual string toCppCode(IncludeSet& includes) const override; + string toCppCode(IncludeSet& includes) const override; private: SOP fOp; /// operator @@ -237,10 +238,10 @@ class SimpleFilter : public Filter * F&E framework * ***********************************************************/ public: - inline virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) override; - inline virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) override; - inline virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) override; - inline virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override; + inline bool getBoolVal(rowgroup::Row& row, bool& isNull) override; + inline int64_t getIntVal(rowgroup::Row& row, bool& isNull) override; + inline double getDoubleVal(rowgroup::Row& row, bool& isNull) override; + inline long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override; // get all simple columns involved in this column const std::vector& simpleColumnList(); diff --git a/dbcon/execplan/simplescalarfilter.h b/dbcon/execplan/simplescalarfilter.h index c7a37a193..dc2e6f3d2 100644 --- a/dbcon/execplan/simplescalarfilter.h +++ b/dbcon/execplan/simplescalarfilter.h @@ -65,7 +65,7 @@ class SimpleScalarFilter : public Filter /** * Destructors */ - virtual ~SimpleScalarFilter(); + ~SimpleScalarFilter() override; /** * Accessor Methods @@ -100,13 +100,13 @@ class SimpleScalarFilter : public Filter fSub = sub; } - virtual const std::string toString() const override; + const std::string toString() const override; - virtual inline const std::string data() const override + inline const std::string data() const override { return fData; } - virtual inline void data(const std::string data) override + inline void data(const std::string data) override { fData = data; } @@ -114,14 +114,14 @@ class SimpleScalarFilter : public Filter /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - inline virtual SimpleScalarFilter* clone() const override + inline SimpleScalarFilter* clone() const override { return new SimpleScalarFilter(*this); } @@ -131,7 +131,7 @@ class SimpleScalarFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -145,7 +145,7 @@ class SimpleScalarFilter : public Filter * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -154,7 +154,7 @@ class SimpleScalarFilter : public Filter */ bool operator!=(const SimpleScalarFilter& t) const; - virtual string toCppCode(IncludeSet& includes) const override; + string toCppCode(IncludeSet& includes) const override; private: // default okay? diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 710c82eaa..25ee452b5 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -269,7 +269,9 @@ class TreeNode ***********************************************************************/ virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) { - isNull = isNull || fResult.strVal.isNull(); // XXX: NullString returns isNull, we should remove that parameter altogether. + isNull = isNull || + fResult.strVal + .isNull(); // XXX: NullString returns isNull, we should remove that parameter altogether. return fResult.strVal; } virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) @@ -475,19 +477,20 @@ inline const utils::NullString& TreeNode::getStrVal(const long timeZone) case CalpontSystemCatalog::VARCHAR: if (fResultType.colWidth <= 7) { - const char *intAsChar = (const char*) (&fResult.origIntVal); + const char* intAsChar = (const char*)(&fResult.origIntVal); fResult.strVal.assign((const uint8_t*)intAsChar, strlen(intAsChar)); } break; case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARBINARY: // XXX: TODO: we don't have varbinary support now, but it may be handled just like varchar. + case CalpontSystemCatalog::VARBINARY: // XXX: TODO: we don't have varbinary support now, but it may be + // handled just like varchar. case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: if (fResultType.colWidth <= 8) { - const char *intAsChar = (const char*) (&fResult.origIntVal); + const char* intAsChar = (const char*)(&fResult.origIntVal); fResult.strVal.assign((const uint8_t*)intAsChar, strlen(intAsChar)); } @@ -865,27 +868,27 @@ inline double TreeNode::getDoubleVal() { case CalpontSystemCatalog::CHAR: if (fResultType.colWidth <= 8) - return strtod((char*)(&fResult.origIntVal), NULL); + return strtod((char*)(&fResult.origIntVal), nullptr); idbassert(fResult.strVal.str()); - return strtod(fResult.strVal.str(), NULL); + return strtod(fResult.strVal.str(), nullptr); case CalpontSystemCatalog::VARCHAR: if (fResultType.colWidth <= 7) - return strtod((char*)(&fResult.origIntVal), NULL); + return strtod((char*)(&fResult.origIntVal), nullptr); idbassert(fResult.strVal.str()); - return strtod(fResult.strVal.str(), NULL); + return strtod(fResult.strVal.str(), nullptr); // FIXME: ??? case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: if (fResultType.colWidth <= 7) - return strtod((char*)(&fResult.origIntVal), NULL); + return strtod((char*)(&fResult.origIntVal), nullptr); - //idbassert(fResult.strVal.str()); - return strtod(fResult.strVal.safeString("").c_str(), NULL); + // idbassert(fResult.strVal.str()); + return strtod(fResult.strVal.safeString("").c_str(), nullptr); case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::TINYINT: @@ -936,27 +939,27 @@ inline long double TreeNode::getLongDoubleVal() { case CalpontSystemCatalog::CHAR: if (fResultType.colWidth <= 8) - return strtold((char*)(&fResult.origIntVal), NULL); + return strtold((char*)(&fResult.origIntVal), nullptr); idbassert(fResult.strVal.str()); - return strtold(fResult.strVal.str(), NULL); + return strtold(fResult.strVal.str(), nullptr); case CalpontSystemCatalog::VARCHAR: if (fResultType.colWidth <= 7) - return strtold((char*)(&fResult.origIntVal), NULL); + return strtold((char*)(&fResult.origIntVal), nullptr); idbassert(fResult.strVal.str()); - return strtold(fResult.strVal.str(), NULL); + return strtold(fResult.strVal.str(), nullptr); // FIXME: ??? case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: if (fResultType.colWidth <= 7) - return strtold((char*)(&fResult.origIntVal), NULL); + return strtold((char*)(&fResult.origIntVal), nullptr); idbassert(fResult.strVal.str()); - return strtold(fResult.strVal.str(), NULL); + return strtold(fResult.strVal.str(), nullptr); case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::TINYINT: diff --git a/dbcon/execplan/treenodeimpl.h b/dbcon/execplan/treenodeimpl.h index bae434ff6..bf98495f7 100644 --- a/dbcon/execplan/treenodeimpl.h +++ b/dbcon/execplan/treenodeimpl.h @@ -50,14 +50,14 @@ class TreeNodeImpl : public TreeNode * Constructors */ TreeNodeImpl(); - TreeNodeImpl(const std::string& sql); + explicit TreeNodeImpl(const std::string& sql); // not needed yet // TreeNodeImpl(const TreeNodeImpl& rhs); /** * Destructors */ - virtual ~TreeNodeImpl(); + ~TreeNodeImpl() override; /** * Accessor Methods */ @@ -65,13 +65,13 @@ class TreeNodeImpl : public TreeNode /** * Operations */ - virtual const std::string toString() const override; + const std::string toString() const override; - virtual const std::string data() const override + const std::string data() const override { return fData; } - virtual void data(const std::string data) override + void data(const std::string data) override { fData = data; } @@ -80,7 +80,7 @@ class TreeNodeImpl : public TreeNode * * deep copy of this pointer and return the copy */ - inline virtual TreeNodeImpl* clone() const override + inline TreeNodeImpl* clone() const override { return new TreeNodeImpl(*this); } @@ -88,15 +88,15 @@ class TreeNodeImpl : public TreeNode /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * * Do a deep, strict (as opposed to semantic) equivalence test. * @return true iff every member of t is a duplicate copy of every member of this; false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -110,7 +110,7 @@ class TreeNodeImpl : public TreeNode * Do a deep, strict (as opposed to semantic) equivalence test. * @return false iff every member of t is a duplicate copy of every member of this; true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -119,7 +119,8 @@ class TreeNodeImpl : public TreeNode */ bool operator!=(const TreeNodeImpl& t) const; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; + private: // default okay // TreeNodeImpl& operator=(const TreeNodeImpl& rhs); diff --git a/dbcon/execplan/udafcolumn.h b/dbcon/execplan/udafcolumn.h index 95191ddac..4691d0aa0 100644 --- a/dbcon/execplan/udafcolumn.h +++ b/dbcon/execplan/udafcolumn.h @@ -48,25 +48,25 @@ class UDAFColumn : public AggregateColumn */ UDAFColumn(); - UDAFColumn(const uint32_t sessionID); + explicit UDAFColumn(const uint32_t sessionID); UDAFColumn(const UDAFColumn& rhs, const uint32_t sessionID = 0); /** * Destructors */ - virtual ~UDAFColumn(); + ~UDAFColumn() override; /** * Overloaded stream operator */ - virtual const std::string toString() const override; + const std::string toString() const override; /** return a copy of this pointer * * deep copy of this pointer and return the copy */ - virtual UDAFColumn* clone() const override + UDAFColumn* clone() const override { return new UDAFColumn(*this); } @@ -82,8 +82,8 @@ class UDAFColumn : public AggregateColumn /** * Serialize interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -91,7 +91,7 @@ class UDAFColumn : public AggregateColumn * @return true iff every member of t is a duplicate copy of every member of this; * false otherwise */ - virtual bool operator==(const TreeNode* t) const override; + bool operator==(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -108,7 +108,7 @@ class UDAFColumn : public AggregateColumn * @return false iff every member of t is a duplicate copy of every member of this; * true otherwise */ - virtual bool operator!=(const TreeNode* t) const override; + bool operator!=(const TreeNode* t) const override; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -119,7 +119,7 @@ class UDAFColumn : public AggregateColumn using AggregateColumn::operator!=; virtual bool operator!=(const UDAFColumn& t) const; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; private: mcsv1sdk::mcsv1Context context; diff --git a/dbcon/execplan/wf_frame.h b/dbcon/execplan/wf_frame.h index a4280c7cf..bb67270c8 100644 --- a/dbcon/execplan/wf_frame.h +++ b/dbcon/execplan/wf_frame.h @@ -43,15 +43,11 @@ enum WF_FRAME struct WF_Boundary { - WF_Boundary() - { - } - WF_Boundary(WF_FRAME frame) : fFrame(frame) - { - } - ~WF_Boundary() + WF_Boundary() = default; + explicit WF_Boundary(WF_FRAME frame) : fFrame(frame) { } + ~WF_Boundary() = default; const std::string toString() const; void serialize(messageqcpp::ByteStream&) const; void unserialize(messageqcpp::ByteStream&); @@ -67,9 +63,7 @@ struct WF_Frame fStart.fFrame = WF_UNBOUNDED_PRECEDING; fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; } - ~WF_Frame() - { - } + ~WF_Frame() = default; const std::string toString() const; void serialize(messageqcpp::ByteStream&) const; void unserialize(messageqcpp::ByteStream&); @@ -83,13 +77,11 @@ struct WF_Frame */ struct WF_OrderBy { - WF_OrderBy() + WF_OrderBy() = default; + explicit WF_OrderBy(std::vector orders) : fOrders(orders) { } - WF_OrderBy(std::vector orders) : fOrders(orders) - { - } - ~WF_OrderBy(){}; + ~WF_OrderBy() = default; const std::string toString() const; void serialize(messageqcpp::ByteStream&) const; void unserialize(messageqcpp::ByteStream&); diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 67f90e885..213ba0abb 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -54,14 +54,12 @@ class WindowFunctionColumn : public ReturnedColumn { public: WindowFunctionColumn(); - WindowFunctionColumn(const std::string& functionName, const uint32_t sessionID = 0); + explicit WindowFunctionColumn(const std::string& functionName, const uint32_t sessionID = 0); WindowFunctionColumn(const std::string& functionName, const std::vector& functionParms, const std::vector& partitions, WF_OrderBy& orderby, const uint32_t sessionID = 0); WindowFunctionColumn(const WindowFunctionColumn& rhs, const uint32_t sessionID = 0); - virtual ~WindowFunctionColumn() - { - } + ~WindowFunctionColumn() override = default; /** get function name */ inline const std::string& functionName() const @@ -112,7 +110,7 @@ class WindowFunctionColumn : public ReturnedColumn } /** make a clone of this window function */ - inline virtual WindowFunctionColumn* clone() const override + inline WindowFunctionColumn* clone() const override { return new WindowFunctionColumn(*this); } @@ -122,22 +120,22 @@ class WindowFunctionColumn : public ReturnedColumn /** output the function for debug purpose */ const std::string toString() const override; - virtual std::string toCppCode(IncludeSet& includes) const override; + std::string toCppCode(IncludeSet& includes) const override; /** * The serialization interface */ - virtual void serialize(messageqcpp::ByteStream&) const override; - virtual void unserialize(messageqcpp::ByteStream&) override; + void serialize(messageqcpp::ByteStream&) const override; + void unserialize(messageqcpp::ByteStream&) override; // util function for connector to use. void addToPartition(std::vector& groupByList); using ReturnedColumn::hasAggregate; - virtual bool hasAggregate() override + bool hasAggregate() override { return false; } - virtual bool hasWindowFunc() override; + bool hasWindowFunc() override; void adjustResultType(); // UDAnF support @@ -170,12 +168,12 @@ class WindowFunctionColumn : public ReturnedColumn WF_OrderBy fOrderBy; /// order by clause // not support for window functions for now. - virtual bool operator==(const TreeNode* t) const override + bool operator==(const TreeNode* t) const override { return false; } bool operator==(const WindowFunctionColumn& t) const; - virtual bool operator!=(const TreeNode* t) const override + bool operator!=(const TreeNode* t) const override { return false; } @@ -190,7 +188,7 @@ class WindowFunctionColumn : public ReturnedColumn ***********************************************************/ public: using TreeNode::getStrVal; - virtual const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override + const utils::NullString& getStrVal(rowgroup::Row& row, bool& isNull) override { bool localIsNull = false; evaluate(row, localIsNull); @@ -198,57 +196,57 @@ class WindowFunctionColumn : public ReturnedColumn return localIsNull ? fResult.strVal.dropString() : TreeNode::getStrVal(fTimeZone); } - virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getIntVal(); } - virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override + uint64_t getUintVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getUintVal(); } - virtual float getFloatVal(rowgroup::Row& row, bool& isNull) override + float getFloatVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getFloatVal(); } - virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) override + double getDoubleVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDoubleVal(); } - virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override + long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getLongDoubleVal(); } - virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override + IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDecimalVal(); } - virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override + int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDateIntVal(); } - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getDatetimeIntVal(); } - virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getTimestampIntVal(); } - virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override + int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) override { evaluate(row, isNull); return TreeNode::getTimeIntVal(); diff --git a/dbcon/joblist/bandeddl.h b/dbcon/joblist/bandeddl.h index 6f85eb5f4..e27177974 100644 --- a/dbcon/joblist/bandeddl.h +++ b/dbcon/joblist/bandeddl.h @@ -29,7 +29,7 @@ #include #include "largedatalist.h" -//#include "bucketdl.h" +// #include "bucketdl.h" #include @@ -68,9 +68,9 @@ class BandedDL : public LargeDataList, element_t> protected: private: - explicit BandedDL(){}; - explicit BandedDL(const BandedDL&){}; - BandedDL& operator=(const BandedDL&){}; + explicit BandedDL() = default; + BandedDL(const BandedDL&){}; + BandedDL& operator=(const BandedDL&) {}; // vars to support the WSDL-like next() fcn boost::condition nextSetLoaded; diff --git a/dbcon/joblist/bucketdl.h b/dbcon/joblist/bucketdl.h index 3af942da2..714f2d537 100644 --- a/dbcon/joblist/bucketdl.h +++ b/dbcon/joblist/bucketdl.h @@ -161,7 +161,7 @@ class BucketDL : public DataList private: // Declare default constructors but don't define to disable their use explicit BucketDL(); - explicit BucketDL(const BucketDL&); + BucketDL(const BucketDL&); BucketDL& operator=(const BucketDL&); ResourceManager* fRm; diff --git a/dbcon/joblist/columncommand-jl.h b/dbcon/joblist/columncommand-jl.h index 037022581..096dbaa19 100644 --- a/dbcon/joblist/columncommand-jl.h +++ b/dbcon/joblist/columncommand-jl.h @@ -40,18 +40,17 @@ namespace joblist class ColumnCommandJL : public CommandJL { public: - ColumnCommandJL(const pColScanStep&, std::vector lastLBID, - bool hasAuxCol_, const std::vector& extentsAux_, - execplan::CalpontSystemCatalog::OID oidAux); - ColumnCommandJL(const pColStep&); + ColumnCommandJL(const pColScanStep&, std::vector lastLBID, bool hasAuxCol_, + const std::vector& extentsAux_, execplan::CalpontSystemCatalog::OID oidAux); + explicit ColumnCommandJL(const pColStep&); ColumnCommandJL(const ColumnCommandJL&, const DictStepJL&); - virtual ~ColumnCommandJL(); + ~ColumnCommandJL() override; - virtual void createCommand(messageqcpp::ByteStream& bs) const override; - virtual void runCommand(messageqcpp::ByteStream& bs) const override; + void createCommand(messageqcpp::ByteStream& bs) const override; + void runCommand(messageqcpp::ByteStream& bs) const override; void setLBID(uint64_t rid, uint32_t dbroot) override; uint8_t getTableColumnType() override; - virtual std::string toString() override; + std::string toString() override; uint16_t getWidth() override; CommandType getCommandType() override { diff --git a/dbcon/joblist/crossenginestep.h b/dbcon/joblist/crossenginestep.h index 1037269d7..b72c4d753 100644 --- a/dbcon/joblist/crossenginestep.h +++ b/dbcon/joblist/crossenginestep.h @@ -60,76 +60,76 @@ class CrossEngineStep : public BatchPrimitive, public TupleDeliveryStep /** @brief CrossEngineStep destructor */ - ~CrossEngineStep(); + ~CrossEngineStep() override; /** @brief virtual void Run method */ - void run(); + void run() override; /** @brief virtual void join method */ - void join(); + void join() override; /** @brief virtual string toString method */ - const std::string toString() const; + const std::string toString() const override; // from BatchPrimitive - bool getFeederFlag() const + bool getFeederFlag() const override { return false; } - uint64_t getLastTupleId() const + uint64_t getLastTupleId() const override { return 0; } - uint32_t getStepCount() const + uint32_t getStepCount() const override { return 1; } - void setBPP(JobStep* jobStep); - void setFirstStepType(PrimitiveStepType firstStepType) + void setBPP(JobStep* jobStep) override; + void setFirstStepType(PrimitiveStepType firstStepType) override { } - void setIsProjectionOnly() + void setIsProjectionOnly() override { } - void setLastTupleId(uint64_t id) + void setLastTupleId(uint64_t id) override { } - void setOutputType(BPSOutputType outputType) + void setOutputType(BPSOutputType outputType) override { } - void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2); - void setStepCount() + void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2) override; + void setStepCount() override { } - void setSwallowRows(const bool swallowRows) + void setSwallowRows(const bool swallowRows) override { } - void setBppStep() + void setBppStep() override { } - void dec(DistributedEngineComm* dec) + void dec(DistributedEngineComm* dec) override { } - const OIDVector& getProjectOids() const + const OIDVector& getProjectOids() const override { return fOIDVector; } - uint64_t blksSkipped() const + uint64_t blksSkipped() const override { return 0; } - bool wasStepRun() const + bool wasStepRun() const override { return fRunExecuted; } - BPSOutputType getOutputType() const + BPSOutputType getOutputType() const override { return ROW_GROUP; } - uint64_t getRows() const + uint64_t getRows() const override { return fRowsReturned; } @@ -145,27 +145,27 @@ class CrossEngineStep : public BatchPrimitive, public TupleDeliveryStep { return fAlias; } - void setJobInfo(const JobInfo* jobInfo) + void setJobInfo(const JobInfo* jobInfo) override { } - void setOutputRowGroup(const rowgroup::RowGroup&); - const rowgroup::RowGroup& getOutputRowGroup() const; + void setOutputRowGroup(const rowgroup::RowGroup&) override; + const rowgroup::RowGroup& getOutputRowGroup() const override; // from DECEventListener - void newPMOnline(uint32_t) + void newPMOnline(uint32_t) override { } - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + const rowgroup::RowGroup& getDeliveredRowGroup() const override; + void deliverStringTableRowGroup(bool b) override; + bool deliverStringTableRowGroup() const override; + uint32_t nextBand(messageqcpp::ByteStream& bs) override; - void addFcnJoinExp(const std::vector&); - void addFcnExpGroup1(const boost::shared_ptr&); - void setFE1Input(const rowgroup::RowGroup&); - void setFcnExpGroup3(const std::vector&); - void setFE23Output(const rowgroup::RowGroup&); + void addFcnJoinExp(const std::vector&) override; + void addFcnExpGroup1(const boost::shared_ptr&) override; + void setFE1Input(const rowgroup::RowGroup&) override; + void setFcnExpGroup3(const std::vector&) override; + void setFE23Output(const rowgroup::RowGroup&) override; void addFilter(JobStep* jobStep); void addProject(JobStep* jobStep); @@ -201,7 +201,7 @@ class CrossEngineStep : public BatchPrimitive, public TupleDeliveryStep class Runner { public: - Runner(CrossEngineStep* step) : fStep(step) + explicit Runner(CrossEngineStep* step) : fStep(step) { } void operator()() @@ -248,4 +248,3 @@ class CrossEngineStep : public BatchPrimitive, public TupleDeliveryStep }; } // namespace joblist - diff --git a/dbcon/joblist/datalist.h b/dbcon/joblist/datalist.h index 669d52931..6c1b253d4 100644 --- a/dbcon/joblist/datalist.h +++ b/dbcon/joblist/datalist.h @@ -161,9 +161,10 @@ DataList::DataList() , consumersFinished(0) , fElemDiskFirstSize(sizeof(uint64_t)) , fElemDiskSecondSize(sizeof(uint64_t)) - , fOID(0){ - // pthread_mutex_init(&mutex, NULL); - }; + , fOID(0) +{ + // pthread_mutex_init(&mutex, NULL); +} template DataList::DataList(const DataList& dl) @@ -174,12 +175,13 @@ DataList::DataList(const DataList& dl) consumersFinished = dl.consumersFinished; fElemDiskFirstSize = dl.fElemDiskFirstSize; fElemDiskSecondSize = dl.fElemDiskSecondSize; -}; +} template -DataList::~DataList(){ - // pthread_mutex_destroy(&mutex); -}; +DataList::~DataList() +{ + // pthread_mutex_destroy(&mutex); +} template DataList& DataList::operator=(const DataList& dl) @@ -189,25 +191,25 @@ DataList& DataList::operator=(const DataList& d consumersFinished = dl.consumersFinished; fElemDiskFirstSize = dl.fElemDiskFirstSize; fElemDiskSecondSize = dl.fElemDiskSecondSize; -}; +} template void DataList::endOfInput() { noMoreInput = true; -}; +} template void DataList::lock() { mutex.lock(); // pthread_mutex_lock(&mutex); -}; +} template void DataList::unlock() { mutex.unlock(); // pthread_mutex_unlock(&mutex); -}; +} template void DataList::setDiskElemSize(uint32_t size1st, uint32_t size2nd) diff --git a/dbcon/joblist/datalistimpl.h b/dbcon/joblist/datalistimpl.h index eb07ce64f..b408104d9 100644 --- a/dbcon/joblist/datalistimpl.h +++ b/dbcon/joblist/datalistimpl.h @@ -44,17 +44,17 @@ template class DataListImpl : public DataList { public: - DataListImpl(uint32_t numConsumers); + explicit DataListImpl(uint32_t numConsumers); DataListImpl(const DataListImpl& dl); - virtual ~DataListImpl(); + ~DataListImpl() override; DataListImpl& operator=(const DataListImpl& dl); // derived classes need to lock around these fcns - virtual void insert(const element_t& e); - virtual void insert(const std::vector& v); - virtual uint64_t getIterator(); - virtual bool next(uint64_t it, element_t* e); + void insert(const element_t& e) override; + void insert(const std::vector& v) override; + uint64_t getIterator() override; + bool next(uint64_t it, element_t* e) override; virtual void setNumConsumers(uint32_t); virtual uint32_t getNumConsumers() const; @@ -141,20 +141,24 @@ DataListImpl::DataListImpl(const DataListImpl DataListImpl::~DataListImpl() { delete c; delete[] cIterators; -}; +} // lock at a higher level template DataListImpl& DataListImpl::operator=( const DataListImpl& dl) { + if (&dl == this) + { + return *this; + } uint64_t i; static_cast >(*this) = static_cast >(dl); @@ -170,7 +174,7 @@ DataListImpl& DataListImpl::oper cIterators[i] = dl.cIterators[i]; return *this; -}; +} template uint64_t DataListImpl::getIterator() @@ -178,8 +182,8 @@ uint64_t DataListImpl::getIterator() if (itIndex >= numConsumers) { std::ostringstream oss; - oss << "DataListImpl::getIterator(): caller attempted to grab too many iterators: " - << "have " << numConsumers << " asked for " << (itIndex + 1); + oss << "DataListImpl::getIterator(): caller attempted to grab too many iterators: " << "have " + << numConsumers << " asked for " << (itIndex + 1); throw std::logic_error(oss.str().c_str()); } @@ -198,7 +202,7 @@ inline void DataListImpl::insert(const std::vector)) { - std::vector* vc = (std::vector*)c; + auto* vc = (std::vector*)c; vc->insert(vc->end(), v.begin(), v.end()); } else @@ -245,7 +249,7 @@ void DataListImpl::eraseUpTo(uint64_t id) #endif c->erase(c->begin(), cIterators[id]); -}; +} template void DataListImpl::reset() diff --git a/dbcon/joblist/dictstep-jl.h b/dbcon/joblist/dictstep-jl.h index 5fe0c618e..195048039 100644 --- a/dbcon/joblist/dictstep-jl.h +++ b/dbcon/joblist/dictstep-jl.h @@ -40,26 +40,27 @@ class DictStepJL : public CommandJL { public: DictStepJL(); - DictStepJL(const pDictionaryStep&); - virtual ~DictStepJL(); + explicit DictStepJL(const pDictionaryStep&); + ~DictStepJL() override; - void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For - // ColumnCommandJL it's a RID, for a DictStep it's a token. - uint8_t getTableColumnType(); - std::string toString(); + void setLBID(uint64_t data, + uint32_t dbroot) override; // converts a rid or dictionary token to an LBID. For + // ColumnCommandJL it's a RID, for a DictStep it's a token. + uint8_t getTableColumnType() override; + std::string toString() override; /* XXXPAT: The width is only valid for projection steps and the output type is TUPLE at the moment. */ void setWidth(uint16_t); - uint16_t getWidth(); + uint16_t getWidth() override; - CommandType getCommandType() + CommandType getCommandType() override { return DICT_STEP; } - void createCommand(messageqcpp::ByteStream&) const; - void runCommand(messageqcpp::ByteStream&) const; + void createCommand(messageqcpp::ByteStream&) const override; + void runCommand(messageqcpp::ByteStream&) const override; messageqcpp::ByteStream getFilterString() const { diff --git a/dbcon/joblist/diskjoinstep.h b/dbcon/joblist/diskjoinstep.h index 1756fb46d..76e760fd3 100644 --- a/dbcon/joblist/diskjoinstep.h +++ b/dbcon/joblist/diskjoinstep.h @@ -33,11 +33,11 @@ class DiskJoinStep : public JobStep public: DiskJoinStep(); DiskJoinStep(TupleHashJoinStep*, int djsIndex, int joinerIndex, bool lastOne); - virtual ~DiskJoinStep(); + ~DiskJoinStep() override; - void run(); - void join(); - const std::string toString() const; + void run() override; + void join() override; + const std::string toString() const override; void loadExistingData(std::vector& data); uint32_t getIterationCount() @@ -73,7 +73,7 @@ class DiskJoinStep : public JobStep void mainRunner(); struct Runner { - Runner(DiskJoinStep* d) : djs(d) + explicit Runner(DiskJoinStep* d) : djs(d) { } void operator()() diff --git a/dbcon/joblist/distributedenginecomm.h b/dbcon/joblist/distributedenginecomm.h index d5c687b87..2c6c6ac86 100644 --- a/dbcon/joblist/distributedenginecomm.h +++ b/dbcon/joblist/distributedenginecomm.h @@ -77,7 +77,7 @@ constexpr uint32_t defaultLocalConnectionId() class DECEventListener { public: - virtual ~DECEventListener(){}; + virtual ~DECEventListener() = default; /* Do whatever needs to be done to init the new PM */ virtual void newPMOnline(uint32_t newConnectionNumber) = 0; @@ -140,7 +140,7 @@ class DistributedEngineComm /** reads queuesize/divisor msgs */ EXPORT void read_some(uint32_t key, uint32_t divisor, std::vector& v, - bool* flowControlOn = NULL); + bool* flowControlOn = nullptr); /** @brief Write a primitive message * diff --git a/dbcon/joblist/elementtype.h b/dbcon/joblist/elementtype.h index a281fe68b..2033728df 100644 --- a/dbcon/joblist/elementtype.h +++ b/dbcon/joblist/elementtype.h @@ -147,9 +147,7 @@ struct RowWrapper et[i] = rg.et[i]; } - ~RowWrapper() - { - } + ~RowWrapper() = default; inline RowWrapper& operator=(const RowWrapper& rg) { @@ -170,7 +168,7 @@ struct RIDElementType uint64_t first; RIDElementType(); - RIDElementType(uint64_t f); + explicit RIDElementType(uint64_t f); const char* getHashString(uint64_t mode, uint64_t* len) const { @@ -194,9 +192,7 @@ struct TupleType { uint64_t first; char* second; - TupleType() - { - } + TupleType() = default; TupleType(uint64_t f, char* s) : first(f), second(s) { } @@ -244,14 +240,14 @@ extern std::ostream& operator<<(std::ostream& out, const TupleType& rhs); #ifndef NO_DATALISTS -//#include "bandeddl.h" -//#include "wsdl.h" +// #include "bandeddl.h" +// #include "wsdl.h" #include "fifo.h" -//#include "bucketdl.h" -//#include "constantdatalist.h" -//#include "swsdl.h" -//#include "zdl.h" -//#include "deliverywsdl.h" +// #include "bucketdl.h" +// #include "constantdatalist.h" +// #include "swsdl.h" +// #include "zdl.h" +// #include "deliverywsdl.h" namespace joblist { @@ -350,7 +346,6 @@ class AnyDataList ~AnyDataList() = default; - inline void rowGroupDL(boost::shared_ptr dl) { fDatalist = dl; @@ -368,8 +363,6 @@ class AnyDataList return fDatalist.get(); } - - enum DataListTypes { UNKNOWN_DATALIST, /*!< 0 Unknown DataList */ @@ -408,8 +401,8 @@ class AnyDataList // bool operator==(const AnyDataList& rhs); private: - AnyDataList(const AnyDataList& rhs); - AnyDataList& operator=(const AnyDataList& rhs); + AnyDataList(const AnyDataList& rhs) = delete; + AnyDataList& operator=(const AnyDataList& rhs) = delete; boost::shared_ptr fDatalist; bool fDisown; }; @@ -435,4 +428,3 @@ extern std::ostream& omitOidInDL(std::ostream& strm); } // namespace joblist #endif - diff --git a/dbcon/joblist/errorinfo.h b/dbcon/joblist/errorinfo.h index 62b404daa..35699a572 100644 --- a/dbcon/joblist/errorinfo.h +++ b/dbcon/joblist/errorinfo.h @@ -38,7 +38,7 @@ struct ErrorInfo uint32_t errCode; std::string errMsg; // for backward compat - ErrorInfo(uint16_t v) : errCode(v) + explicit ErrorInfo(uint16_t v) : errCode(v) { } ErrorInfo& operator=(uint16_t v) @@ -51,4 +51,3 @@ struct ErrorInfo typedef boost::shared_ptr SErrorInfo; } // namespace joblist - diff --git a/dbcon/joblist/expressionstep.h b/dbcon/joblist/expressionstep.h index 8cc223cb1..7e7778ec2 100644 --- a/dbcon/joblist/expressionstep.h +++ b/dbcon/joblist/expressionstep.h @@ -23,7 +23,7 @@ #pragma once -//#define NDEBUG +// #define NDEBUG #include "jobstep.h" #include "filter.h" @@ -48,35 +48,35 @@ class ExpressionStep : public JobStep public: // constructors ExpressionStep(); - ExpressionStep(const JobInfo&); + explicit ExpressionStep(const JobInfo&); // destructor constructors - virtual ~ExpressionStep(); + ~ExpressionStep() override; // inherited methods - void run(); - void join(); - const std::string toString() const; + void run() override; + void join() override; + const std::string toString() const override; - execplan::CalpontSystemCatalog::OID oid() const + execplan::CalpontSystemCatalog::OID oid() const override { return 0; } - execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOids.empty() ? 0 : fTableOids.front(); } using JobStep::alias; - std::string alias() const + std::string alias() const override { return fAliases.empty() ? "" : fAliases.front(); } using JobStep::view; - std::string view() const + std::string view() const override { return fViews.empty() ? "" : fViews.front(); } using JobStep::schema; - std::string schema() const + std::string schema() const override { return fSchemas.empty() ? "" : fSchemas.front(); } diff --git a/dbcon/joblist/fifo.h b/dbcon/joblist/fifo.h index 46ebf9922..8d9e0a0dd 100644 --- a/dbcon/joblist/fifo.h +++ b/dbcon/joblist/fifo.h @@ -45,7 +45,7 @@ template class FIFO : public DataListImpl, element_t> { private: - typedef DataListImpl, element_t> base; + using base = DataListImpl, element_t>; public: enum ElementMode @@ -55,15 +55,15 @@ class FIFO : public DataListImpl, element_t> }; FIFO(uint32_t numConsumers, uint32_t maxElements); - virtual ~FIFO(); + ~FIFO() override; /* DataList interface */ - inline void insert(const element_t& e); - inline void insert(const std::vector& v); - inline bool next(uint64_t it, element_t* e); - uint64_t getIterator(); - void endOfInput(); - void setMultipleProducers(bool b); + inline void insert(const element_t& e) override; + inline void insert(const std::vector& v) override; + inline bool next(uint64_t it, element_t* e) override; + uint64_t getIterator() override; + void endOfInput() override; + void setMultipleProducers(bool b) override; /* Use this insert() to detect when insertion fills up buffer. */ /* When this happens, call waitTillReadyForInserts() before resuming*/ @@ -72,16 +72,16 @@ class FIFO : public DataListImpl, element_t> inline void waitTillReadyForInserts(); inline bool isOutputBlocked() const; - void OID(execplan::CalpontSystemCatalog::OID oid) + void OID(execplan::CalpontSystemCatalog::OID oid) override { base::OID(oid); } - execplan::CalpontSystemCatalog::OID OID() const + execplan::CalpontSystemCatalog::OID OID() const override { return base::OID(); } - inline void dropToken(){}; + inline void dropToken() {}; inline void dropToken(uint32_t){}; // Counters that reflect how many many times this FIFO blocked on reads/writes @@ -89,7 +89,7 @@ class FIFO : public DataListImpl, element_t> uint64_t blockedReadCount() const; // @bug 653 set number of consumers when it is empty. - void setNumConsumers(uint32_t nc); + void setNumConsumers(uint32_t nc) override; void inOrder(bool order) { @@ -109,7 +109,7 @@ class FIFO : public DataListImpl, element_t> { fTotSize = totSize; } - uint64_t totalSize() + uint64_t totalSize() override { return fTotSize; } @@ -495,14 +495,11 @@ void FIFO::maxElements(uint64_t max) { fMaxElements = max; - if (pBuffer) - delete[] pBuffer; + delete[] pBuffer; + delete[] cBuffer; - if (cBuffer) - delete[] cBuffer; - - pBuffer = 0; - cBuffer = 0; + pBuffer = nullptr; + cBuffer = nullptr; for (uint64_t i = 0; i < base::numConsumers; ++i) cpos[i] = fMaxElements; @@ -528,4 +525,3 @@ void FIFO::totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) co } } // namespace joblist - diff --git a/dbcon/joblist/filtercommand-jl.h b/dbcon/joblist/filtercommand-jl.h index 7526e1d1a..2bbe3eccd 100644 --- a/dbcon/joblist/filtercommand-jl.h +++ b/dbcon/joblist/filtercommand-jl.h @@ -34,16 +34,16 @@ namespace joblist class FilterCommandJL : public CommandJL { public: - FilterCommandJL(const FilterStep&); - virtual ~FilterCommandJL(); + explicit FilterCommandJL(const FilterStep&); + ~FilterCommandJL() override; - void setLBID(uint64_t rid, uint32_t dbroot); - uint8_t getTableColumnType(); - CommandType getCommandType(); - std::string toString(); - void createCommand(messageqcpp::ByteStream& bs) const; - void runCommand(messageqcpp::ByteStream& bs) const; - uint16_t getWidth(); + void setLBID(uint64_t rid, uint32_t dbroot) override; + uint8_t getTableColumnType() override; + CommandType getCommandType() override; + std::string toString() override; + void createCommand(messageqcpp::ByteStream& bs) const override; + void runCommand(messageqcpp::ByteStream& bs) const override; + uint16_t getWidth() override; uint8_t getBOP() const { return fBOP; diff --git a/dbcon/joblist/groupconcat.h b/dbcon/joblist/groupconcat.h index 9f9ba9e0e..705b691ef 100644 --- a/dbcon/joblist/groupconcat.h +++ b/dbcon/joblist/groupconcat.h @@ -26,7 +26,6 @@ #include #include - #include "returnedcolumn.h" // SRCP #include "rowgroup.h" // RowGroup #include "rowaggregation.h" // SP_GroupConcat @@ -48,7 +47,7 @@ class GroupConcatInfo virtual ~GroupConcatInfo(); void prepGroupConcat(JobInfo&); - void mapColumns(const rowgroup::RowGroup&); + virtual void mapColumns(const rowgroup::RowGroup&); std::set& columns() { @@ -59,11 +58,11 @@ class GroupConcatInfo return fGroupConcat; } - const std::string toString() const; + virtual const std::string toString() const; protected: - uint32_t getColumnKey(const execplan::SRCP& srcp, JobInfo& jobInfo); - std::shared_ptr makeMapping(const rowgroup::RowGroup&, const rowgroup::RowGroup&); + virtual uint32_t getColumnKey(const execplan::SRCP& srcp, JobInfo& jobInfo); + virtual std::shared_ptr makeMapping(const rowgroup::RowGroup&, const rowgroup::RowGroup&); std::set fColumns; std::vector fGroupConcat; @@ -72,22 +71,22 @@ class GroupConcatInfo class GroupConcatAgUM : public rowgroup::GroupConcatAg { public: - EXPORT GroupConcatAgUM(rowgroup::SP_GroupConcat&); - EXPORT ~GroupConcatAgUM(); + EXPORT explicit GroupConcatAgUM(rowgroup::SP_GroupConcat&); + EXPORT ~GroupConcatAgUM() override; using rowgroup::GroupConcatAg::merge; - void initialize(); - void processRow(const rowgroup::Row&); - EXPORT void merge(const rowgroup::Row&, int64_t); + void initialize() override; + void processRow(const rowgroup::Row&) override; + EXPORT virtual void merge(const rowgroup::Row&, int64_t); boost::scoped_ptr& concator() { return fConcator; } - EXPORT uint8_t* getResult(); + EXPORT uint8_t* getResult() override; protected: - void applyMapping(const std::shared_ptr&, const rowgroup::Row&); + virtual void applyMapping(const std::shared_ptr&, const rowgroup::Row&); boost::scoped_ptr fConcator; boost::scoped_array fData; @@ -133,17 +132,17 @@ class GroupConcatNoOrder : public GroupConcator { public: GroupConcatNoOrder(); - virtual ~GroupConcatNoOrder(); + ~GroupConcatNoOrder() override; - void initialize(const rowgroup::SP_GroupConcat&); - void processRow(const rowgroup::Row&); + void initialize(const rowgroup::SP_GroupConcat&) override; + void processRow(const rowgroup::Row&) override; - void merge(GroupConcator*); + void merge(GroupConcator*) override; using GroupConcator::getResult; - uint8_t* getResultImpl(const std::string& sep); - //uint8_t* getResult(const std::string& sep); + uint8_t* getResultImpl(const std::string& sep) override; + // uint8_t* getResult(const std::string& sep); - const std::string toString() const; + const std::string toString() const override; protected: rowgroup::RowGroup fRowGroup; @@ -163,19 +162,19 @@ class GroupConcatOrderBy : public GroupConcator, public ordering::IdbOrderBy { public: GroupConcatOrderBy(); - virtual ~GroupConcatOrderBy(); + ~GroupConcatOrderBy() override; using ordering::IdbOrderBy::initialize; - void initialize(const rowgroup::SP_GroupConcat&); - void processRow(const rowgroup::Row&); - uint64_t getKeyLength() const; + void initialize(const rowgroup::SP_GroupConcat&) override; + void processRow(const rowgroup::Row&) override; + uint64_t getKeyLength() const override; - void merge(GroupConcator*); + void merge(GroupConcator*) override; using GroupConcator::getResult; - uint8_t* getResultImpl(const std::string& sep); - //uint8_t* getResult(const std::string& sep); + uint8_t* getResultImpl(const std::string& sep) override; + // uint8_t* getResult(const std::string& sep); - const std::string toString() const; + const std::string toString() const override; protected: }; diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index a0c03548a..befba9a94 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -73,15 +73,14 @@ const int32_t CNX_EXP_TABLE_ID = 999; struct TupleInfo { - TupleInfo(uint32_t w = 0, uint32_t o = 0, uint32_t k = -1, uint32_t t = -1, uint32_t s = 0, uint32_t p = 0, - execplan::CalpontSystemCatalog::ColDataType dt = execplan::CalpontSystemCatalog::BIT, - uint32_t csn = 8) + explicit TupleInfo(uint32_t w = 0, uint32_t o = 0, uint32_t k = -1, uint32_t t = -1, uint32_t s = 0, + uint32_t p = 0, + execplan::CalpontSystemCatalog::ColDataType dt = execplan::CalpontSystemCatalog::BIT, + uint32_t csn = 8) : width(w), oid(o), key(k), tkey(t), scale(s), precision(p), dtype(dt), csNum(csn) { } - ~TupleInfo() - { - } + ~TupleInfo() = default; uint32_t width; uint32_t oid; @@ -146,7 +145,7 @@ struct UniqId : fId(i), fTable(t), fSchema(s), fView(v), fPseudo(pi), fSubId(l) { } - UniqId(const execplan::SimpleColumn* sc); + explicit UniqId(const execplan::SimpleColumn* sc); UniqId(int o, const execplan::SimpleColumn* sc); std::string toString() const; @@ -193,7 +192,7 @@ struct TupleKeyInfo //------------------------------------------------------------------------------ struct JobInfo { - JobInfo(ResourceManager* r) + explicit JobInfo(ResourceManager* r) : rm(r) , sessionId(0) , txnId(0) @@ -204,7 +203,7 @@ struct JobInfo , fifoSize(rm->getJlFifoSize()) , logger(new Logger()) , traceFlags(0) - , projectingTableOID(0) + , projectingTableOID(nullptr) , isExeMgr(false) , trace(false) , tryTuples(false) @@ -219,7 +218,7 @@ struct JobInfo , subLevel(0) , subNum(0) , subId(0) - , pJobInfo(NULL) + , pJobInfo(nullptr) , constantFalse(false) , cntStarPos(-1) , stringScanThreshold(1) diff --git a/dbcon/joblist/joblist.h b/dbcon/joblist/joblist.h index e20b34a4f..52b935a69 100644 --- a/dbcon/joblist/joblist.h +++ b/dbcon/joblist/joblist.h @@ -217,10 +217,10 @@ class JobList class TupleJobList : public JobList { public: - TupleJobList(bool isEM = false); - virtual ~TupleJobList(); + explicit TupleJobList(bool isEM = false); + ~TupleJobList() override; - EXPORT uint32_t projectTable(execplan::CalpontSystemCatalog::OID, messageqcpp::ByteStream&); + EXPORT uint32_t projectTable(execplan::CalpontSystemCatalog::OID, messageqcpp::ByteStream&) override; EXPORT const rowgroup::RowGroup& getOutputRowGroup() const; TupleDeliveryStep* getDeliveryStep() { @@ -231,7 +231,7 @@ class TupleJobList : public JobList return fQuery; } void setDeliveryFlag(bool f); - void abort(); + void abort() override; /** Does some light validation on the final joblist * @@ -239,7 +239,7 @@ class TupleJobList : public JobList * there's one and only one projection step, and that its fake table OID is 100. * @note The fake OID check is disabled atm because it's not always 100 although it's supposed to be. */ - EXPORT void validate() const; + EXPORT void validate() const override; private: // defaults okay @@ -255,4 +255,3 @@ typedef boost::shared_ptr STJLP; } // namespace joblist #undef EXPORT - diff --git a/dbcon/joblist/jobstep.h b/dbcon/joblist/jobstep.h index 3796524b5..48e1c8946 100644 --- a/dbcon/joblist/jobstep.h +++ b/dbcon/joblist/jobstep.h @@ -60,12 +60,8 @@ namespace joblist class JobStepAssociation { public: - JobStepAssociation() - { - } - virtual ~JobStepAssociation() - { - } + JobStepAssociation() = default; + virtual ~JobStepAssociation() = default; void inAdd(const AnyDataListSPtr& spdl) __attribute__((deprecated)) { @@ -127,10 +123,8 @@ class JobStep public: /** constructor */ - JobStep() - { - } - JobStep(const JobInfo&); + JobStep() = default; + explicit JobStep(const JobInfo&); /** destructor */ virtual ~JobStep() @@ -508,12 +502,8 @@ class JobStep class TupleJobStep { public: - TupleJobStep() - { - } - virtual ~TupleJobStep() - { - } + TupleJobStep() = default; + virtual ~TupleJobStep() = default; virtual void setOutputRowGroup(const rowgroup::RowGroup&) = 0; virtual void setFcnExpGroup3(const std::vector&) { @@ -527,9 +517,7 @@ class TupleJobStep class TupleDeliveryStep : public TupleJobStep { public: - virtual ~TupleDeliveryStep() - { - } + ~TupleDeliveryStep() override = default; virtual uint32_t nextBand(messageqcpp::ByteStream& bs) = 0; virtual const rowgroup::RowGroup& getDeliveredRowGroup() const = 0; virtual void deliverStringTableRowGroup(bool b) = 0; @@ -541,17 +529,17 @@ class NullStep : public JobStep public: /** @brief virtual void Run method */ - virtual void run() + void run() override { } /** @brief virtual void join method */ - virtual void join() + void join() override { } /** @brief virtual string toString method */ - virtual const std::string toString() const + const std::string toString() const override { return "NullStep"; } @@ -566,4 +554,3 @@ typedef boost::shared_ptr JobStepAssociationSPtr; typedef boost::shared_ptr SJSTEP; } // namespace joblist - diff --git a/dbcon/joblist/jsonarrayagg.h b/dbcon/joblist/jsonarrayagg.h index 77319c0b9..72ca5361d 100644 --- a/dbcon/joblist/jsonarrayagg.h +++ b/dbcon/joblist/jsonarrayagg.h @@ -24,7 +24,6 @@ #include #include - #include "groupconcat.h" #define EXPORT @@ -39,31 +38,31 @@ class JsonArrayInfo : public GroupConcatInfo { public: void prepJsonArray(JobInfo&); - void mapColumns(const rowgroup::RowGroup&); + void mapColumns(const rowgroup::RowGroup&) override; - const std::string toString() const; + const std::string toString() const override; protected: - uint32_t getColumnKey(const execplan::SRCP& srcp, JobInfo& jobInfo); - std::shared_ptr makeMapping(const rowgroup::RowGroup&, const rowgroup::RowGroup&); + uint32_t getColumnKey(const execplan::SRCP& srcp, JobInfo& jobInfo) override; + std::shared_ptr makeMapping(const rowgroup::RowGroup&, const rowgroup::RowGroup&) override; }; class JsonArrayAggregatAgUM : public GroupConcatAgUM { public: - EXPORT JsonArrayAggregatAgUM(rowgroup::SP_GroupConcat&); - EXPORT ~JsonArrayAggregatAgUM(); + EXPORT explicit JsonArrayAggregatAgUM(rowgroup::SP_GroupConcat&); + EXPORT ~JsonArrayAggregatAgUM() override; using rowgroup::GroupConcatAg::merge; - void initialize(); - void processRow(const rowgroup::Row&); - EXPORT void merge(const rowgroup::Row&, int64_t); + void initialize() override; + void processRow(const rowgroup::Row&) override; + EXPORT void merge(const rowgroup::Row&, int64_t) override; EXPORT void getResult(uint8_t*); - EXPORT uint8_t* getResult(); + EXPORT uint8_t* getResult() override; protected: - void applyMapping(const std::shared_ptr&, const rowgroup::Row&); + void applyMapping(const std::shared_ptr&, const rowgroup::Row&) override; }; // JSON_ARRAYAGG base @@ -71,17 +70,17 @@ class JsonArrayAggregator : public GroupConcator { public: JsonArrayAggregator(); - virtual ~JsonArrayAggregator(); + ~JsonArrayAggregator() override; - virtual void initialize(const rowgroup::SP_GroupConcat&); - virtual void processRow(const rowgroup::Row&) = 0; + void initialize(const rowgroup::SP_GroupConcat&) override; + void processRow(const rowgroup::Row&) override = 0; - virtual const std::string toString() const; + const std::string toString() const override; protected: - virtual bool concatColIsNull(const rowgroup::Row&); - virtual void outputRow(std::ostringstream&, const rowgroup::Row&); - virtual int64_t lengthEstimate(const rowgroup::Row&); + bool concatColIsNull(const rowgroup::Row&) override; + void outputRow(std::ostringstream&, const rowgroup::Row&) override; + int64_t lengthEstimate(const rowgroup::Row&) override; }; // For JSON_ARRAYAGG withour distinct or orderby @@ -89,17 +88,17 @@ class JsonArrayAggNoOrder : public JsonArrayAggregator { public: JsonArrayAggNoOrder(); - virtual ~JsonArrayAggNoOrder(); + ~JsonArrayAggNoOrder() override; - void initialize(const rowgroup::SP_GroupConcat&); - void processRow(const rowgroup::Row&); + void initialize(const rowgroup::SP_GroupConcat&) override; + void processRow(const rowgroup::Row&) override; using GroupConcator::merge; - void merge(GroupConcator*); + void merge(GroupConcator*) override; using GroupConcator::getResult; - uint8_t* getResultImpl(const std::string& sep); + uint8_t* getResultImpl(const std::string& sep) override; - const std::string toString() const; + const std::string toString() const override; protected: rowgroup::RowGroup fRowGroup; @@ -118,19 +117,19 @@ class JsonArrayAggOrderBy : public JsonArrayAggregator, public ordering::IdbOrde { public: JsonArrayAggOrderBy(); - virtual ~JsonArrayAggOrderBy(); + ~JsonArrayAggOrderBy() override; using ordering::IdbOrderBy::initialize; - void initialize(const rowgroup::SP_GroupConcat&); - void processRow(const rowgroup::Row&); - uint64_t getKeyLength() const; + void initialize(const rowgroup::SP_GroupConcat&) override; + void processRow(const rowgroup::Row&) override; + uint64_t getKeyLength() const override; using GroupConcator::merge; - void merge(GroupConcator*); + void merge(GroupConcator*) override; using GroupConcator::getResult; - uint8_t* getResultImpl(const std::string& sep); + uint8_t* getResultImpl(const std::string& sep) override; - const std::string toString() const; + const std::string toString() const override; protected: }; diff --git a/dbcon/joblist/largedatalist.h b/dbcon/joblist/largedatalist.h index cd794a832..9ab607520 100644 --- a/dbcon/joblist/largedatalist.h +++ b/dbcon/joblist/largedatalist.h @@ -89,7 +89,7 @@ struct DiskIoInfo bool fWrite; // c: byte count; b: is write operation? - DiskIoInfo(bool b = false) : fBytes(0), fWrite(b) + explicit DiskIoInfo(bool b = false) : fBytes(0), fWrite(b) { } }; @@ -205,7 +205,7 @@ LargeDataList::LargeDataList(uint32_t nc, uint32_t eleme , fTraceOn(false) , fReUse(false) , fSaveForReuse(false) - , fRestoreInfo(NULL) + , fRestoreInfo(nullptr) { loadedSet = 0; setCount = 1; diff --git a/dbcon/joblist/largehashjoin.h b/dbcon/joblist/largehashjoin.h index a61faf879..dd1a96280 100644 --- a/dbcon/joblist/largehashjoin.h +++ b/dbcon/joblist/largehashjoin.h @@ -256,9 +256,7 @@ class HashJoin }; template -HashJoin::HashJoin() -{ -} +HashJoin::HashJoin() = default; template HashJoin::HashJoin(joblist::BDLWrapper& set1, joblist::BDLWrapper& set2, diff --git a/dbcon/joblist/limitedorderby.h b/dbcon/joblist/limitedorderby.h index 40b1ac005..ff0daafd2 100644 --- a/dbcon/joblist/limitedorderby.h +++ b/dbcon/joblist/limitedorderby.h @@ -38,17 +38,17 @@ class LimitedOrderBy : public ordering::IdbOrderBy { public: LimitedOrderBy(); - virtual ~LimitedOrderBy(); + ~LimitedOrderBy() override; using ordering::IdbOrderBy::initialize; void initialize(const rowgroup::RowGroup&, const JobInfo&, bool invertRules = false, bool isMultiThreded = false); - void processRow(const rowgroup::Row&); - uint64_t getKeyLength() const; + void processRow(const rowgroup::Row&) override; + uint64_t getKeyLength() const override; uint64_t getLimitCount() const { return fCount; } - const std::string toString() const; + const std::string toString() const override; void finalize(); diff --git a/dbcon/joblist/passthrucommand-jl.h b/dbcon/joblist/passthrucommand-jl.h index 965c51bab..ea7c93bb3 100644 --- a/dbcon/joblist/passthrucommand-jl.h +++ b/dbcon/joblist/passthrucommand-jl.h @@ -41,18 +41,19 @@ class PassThruStep; class PassThruCommandJL : public CommandJL { public: - PassThruCommandJL(const PassThruStep&); - virtual ~PassThruCommandJL(); + explicit PassThruCommandJL(const PassThruStep&); + ~PassThruCommandJL() override; - void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For - // ColumnCommandJL it's a RID, for a DictStep it's a token. - uint8_t getTableColumnType(); - std::string toString(); + void setLBID(uint64_t data, + uint32_t dbroot) override; // converts a rid or dictionary token to an LBID. For + // ColumnCommandJL it's a RID, for a DictStep it's a token. + uint8_t getTableColumnType() override; + std::string toString() override; - void createCommand(messageqcpp::ByteStream&) const; - void runCommand(messageqcpp::ByteStream&) const; - uint16_t getWidth(); - CommandType getCommandType() + void createCommand(messageqcpp::ByteStream&) const override; + void runCommand(messageqcpp::ByteStream&) const override; + uint16_t getWidth() override; + CommandType getCommandType() override { return PASS_THRU; } diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index f06a7e99c..0d3cdaccc 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -35,7 +35,6 @@ #pragma pack(push, 1) - // from blocksize.h const int32_t DATA_BLOCK_SIZE = BLOCK_SIZE; @@ -65,7 +64,7 @@ using utils::ConstString; class StringComparator : public datatypes::Charset { public: - StringComparator(const Charset& cs) : Charset(cs) + explicit StringComparator(const Charset& cs) : Charset(cs) { } bool op(int* error, uint8_t COP, const ConstString& str1, const ConstString& str2) const @@ -74,7 +73,8 @@ class StringComparator : public datatypes::Charset return like(COP & COMPARE_NOT, str1, str2); if (COP == COMPARE_NULLEQ) - return str1.isNull() == str2.isNull(); // XXX: TODO: I do not know the logic here, so it is temporary solution. + return str1.isNull() == + str2.isNull(); // XXX: TODO: I do not know the logic here, so it is temporary solution. int cmp = strnncollsp(str1, str2); @@ -335,7 +335,7 @@ struct ColRequestHeaderDataType : public datatypes::Charset ColRequestHeaderDataType() : Charset(my_charset_bin), CompType(0), DataSize(0), DataType(0) { } - ColRequestHeaderDataType(const execplan::CalpontSystemCatalog::ColType& rhs) + explicit ColRequestHeaderDataType(const execplan::CalpontSystemCatalog::ColType& rhs) : Charset(rhs.charsetNumber) , CompType(rhs.compressionType) , DataSize(rhs.colWidth) @@ -633,7 +633,7 @@ struct TokenByScanResultHeader uint16_t Pad1; uint32_t CacheIO; // I/O count from buffer cache uint32_t PhysicalIO; // Physical I/O count from disk -}; // what follows is NVALS Tokens or DataValues. +}; // what follows is NVALS Tokens or DataValues. // DICT_SIGNATURE @@ -889,5 +889,4 @@ struct LbidAtVer uint32_t Ver; }; - #pragma pack(pop) diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index 4b94dd453..008a0e81f 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -102,26 +102,26 @@ class pColStep : public JobStep pColStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - pColStep(const pColScanStep& rhs); + explicit pColStep(const pColScanStep& rhs); - pColStep(const PassThruStep& rhs); + explicit pColStep(const PassThruStep& rhs); - virtual ~pColStep(){}; + ~pColStep() override = default; /** @brief Starts processing. Set at least the RID list before calling. * * Starts processing. Set at least the RID list before calling this. */ - virtual void run() {}; + void run() override {}; /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join() {}; + void join() override {}; - virtual const std::string toString() const; + const std::string toString() const override; - virtual bool isDictCol() const + bool isDictCol() const override { return fIsDict; } @@ -186,12 +186,12 @@ class pColStep : public JobStep return fSwallowRows; } - virtual execplan::CalpontSystemCatalog::OID oid() const + execplan::CalpontSystemCatalog::OID oid() const override { return fOid; } - virtual execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOid; } @@ -222,15 +222,15 @@ class pColStep : public JobStep { isFilterFeeder = filterFeeder; } - virtual uint64_t msgsRcvdCount() const + uint64_t msgsRcvdCount() const override { return msgsRecvd; } - virtual uint64_t msgBytesIn() const + uint64_t msgBytesIn() const override { return fMsgBytesIn; } - virtual uint64_t msgBytesOut() const + uint64_t msgBytesOut() const override { return fMsgBytesOut; } @@ -336,16 +336,14 @@ class pColScanStep : public JobStep pColScanStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - pColScanStep(const pColStep& rhs); - ~pColScanStep() - { - } + explicit pColScanStep(const pColStep& rhs); + ~pColScanStep() override = default; /** @brief Starts processing. * * Starts processing. */ - virtual void run() + void run() override { } @@ -353,11 +351,11 @@ class pColScanStep : public JobStep * * Does nothing. Returns when this instance is finished. */ - virtual void join() + void join() override { } - virtual bool isDictCol() const + bool isDictCol() const override { return fIsDict; }; @@ -413,14 +411,14 @@ class pColScanStep : public JobStep return fFilterCount; } - virtual const std::string toString() const; + const std::string toString() const override; - virtual execplan::CalpontSystemCatalog::OID oid() const + execplan::CalpontSystemCatalog::OID oid() const override { return fOid; } - virtual execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOid; } @@ -433,11 +431,11 @@ class pColScanStep : public JobStep return fRm; } - virtual uint64_t msgBytesIn() const + uint64_t msgBytesIn() const override { return fMsgBytesIn; } - virtual uint64_t msgBytesOut() const + uint64_t msgBytesOut() const override { return fMsgBytesOut; } @@ -479,7 +477,6 @@ class pColScanStep : public JobStep return fFilters; } - private: // defaults okay? // pColScanStep(const pColScanStep& rhs); @@ -545,16 +542,14 @@ class pDictionaryStep : public JobStep pDictionaryStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tabelOid, const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - virtual ~pDictionaryStep() - { - } + ~pDictionaryStep() override = default; /** @brief virtual void Run method */ - virtual void run() + void run() override { } - virtual void join() + void join() override { } // void setOutList(StringDataList* rids); @@ -568,7 +563,7 @@ class pDictionaryStep : public JobStep fBOP = b; } - virtual const std::string toString() const; + const std::string toString() const override; execplan::CalpontSystemCatalog::ColType& colType() { @@ -579,23 +574,23 @@ class pDictionaryStep : public JobStep return fColType; } - virtual execplan::CalpontSystemCatalog::OID oid() const + execplan::CalpontSystemCatalog::OID oid() const override { return fOid; } - virtual execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOid; } - virtual uint64_t msgsRcvdCount() const + uint64_t msgsRcvdCount() const override { return msgsRecvd; } - virtual uint64_t msgBytesIn() const + uint64_t msgBytesIn() const override { return fMsgBytesIn; } - virtual uint64_t msgBytesOut() const + uint64_t msgBytesOut() const override { return fMsgBytesOut; } @@ -677,18 +672,18 @@ class pDictionaryScan : public JobStep pDictionaryScan(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - ~pDictionaryScan(); + ~pDictionaryScan() override; /** @brief virtual void Run method */ - virtual void run(); - virtual void join(); + void run() override; + void join() override; void setInputList(DataList_t* rids); void setBOP(int8_t b); void sendPrimitiveMessages(); void receivePrimitiveMessages(); void setSingleThread(); - virtual const std::string toString() const; + const std::string toString() const override; void setRidList(DataList* rids); @@ -713,32 +708,32 @@ class pDictionaryScan : public JobStep fDec->addQueue(uniqueID); } - virtual execplan::CalpontSystemCatalog::OID oid() const + execplan::CalpontSystemCatalog::OID oid() const override { return fOid; } - virtual execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOid; } - uint64_t phyIOCount() const + uint64_t phyIOCount() const override { return fPhysicalIO; } - uint64_t cacheIOCount() const + uint64_t cacheIOCount() const override { return fCacheIO; } - uint64_t msgsRcvdCount() const + uint64_t msgsRcvdCount() const override { return msgsRecvd; } - uint64_t msgBytesIn() const + uint64_t msgBytesIn() const override { return fMsgBytesIn; } - uint64_t msgBytesOut() const + uint64_t msgBytesOut() const override { return fMsgBytesOut; } @@ -781,7 +776,7 @@ class pDictionaryScan : public JobStep } void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); - virtual void abort(); + void abort() override; const execplan::CalpontSystemCatalog::ColType& colType() const { @@ -859,7 +854,7 @@ class pDictionaryScan : public JobStep class BatchPrimitive : public JobStep, public DECEventListener { public: - BatchPrimitive(const JobInfo& jobInfo) : JobStep(jobInfo) + explicit BatchPrimitive(const JobInfo& jobInfo) : JobStep(jobInfo) { } virtual bool getFeederFlag() const = 0; @@ -922,20 +917,20 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep TupleBPS(const pDictionaryStep& rhs, const JobInfo& jobInfo); TupleBPS(const pDictionaryScan& rhs, const JobInfo& jobInfo); TupleBPS(const PassThruStep& rhs, const JobInfo& jobInfo); - virtual ~TupleBPS(); + ~TupleBPS() override; /** @brief Starts processing. * * Starts processing. */ - virtual void run(); + void run() override; /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(); + void join() override; - void abort(); + void abort() override; void abort_nolock(); /** @brief The main loop for the send-side thread @@ -960,11 +955,11 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep * Add a filter when the column is anything but a 4-byte float type, including * 8-byte doubles. */ - void setBPP(JobStep* jobStep); - void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2); + void setBPP(JobStep* jobStep) override; + void setProjectBPP(JobStep* jobStep1, JobStep* jobStep2) override; bool scanit(uint64_t rid); void storeCasualPartitionInfo(const bool estimateRowCounts); - bool getFeederFlag() const + bool getFeederFlag() const override { return isFilterFeeder; } @@ -972,7 +967,7 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep { isFilterFeeder = filterFeeder; } - void setSwallowRows(const bool swallowRows) + void setSwallowRows(const bool swallowRows) override { fSwallowRows = swallowRows; } @@ -982,25 +977,25 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep } /* Base class interface fcn that can go away */ - void setOutputType(BPSOutputType) + void setOutputType(BPSOutputType) override { } // Can't change the ot of a TupleBPS - BPSOutputType getOutputType() const + BPSOutputType getOutputType() const override { return ROW_GROUP; } - void setBppStep() + void setBppStep() override { } - void setIsProjectionOnly() + void setIsProjectionOnly() override { } - uint64_t getRows() const + uint64_t getRows() const override { return ridsReturned; } - void setFirstStepType(PrimitiveStepType firstStepType) + void setFirstStepType(PrimitiveStepType firstStepType) override { ffirstStepType = firstStepType; } @@ -1008,19 +1003,19 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep { return ffirstStepType; } - void setStepCount() + void setStepCount() override { fStepCount++; } - uint32_t getStepCount() const + uint32_t getStepCount() const override { return fStepCount; } - void setLastTupleId(uint64_t id) + void setLastTupleId(uint64_t id) override { fLastTupleId = id; } - uint64_t getLastTupleId() const + uint64_t getLastTupleId() const override { return fLastTupleId; } @@ -1029,20 +1024,20 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep * * Set the DistributedEngineComm object this instance should use */ - void dec(DistributedEngineComm* dec); + void dec(DistributedEngineComm* dec) override; - virtual void stepId(uint16_t stepId); - virtual uint16_t stepId() const + void stepId(uint16_t stepId) override; + uint16_t stepId() const override { return fStepId; } - virtual const std::string toString() const; + const std::string toString() const override; - virtual execplan::CalpontSystemCatalog::OID oid() const + execplan::CalpontSystemCatalog::OID oid() const override { return fOid; } - virtual execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOid; } @@ -1050,40 +1045,40 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep { return fColType; } - const OIDVector& getProjectOids() const + const OIDVector& getProjectOids() const override { return projectOids; } - virtual uint64_t phyIOCount() const + uint64_t phyIOCount() const override { return fPhysicalIO; } - virtual uint64_t cacheIOCount() const + uint64_t cacheIOCount() const override { return fCacheIO; } - virtual uint64_t msgsRcvdCount() const + uint64_t msgsRcvdCount() const override { return msgsRecvd; } - virtual uint64_t msgBytesIn() const + uint64_t msgBytesIn() const override { return fMsgBytesIn; } - virtual uint64_t msgBytesOut() const + uint64_t msgBytesOut() const override { return fMsgBytesOut; } - virtual uint64_t blockTouched() const + uint64_t blockTouched() const override { return fBlockTouched; } - uint32_t nextBand(messageqcpp::ByteStream& bs); + uint32_t nextBand(messageqcpp::ByteStream& bs) override; //...Currently only supported by pColStep and pColScanStep, so didn't bother //...to define abstract method in base class, but if start adding to other //...classes, then should consider adding pure virtual method to JobStep. - uint64_t blksSkipped() const + uint64_t blksSkipped() const override { return fNumBlksSkipped; } @@ -1094,17 +1089,17 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep } void useJoiner(std::shared_ptr); void useJoiners(const std::vector>&); - bool wasStepRun() const + bool wasStepRun() const override { return fRunExecuted; } // DEC event listener interface - void newPMOnline(uint32_t connectionNumber); + void newPMOnline(uint32_t connectionNumber) override; void setInputRowGroup(const rowgroup::RowGroup& rg); - void setOutputRowGroup(const rowgroup::RowGroup& rg); - const rowgroup::RowGroup& getOutputRowGroup() const; + void setOutputRowGroup(const rowgroup::RowGroup& rg) override; + const rowgroup::RowGroup& getOutputRowGroup() const override; void setAggregateStep(const rowgroup::SP_ROWAGG_PM_t& agg, const rowgroup::RowGroup& rg); @@ -1118,7 +1113,7 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep return bop; } - void setJobInfo(const JobInfo* jobInfo); + void setJobInfo(const JobInfo* jobInfo) override; // @bug 2123. Added getEstimatedRowCount function. /* @brief estimates the number of rows that will be returned for use in determining the @@ -1141,9 +1136,9 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep JLF should register that object with the TBPS for that table. If it's cross-table, then JLF should register it with the join step. */ - void addFcnJoinExp(const std::vector& fe); - void addFcnExpGroup1(const boost::shared_ptr& fe); - void setFE1Input(const rowgroup::RowGroup& feInput); + void addFcnJoinExp(const std::vector& fe) override; + void addFcnExpGroup1(const boost::shared_ptr& fe) override; + void setFE1Input(const rowgroup::RowGroup& feInput) override; /* for use by the THJS only... */ void setFcnExpGroup2(const boost::shared_ptr& fe2, @@ -1152,17 +1147,17 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep /* Functions & Expressions in select and groupby clause. JLF should use these only if there isn't a join. If there is, call the equivalent fcn on THJS instead */ - void setFcnExpGroup3(const std::vector& fe); - void setFE23Output(const rowgroup::RowGroup& rg); + void setFcnExpGroup3(const std::vector& fe) override; + void setFE23Output(const rowgroup::RowGroup& rg) override; bool hasFcnExpGroup3() { return (fe2 != NULL); } // rowgroup to connector - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; + const rowgroup::RowGroup& getDeliveredRowGroup() const override; + void deliverStringTableRowGroup(bool b) override; + bool deliverStringTableRowGroup() const override; /* Interface for adding add'l predicates for casual partitioning. * This fcn checks for any intersection between the values in vals @@ -1185,7 +1180,7 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep void reloadExtentLists(); void initExtentMarkers(); // need a better name for this - virtual bool stringTableFriendly() + bool stringTableFriendly() override { return true; } @@ -1486,16 +1481,16 @@ class FilterStep : public JobStep { public: FilterStep(const execplan::CalpontSystemCatalog::ColType& colType, const JobInfo& jobInfo); - ~FilterStep(); + ~FilterStep() override; /** @brief virtual void Run method */ - void run(); - void join(); + void run() override; + void join() override; - const std::string toString() const; + const std::string toString() const override; - execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOID; } @@ -1526,7 +1521,7 @@ class FilterStep : public JobStep private: // This i/f is not meaningful in this step - execplan::CalpontSystemCatalog::OID oid() const + execplan::CalpontSystemCatalog::OID oid() const override { return 0; } @@ -1562,31 +1557,31 @@ class PassThruStep : public JobStep PassThruStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tableOid, const execplan::CalpontSystemCatalog::ColType& colType, const JobInfo& jobInfo); - PassThruStep(const pColStep& rhs); - PassThruStep(const PseudoColStep& rhs); + explicit PassThruStep(const pColStep& rhs); + explicit PassThruStep(const PseudoColStep& rhs); - virtual ~PassThruStep(); + ~PassThruStep() override; /** @brief Starts processing. Set at least the RID list before calling. * * Starts processing. Set at least the RID list before calling this. */ - virtual void run(); + void run() override; /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(); + void join() override; - virtual const std::string toString() const; + const std::string toString() const override; - virtual execplan::CalpontSystemCatalog::OID oid() const + execplan::CalpontSystemCatalog::OID oid() const override { return fOid; } - virtual execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOid; } @@ -1595,7 +1590,7 @@ class PassThruStep : public JobStep { return colWidth; } - bool isDictCol() const + bool isDictCol() const override { return isDictColumn; } @@ -1663,13 +1658,11 @@ class PseudoColStep : public pColStep { } - PseudoColStep(const PassThruStep& rhs) : pColStep(rhs), fPseudoColumnId(rhs.pseudoType()) + explicit PseudoColStep(const PassThruStep& rhs) : pColStep(rhs), fPseudoColumnId(rhs.pseudoType()) { } - virtual ~PseudoColStep() - { - } + ~PseudoColStep() override = default; uint32_t pseudoColumnId() const { @@ -1686,8 +1679,8 @@ class PseudoColStep : public pColStep private: /** @brief disabled constuctor */ - PseudoColStep(const pColScanStep&); - PseudoColStep(const pColStep&); + explicit PseudoColStep(const pColScanStep&); + explicit PseudoColStep(const pColStep&); }; } // namespace joblist diff --git a/dbcon/joblist/pseudocc-jl.h b/dbcon/joblist/pseudocc-jl.h index efba4f304..04cf13ef7 100644 --- a/dbcon/joblist/pseudocc-jl.h +++ b/dbcon/joblist/pseudocc-jl.h @@ -24,12 +24,12 @@ namespace joblist class PseudoCCJL : public ColumnCommandJL { public: - PseudoCCJL(const PseudoColStep&); - virtual ~PseudoCCJL(); + explicit PseudoCCJL(const PseudoColStep&); + ~PseudoCCJL() override; - virtual void createCommand(messageqcpp::ByteStream&) const; - virtual void runCommand(messageqcpp::ByteStream&) const; - virtual std::string toString(); + void createCommand(messageqcpp::ByteStream&) const override; + void runCommand(messageqcpp::ByteStream&) const override; + std::string toString() override; uint32_t getFunction() const { return function; diff --git a/dbcon/joblist/resourcedistributor.h b/dbcon/joblist/resourcedistributor.h index 8feb4e36b..6a04bdbd1 100644 --- a/dbcon/joblist/resourcedistributor.h +++ b/dbcon/joblist/resourcedistributor.h @@ -58,7 +58,7 @@ extern const unsigned maxSessionsDefault; class LockedSessionMap { public: - LockedSessionMap(uint64_t resource, unsigned maxSessions = maxSessionsDefault) + explicit LockedSessionMap(uint64_t resource, unsigned maxSessions = maxSessionsDefault) : fResourceBlock(resource), fMaxSessions(maxSessions) { } @@ -93,9 +93,7 @@ class ResourceDistributor { } - virtual ~ResourceDistributor() - { - } + virtual ~ResourceDistributor() = default; typedef std::map SessionMap; diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index 733539d3a..0bf5a9c7f 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -100,9 +100,9 @@ const uint64_t defaultRowsPerBatch = 10000; /* HJ CP feedback, see bug #1465 */ const uint32_t defaultHjCPUniqueLimit = 100; -const constexpr uint64_t defaultFlowControlEnableBytesThresh = 50000000; // ~50Mb +const constexpr uint64_t defaultFlowControlEnableBytesThresh = 50000000; // ~50Mb const constexpr uint64_t defaultFlowControlDisableBytesThresh = 10000000; // ~10 MB -const constexpr uint64_t defaultBPPSendThreadBytesThresh = 250000000; // ~250 MB +const constexpr uint64_t defaultBPPSendThreadBytesThresh = 250000000; // ~250 MB const constexpr uint64_t BPPSendThreadMsgThresh = 100; const bool defaultAllowDiskAggregation = false; @@ -117,7 +117,7 @@ class ResourceManager /** @brief ctor * */ - ResourceManager(bool runningInExeMgr = false, config::Config* aConfig = nullptr); + explicit ResourceManager(bool runningInExeMgr = false, config::Config* aConfig = nullptr); static ResourceManager* instance(bool runningInExeMgr = false, config::Config* aConfig = nullptr); config::Config* getConfig() { @@ -126,9 +126,7 @@ class ResourceManager /** @brief dtor */ - virtual ~ResourceManager() - { - } + virtual ~ResourceManager() = default; typedef std::map MemMap; @@ -286,12 +284,14 @@ class ResourceManager uint64_t getDECEnableBytesThresh() const { - return getUintVal(FlowControlStr, "DECFlowControlEnableBytesThresh(", defaultFlowControlEnableBytesThresh); + return getUintVal(FlowControlStr, "DECFlowControlEnableBytesThresh(", + defaultFlowControlEnableBytesThresh); } uint32_t getDECDisableBytesThresh() const { - return getUintVal(FlowControlStr, "DECFlowControlDisableBytesThresh", defaultFlowControlDisableBytesThresh); + return getUintVal(FlowControlStr, "DECFlowControlDisableBytesThresh", + defaultFlowControlDisableBytesThresh); } uint32_t getBPPSendThreadBytesThresh() const diff --git a/dbcon/joblist/rtscommand-jl.h b/dbcon/joblist/rtscommand-jl.h index 3c114fb05..c92fc6926 100644 --- a/dbcon/joblist/rtscommand-jl.h +++ b/dbcon/joblist/rtscommand-jl.h @@ -42,24 +42,25 @@ class RTSCommandJL : public CommandJL public: RTSCommandJL(const pColStep&, const pDictionaryStep&); RTSCommandJL(const PassThruStep&, const pDictionaryStep&); - virtual ~RTSCommandJL(); + ~RTSCommandJL() override; - void setLBID(uint64_t data, uint32_t dbroot); // converts a rid or dictionary token to an LBID. For - // ColumnCommandJL it's a RID, for a DictStep it's a token. - uint8_t getTableColumnType(); - std::string toString(); + void setLBID(uint64_t data, + uint32_t dbroot) override; // converts a rid or dictionary token to an LBID. For + // ColumnCommandJL it's a RID, for a DictStep it's a token. + uint8_t getTableColumnType() override; + std::string toString() override; bool isPassThru() { return (passThru != 0); } - uint16_t getWidth(); - CommandType getCommandType() + uint16_t getWidth() override; + CommandType getCommandType() override { return RID_TO_STRING; } - void createCommand(messageqcpp::ByteStream&) const; - void runCommand(messageqcpp::ByteStream&) const; + void createCommand(messageqcpp::ByteStream&) const override; + void runCommand(messageqcpp::ByteStream&) const override; private: RTSCommandJL(); diff --git a/dbcon/joblist/subquerystep.h b/dbcon/joblist/subquerystep.h index d39572b40..ad81ca360 100644 --- a/dbcon/joblist/subquerystep.h +++ b/dbcon/joblist/subquerystep.h @@ -40,28 +40,28 @@ class SubQueryStep : public JobStep public: /** @brief SubQueryStep constructor */ - SubQueryStep(const JobInfo&); + explicit SubQueryStep(const JobInfo&); /** @brief SubQueryStep destructor */ - ~SubQueryStep(); + ~SubQueryStep() override; /** @brief virtual void run method */ - void run(); + void run() override; /** @brief virtual void join method */ - void join(); + void join() override; /** @brief virtual void abort method */ - void abort(); + void abort() override; /** @brief virtual get table OID * @returns OID */ - execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOid; } @@ -76,7 +76,7 @@ class SubQueryStep : public JobStep /** @brief virtual output info to a string * @returns string */ - const std::string toString() const; + const std::string toString() const override; /** @brief virtual set the output rowgroup */ @@ -128,24 +128,24 @@ class SubAdapterStep : public JobStep, public TupleDeliveryStep /** @brief SubAdapterStep destructor */ - ~SubAdapterStep(); + ~SubAdapterStep() override; /** @brief virtual void run method */ - void run(); + void run() override; /** @brief virtual void join method */ - void join(); + void join() override; /** @brief virtual void abort method */ - void abort(); + void abort() override; /** @brief virtual get table OID * @returns OID */ - execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOid; } @@ -160,16 +160,16 @@ class SubAdapterStep : public JobStep, public TupleDeliveryStep /** @brief virtual output info to a string * @returns string */ - const std::string toString() const; + const std::string toString() const override; /** @brief virtual set the output rowgroup */ - void setOutputRowGroup(const rowgroup::RowGroup& rg); + void setOutputRowGroup(const rowgroup::RowGroup& rg) override; /** @brief virtual get the output rowgroup * @returns RowGroup */ - const rowgroup::RowGroup& getOutputRowGroup() const + const rowgroup::RowGroup& getOutputRowGroup() const override { return fRowGroupOut; } @@ -177,24 +177,24 @@ class SubAdapterStep : public JobStep, public TupleDeliveryStep /** @brief TupleDeliveryStep's pure virtual methods nextBand * @returns row count */ - uint32_t nextBand(messageqcpp::ByteStream& bs); + uint32_t nextBand(messageqcpp::ByteStream& bs) override; /** @brief Delivered Row Group * @returns RowGroup */ - const rowgroup::RowGroup& getDeliveredRowGroup() const + const rowgroup::RowGroup& getDeliveredRowGroup() const override { return fRowGroupDeliver; } /** @brief Turn on/off string table delivery */ - void deliverStringTableRowGroup(bool b); + void deliverStringTableRowGroup(bool b) override; /** @brief Check useStringTable flag on delivered RowGroup * @returns boolean */ - bool deliverStringTableRowGroup() const; + bool deliverStringTableRowGroup() const override; /** @brief set the rowgroup for FE to work on */ @@ -255,7 +255,7 @@ class SubAdapterStep : public JobStep, public TupleDeliveryStep class Runner { public: - Runner(SubAdapterStep* step) : fStep(step) + explicit Runner(SubAdapterStep* step) : fStep(step) { } void operator()() @@ -272,4 +272,3 @@ class SubAdapterStep : public JobStep, public TupleDeliveryStep }; } // namespace joblist - diff --git a/dbcon/joblist/subquerytransformer.h b/dbcon/joblist/subquerytransformer.h index 5f9ea4e86..1d973e81a 100644 --- a/dbcon/joblist/subquerytransformer.h +++ b/dbcon/joblist/subquerytransformer.h @@ -176,15 +176,15 @@ class SimpleScalarTransformer : public SubQueryTransformer /** @brief SimpleScalarTransformer constructor * @param SubQueryTransformer */ - SimpleScalarTransformer(const SubQueryTransformer& rhs); + explicit SimpleScalarTransformer(const SubQueryTransformer& rhs); /** @brief SimpleScalarTransformer destructor */ - virtual ~SimpleScalarTransformer(); + ~SimpleScalarTransformer() override; /** @brief virtual void run method */ - void run(); + void run() override; /** @brief virtual get scalar result * @param jobInfo @@ -222,4 +222,3 @@ class SimpleScalarTransformer : public SubQueryTransformer }; } // namespace joblist - diff --git a/dbcon/joblist/threadsafequeue.h b/dbcon/joblist/threadsafequeue.h index 36958ba14..71d6ca44c 100644 --- a/dbcon/joblist/threadsafequeue.h +++ b/dbcon/joblist/threadsafequeue.h @@ -28,8 +28,6 @@ #include #include - - namespace joblist { struct TSQSize_t @@ -56,7 +54,7 @@ class ThreadSafeQueue * * @warning this class takes ownership of the passed-in pointers. */ - ThreadSafeQueue(boost::mutex* pimplLock = 0, boost::condition* pimplCond = 0) + ThreadSafeQueue(boost::mutex* pimplLock = nullptr, boost::condition* pimplCond = nullptr) : fShutdown(false), bytes(0), zeroCount(0) { fPimplLock.reset(pimplLock); @@ -113,7 +111,7 @@ class ThreadSafeQueue */ T& front() { - if (fPimplLock == 0 || fPimplCond == 0) + if (fPimplLock == nullptr || fPimplCond == nullptr) throw std::runtime_error("TSQ: front(): no sync!"); boost::mutex::scoped_lock lk(*fPimplLock); @@ -140,7 +138,7 @@ class ThreadSafeQueue { TSQSize_t ret = {0, 0}; - if (fPimplLock == 0 || fPimplCond == 0) + if (fPimplLock == nullptr || fPimplCond == nullptr) throw std::runtime_error("TSQ: push(): no sync!"); if (fShutdown) @@ -161,7 +159,7 @@ class ThreadSafeQueue { TSQSize_t ret = {0, 0}; - if (fPimplLock == 0) + if (fPimplLock == nullptr) throw std::runtime_error("TSQ: pop(): no sync!"); if (fShutdown) @@ -213,7 +211,7 @@ class ThreadSafeQueue uint32_t curSize, workSize; TSQSize_t ret = {0, 0}; - if (fPimplLock == 0) + if (fPimplLock == nullptr) throw std::runtime_error("TSQ: pop_some(): no sync!"); t.clear(); @@ -268,7 +266,7 @@ class ThreadSafeQueue */ bool empty() const { - if (fPimplLock == 0) + if (fPimplLock == nullptr) throw std::runtime_error("TSQ: empty(): no sync!"); boost::mutex::scoped_lock lk(*fPimplLock); @@ -281,7 +279,7 @@ class ThreadSafeQueue { TSQSize_t ret; - if (fPimplLock == 0) + if (fPimplLock == nullptr) throw std::runtime_error("TSQ: size(): no sync!"); boost::mutex::scoped_lock lk(*fPimplLock); @@ -298,15 +296,13 @@ class ThreadSafeQueue { fShutdown = true; - if (fPimplCond != 0) + if (fPimplCond != nullptr) fPimplCond->notify_all(); - - return; } void clear() { - if (fPimplLock == 0) + if (fPimplLock == nullptr) throw std::runtime_error("TSQ: clear(): no sync!"); boost::mutex::scoped_lock lk(*fPimplLock); @@ -315,7 +311,6 @@ class ThreadSafeQueue fImpl.pop(); bytes = 0; - return; } private: diff --git a/dbcon/joblist/timestamp.h b/dbcon/joblist/timestamp.h index 2aa188d34..20597fb4f 100644 --- a/dbcon/joblist/timestamp.h +++ b/dbcon/joblist/timestamp.h @@ -33,27 +33,27 @@ class JSTimeStamp { public: JSTimeStamp(); - ~JSTimeStamp(){}; + ~JSTimeStamp() = default; inline void setFirstInsertTime() { - gettimeofday(&fFirstInsertTime, 0); + gettimeofday(&fFirstInsertTime, nullptr); } inline void setLastInsertTime() { - gettimeofday(&fLastInsertTime, 0); + gettimeofday(&fLastInsertTime, nullptr); } inline void setEndOfInputTime() { - gettimeofday(&fEndofInputTime, 0); + gettimeofday(&fEndofInputTime, nullptr); } inline void setFirstReadTime() { - gettimeofday(&fFirstReadTime, 0); + gettimeofday(&fFirstReadTime, nullptr); } inline void setLastReadTime() { - gettimeofday(&fLastReadTime, 0); + gettimeofday(&fLastReadTime, nullptr); } inline void setFirstInsertTime(const struct timeval& t) diff --git a/dbcon/joblist/tupleaggregatestep.h b/dbcon/joblist/tupleaggregatestep.h index 18dc22a91..3a816cd3f 100644 --- a/dbcon/joblist/tupleaggregatestep.h +++ b/dbcon/joblist/tupleaggregatestep.h @@ -103,21 +103,21 @@ class TupleAggregateStep : public JobStep, public TupleDeliveryStep /** @brief TupleAggregateStep destructor */ - ~TupleAggregateStep(); + ~TupleAggregateStep() override; /** @brief virtual void Run method */ - void run(); - void join(); + void run() override; + void join() override; - const std::string toString() const; + const std::string toString() const override; - void setOutputRowGroup(const rowgroup::RowGroup&); - const rowgroup::RowGroup& getOutputRowGroup() const; - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + void setOutputRowGroup(const rowgroup::RowGroup&) override; + const rowgroup::RowGroup& getOutputRowGroup() const override; + const rowgroup::RowGroup& getDeliveredRowGroup() const override; + void deliverStringTableRowGroup(bool b) override; + bool deliverStringTableRowGroup() const override; + uint32_t nextBand(messageqcpp::ByteStream& bs) override; uint32_t nextBand_singleThread(messageqcpp::ByteStream& bs); bool setPmHJAggregation(JobStep* step); void savePmHJData(rowgroup::SP_ROWAGG_t&, rowgroup::SP_ROWAGG_t&, rowgroup::RowGroup&); @@ -167,9 +167,10 @@ class TupleAggregateStep : public JobStep, public TupleDeliveryStep template static bool tryToFindEqualFunctionColumnByTupleKey(JobInfo& jobInfo, GroupByMap& groupByMap, const uint32_t tupleKey, uint32_t& foundTypleKey); - // This functions are workaround for the function above. For some reason different parts of the code with same - // semantics use different containers. - static uint32_t getTupleKeyFromTuple(const boost::tuple*>& tuple); + // This functions are workaround for the function above. For some reason different parts of the code with + // same semantics use different containers. + static uint32_t getTupleKeyFromTuple( + const boost::tuple*>& tuple); static uint32_t getTupleKeyFromTuple(uint32_t key); boost::shared_ptr fCatalog; @@ -198,7 +199,7 @@ class TupleAggregateStep : public JobStep, public TupleDeliveryStep class Aggregator { public: - Aggregator(TupleAggregateStep* step) : fStep(step) + explicit Aggregator(TupleAggregateStep* step) : fStep(step) { } void operator()() diff --git a/dbcon/joblist/tupleannexstep.h b/dbcon/joblist/tupleannexstep.h index 4443d3641..255306092 100644 --- a/dbcon/joblist/tupleannexstep.h +++ b/dbcon/joblist/tupleannexstep.h @@ -42,30 +42,30 @@ class TupleAnnexStep : public JobStep, public TupleDeliveryStep public: /** @brief TupleAnnexStep constructor */ - TupleAnnexStep(const JobInfo& jobInfo); + explicit TupleAnnexStep(const JobInfo& jobInfo); // Copy ctor to have a class mutex TupleAnnexStep(const TupleAnnexStep& copy); /** @brief TupleAnnexStep destructor */ - ~TupleAnnexStep(); + ~TupleAnnexStep() override; // inherited methods - void run(); - void join(); - const std::string toString() const; + void run() override; + void join() override; + const std::string toString() const override; /** @brief TupleJobStep's pure virtual methods */ - const rowgroup::RowGroup& getOutputRowGroup() const; - void setOutputRowGroup(const rowgroup::RowGroup&); + const rowgroup::RowGroup& getOutputRowGroup() const override; + void setOutputRowGroup(const rowgroup::RowGroup&) override; /** @brief TupleDeliveryStep's pure virtual methods */ - uint32_t nextBand(messageqcpp::ByteStream& bs); - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; + uint32_t nextBand(messageqcpp::ByteStream& bs) override; + const rowgroup::RowGroup& getDeliveredRowGroup() const override; + void deliverStringTableRowGroup(bool b) override; + bool deliverStringTableRowGroup() const override; void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); @@ -95,7 +95,7 @@ class TupleAnnexStep : public JobStep, public TupleDeliveryStep fMaxThreads = number; } - virtual bool stringTableFriendly() + bool stringTableFriendly() override { return true; } @@ -132,7 +132,7 @@ class TupleAnnexStep : public JobStep, public TupleDeliveryStep class Runner { public: - Runner(TupleAnnexStep* step) : fStep(step), id(0) + explicit Runner(TupleAnnexStep* step) : fStep(step), id(0) { } Runner(TupleAnnexStep* step, uint32_t id) : fStep(step), id(id) @@ -178,7 +178,7 @@ class reservablePQ : private std::priority_queue { public: typedef typename std::priority_queue::size_type size_type; - reservablePQ(size_type capacity = 0) + explicit reservablePQ(size_type capacity = 0) { reserve(capacity); }; @@ -193,4 +193,3 @@ class reservablePQ : private std::priority_queue }; } // namespace joblist - diff --git a/dbcon/joblist/tupleconstantstep.h b/dbcon/joblist/tupleconstantstep.h index ad36b5c9a..3f92470ab 100644 --- a/dbcon/joblist/tupleconstantstep.h +++ b/dbcon/joblist/tupleconstantstep.h @@ -32,34 +32,34 @@ class TupleConstantStep : public JobStep, public TupleDeliveryStep public: /** @brief TupleConstantStep constructor */ - TupleConstantStep(const JobInfo& jobInfo); + explicit TupleConstantStep(const JobInfo& jobInfo); /** @brief TupleConstantStep destructor */ - ~TupleConstantStep(); + ~TupleConstantStep() override; /** @brief virtual void Run method */ - void run(); + void run() override; /** @brief virtual void join method */ - void join(); + void join() override; /** @brief virtual string toString method */ - const std::string toString() const; + const std::string toString() const override; - void setOutputRowGroup(const rowgroup::RowGroup&); - const rowgroup::RowGroup& getOutputRowGroup() const; - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + void setOutputRowGroup(const rowgroup::RowGroup&) override; + const rowgroup::RowGroup& getOutputRowGroup() const override; + const rowgroup::RowGroup& getDeliveredRowGroup() const override; + void deliverStringTableRowGroup(bool b) override; + bool deliverStringTableRowGroup() const override; + uint32_t nextBand(messageqcpp::ByteStream& bs) override; virtual void initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn); virtual void fillInConstants(const rowgroup::Row& rowIn, rowgroup::Row& rowOut); - static SJSTEP addConstantStep(const JobInfo& jobInfo, const rowgroup::RowGroup* rg = NULL); + static SJSTEP addConstantStep(const JobInfo& jobInfo, const rowgroup::RowGroup* rg = nullptr); protected: virtual void execute(); @@ -93,7 +93,7 @@ class TupleConstantStep : public JobStep, public TupleDeliveryStep class Runner { public: - Runner(TupleConstantStep* step) : fStep(step) + explicit Runner(TupleConstantStep* step) : fStep(step) { } void operator()() @@ -114,26 +114,26 @@ class TupleConstantOnlyStep : public TupleConstantStep public: /** @brief TupleConstantOnlyStep constructor */ - TupleConstantOnlyStep(const JobInfo& jobInfo); + explicit TupleConstantOnlyStep(const JobInfo& jobInfo); /** @brief TupleConstantOnlyStep destructor */ - ~TupleConstantOnlyStep(); + ~TupleConstantOnlyStep() override; /** @brief virtual void Run method */ - void run(); + void run() override; /** @brief virtual void initialize method */ - virtual void initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn); + void initialize(const JobInfo& jobInfo, const rowgroup::RowGroup* rgIn) override; - const std::string toString() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + const std::string toString() const override; + uint32_t nextBand(messageqcpp::ByteStream& bs) override; protected: using TupleConstantStep::fillInConstants; - void fillInConstants(); + void fillInConstants() override; }; class TupleConstantBooleanStep : public TupleConstantStep @@ -145,11 +145,11 @@ class TupleConstantBooleanStep : public TupleConstantStep /** @brief TupleConstantBooleanStep destructor */ - ~TupleConstantBooleanStep(); + ~TupleConstantBooleanStep() override; /** @brief virtual void Run method */ - void run(); + void run() override; /** @brief virtual void initialize method For some reason, this doesn't match the base class's virtual signature. @@ -159,8 +159,8 @@ class TupleConstantBooleanStep : public TupleConstantStep using TupleConstantStep::initialize; void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); - const std::string toString() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + const std::string toString() const override; + uint32_t nextBand(messageqcpp::ByteStream& bs) override; virtual void boolValue(bool b) { @@ -172,14 +172,14 @@ class TupleConstantBooleanStep : public TupleConstantStep } protected: - void execute() + void execute() override { } using TupleConstantStep::fillInConstants; - void fillInConstants() + void fillInConstants() override { } - void constructContanstRow(const JobInfo& jobInfo) + void constructContanstRow(const JobInfo& jobInfo) override { } @@ -188,4 +188,3 @@ class TupleConstantBooleanStep : public TupleConstantStep }; } // namespace joblist - diff --git a/dbcon/joblist/tuplehashjoin.h b/dbcon/joblist/tuplehashjoin.h index 1636ccd18..9b8adafe0 100644 --- a/dbcon/joblist/tuplehashjoin.h +++ b/dbcon/joblist/tuplehashjoin.h @@ -46,20 +46,20 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep /** * @param */ - TupleHashJoinStep(const JobInfo& jobInfo); - virtual ~TupleHashJoinStep(); + explicit TupleHashJoinStep(const JobInfo& jobInfo); + ~TupleHashJoinStep() override; void setLargeSideBPS(BatchPrimitive*); void setLargeSideStepsOut(const std::vector& largeSideSteps); void setSmallSideStepsOut(const std::vector& smallSideSteps); /* mandatory JobStep interface */ - void run(); - void join(); - const std::string toString() const; + void run() override; + void join() override; + const std::string toString() const override; /* These tableOID accessors can go away soon */ - execplan::CalpontSystemCatalog::OID tableOid() const + execplan::CalpontSystemCatalog::OID tableOid() const override { return fTableOID2; } @@ -178,7 +178,7 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep fCorrelatedSide = c; } using JobStep::tupleId; - uint64_t tupleId() const + uint64_t tupleId() const override { return fTupleId2; } @@ -212,11 +212,11 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep const std::vector>& smallkeys, const std::vector>& largekeys); - void setOutputRowGroup(const rowgroup::RowGroup& rg); + void setOutputRowGroup(const rowgroup::RowGroup& rg) override; - uint32_t nextBand(messageqcpp::ByteStream& bs); + uint32_t nextBand(messageqcpp::ByteStream& bs) override; - const rowgroup::RowGroup& getOutputRowGroup() const + const rowgroup::RowGroup& getOutputRowGroup() const override { return outputRG; } @@ -311,17 +311,17 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep void addFcnExpGroup2(const boost::shared_ptr& fe); bool hasFcnExpGroup2() { - return (fe2 != NULL); + return (fe2 != nullptr); } /* Functions & Expressions in select and groupby clause */ - void setFcnExpGroup3(const std::vector& fe); - void setFE23Output(const rowgroup::RowGroup& rg); + void setFcnExpGroup3(const std::vector& fe) override; + void setFE23Output(const rowgroup::RowGroup& rg) override; /* result rowgroup */ - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; + const rowgroup::RowGroup& getDeliveredRowGroup() const override; + void deliverStringTableRowGroup(bool b) override; + bool deliverStringTableRowGroup() const override; // joinId void joinId(int64_t id) @@ -343,7 +343,7 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep boost::shared_ptr getJoinFilter(uint32_t index) const; void setJoinFilterInputRG(const rowgroup::RowGroup& rg); - virtual bool stringTableFriendly() + bool stringTableFriendly() override { return true; } @@ -367,7 +367,7 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep fFunctionJoinInfo = fji; } - void abort(); + void abort() override; void returnMemory() { if (fMemSizeForOutputRG > 0) @@ -462,7 +462,7 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep struct HJRunner { - HJRunner(TupleHashJoinStep* hj) : HJ(hj) + explicit HJRunner(TupleHashJoinStep* hj) : HJ(hj) { } void operator()() @@ -553,10 +553,10 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep rowgroup::Row& joinedRow, rowgroup::Row& baseRow, std::vector>& joinMatches, std::shared_ptr& smallRowTemplates, RowGroupDL* outputDL, - std::vector>* joiners = NULL, - std::shared_ptr[]>* rgMappings = NULL, - std::shared_ptr[]>* feMappings = NULL, - boost::scoped_array>* smallNullMem = NULL); + std::vector>* joiners = nullptr, + std::shared_ptr[]>* rgMappings = nullptr, + std::shared_ptr[]>* feMappings = nullptr, + boost::scoped_array>* smallNullMem = nullptr); void finishSmallOuterJoin(); void makeDupList(const rowgroup::RowGroup& rg); void processDupList(uint32_t threadID, rowgroup::RowGroup& ingrp, std::vector* rowData); @@ -587,7 +587,7 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep /* Disk-based join support */ std::vector> djs; - boost::scoped_array > fifos; + boost::scoped_array> fifos; void djsReaderFcn(int index); uint64_t djsReader; // thread handle from thread pool struct DJSReader @@ -608,7 +608,7 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep void djsRelayFcn(); struct DJSRelay { - DJSRelay(TupleHashJoinStep* hj) : HJ(hj) + explicit DJSRelay(TupleHashJoinStep* hj) : HJ(hj) { } void operator()() diff --git a/dbcon/joblist/tuplehavingstep.h b/dbcon/joblist/tuplehavingstep.h index b4c185e31..8e61ae6d0 100644 --- a/dbcon/joblist/tuplehavingstep.h +++ b/dbcon/joblist/tuplehavingstep.h @@ -39,36 +39,36 @@ class TupleHavingStep : public ExpressionStep, public TupleDeliveryStep public: /** @brief TupleHavingStep constructor */ - TupleHavingStep(const JobInfo& jobInfo); + explicit TupleHavingStep(const JobInfo& jobInfo); /** @brief TupleHavingStep destructor */ - ~TupleHavingStep(); + ~TupleHavingStep() override; /** @brief virtual void Run method */ - void run(); - void join(); + void run() override; + void join() override; - const std::string toString() const; + const std::string toString() const override; /** @brief TupleJobStep's pure virtual methods */ - const rowgroup::RowGroup& getOutputRowGroup() const; - void setOutputRowGroup(const rowgroup::RowGroup&); + const rowgroup::RowGroup& getOutputRowGroup() const override; + void setOutputRowGroup(const rowgroup::RowGroup&) override; /** @brief TupleDeliveryStep's pure virtual methods */ - uint32_t nextBand(messageqcpp::ByteStream& bs); - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; + uint32_t nextBand(messageqcpp::ByteStream& bs) override; + const rowgroup::RowGroup& getDeliveredRowGroup() const override; + void deliverStringTableRowGroup(bool b) override; + bool deliverStringTableRowGroup() const override; void initialize(const rowgroup::RowGroup& rgIn, const JobInfo& jobInfo); using ExpressionStep::expressionFilter; - void expressionFilter(const execplan::ParseTree* filter, JobInfo& jobInfo); + void expressionFilter(const execplan::ParseTree* filter, JobInfo& jobInfo) override; - virtual bool stringTableFriendly() + bool stringTableFriendly() override { return true; } @@ -93,7 +93,7 @@ class TupleHavingStep : public ExpressionStep, public TupleDeliveryStep class Runner { public: - Runner(TupleHavingStep* step) : fStep(step) + explicit Runner(TupleHavingStep* step) : fStep(step) { } void operator()() @@ -114,4 +114,3 @@ class TupleHavingStep : public ExpressionStep, public TupleDeliveryStep }; } // namespace joblist - diff --git a/dbcon/joblist/tupleunion.h b/dbcon/joblist/tupleunion.h index 9b9a0d75f..8ede13b49 100644 --- a/dbcon/joblist/tupleunion.h +++ b/dbcon/joblist/tupleunion.h @@ -37,37 +37,38 @@ namespace joblist { -using normalizeFunctionsT = std::vector>; +using normalizeFunctionsT = + std::vector>; class TupleUnion : public JobStep, public TupleDeliveryStep { public: TupleUnion(execplan::CalpontSystemCatalog::OID tableOID, const JobInfo& jobInfo); - ~TupleUnion(); + ~TupleUnion() override; - void run(); - void join(); + void run() override; + void join() override; - const std::string toString() const; - execplan::CalpontSystemCatalog::OID tableOid() const; + const std::string toString() const override; + execplan::CalpontSystemCatalog::OID tableOid() const override; void setInputRowGroups(const std::vector&); - void setOutputRowGroup(const rowgroup::RowGroup&); + void setOutputRowGroup(const rowgroup::RowGroup&) override; void setDistinctFlags(const std::vector&); - const rowgroup::RowGroup& getOutputRowGroup() const + const rowgroup::RowGroup& getOutputRowGroup() const override { return outputRG; } - const rowgroup::RowGroup& getDeliveredRowGroup() const + const rowgroup::RowGroup& getDeliveredRowGroup() const override { return outputRG; } - void deliverStringTableRowGroup(bool b) + void deliverStringTableRowGroup(bool b) override { outputRG.setUseStringTable(b); } - bool deliverStringTableRowGroup() const + bool deliverStringTableRowGroup() const override { return outputRG.usesStringTable(); } @@ -107,7 +108,7 @@ class TupleUnion : public JobStep, public TupleDeliveryStep fView2 = vw; } - uint32_t nextBand(messageqcpp::ByteStream& bs); + uint32_t nextBand(messageqcpp::ByteStream& bs) override; private: struct RowPosition @@ -115,12 +116,13 @@ class TupleUnion : public JobStep, public TupleDeliveryStep uint64_t group : 48; uint64_t row : 16; - inline RowPosition(uint64_t i = 0, uint64_t j = 0) : group(i), row(j){}; + inline explicit RowPosition(uint64_t i = 0, uint64_t j = 0) : group(i), row(j){}; static const uint64_t normalizedFlag = 0x800000000000ULL; // 48th bit is set }; void getOutput(rowgroup::RowGroup* rg, rowgroup::Row* row, rowgroup::RGData* data); - void addToOutput(rowgroup::Row* r, rowgroup::RowGroup* rg, bool keepit, rowgroup::RGData& data, uint32_t& tmpOutputRowCount); + void addToOutput(rowgroup::Row* r, rowgroup::RowGroup* rg, bool keepit, rowgroup::RGData& data, + uint32_t& tmpOutputRowCount); void normalize(const rowgroup::Row& in, rowgroup::Row* out, const normalizeFunctionsT& normalizeFunctions); void writeNull(rowgroup::Row* out, uint32_t col); void readInput(uint32_t); @@ -159,7 +161,7 @@ class TupleUnion : public JobStep, public TupleDeliveryStep { TupleUnion* ts; utils::Hasher_r h; - Hasher(TupleUnion* t) : ts(t) + explicit Hasher(TupleUnion* t) : ts(t) { } uint64_t operator()(const RowPosition&) const; @@ -167,13 +169,13 @@ class TupleUnion : public JobStep, public TupleDeliveryStep struct Eq { TupleUnion* ts; - Eq(TupleUnion* t) : ts(t) + explicit Eq(TupleUnion* t) : ts(t) { } bool operator()(const RowPosition&, const RowPosition&) const; }; - typedef std::tr1::unordered_set > Uniquer_t; + typedef std::tr1::unordered_set> Uniquer_t; boost::scoped_ptr uniquer; std::vector rowMemory; diff --git a/dbcon/joblist/unique32generator.h b/dbcon/joblist/unique32generator.h index 0881846ca..71e3d57be 100644 --- a/dbcon/joblist/unique32generator.h +++ b/dbcon/joblist/unique32generator.h @@ -49,12 +49,8 @@ class UniqueNumberGenerator uint64_t getUnique64(); // generate unique 64-bit int private: - UniqueNumberGenerator() - { - } - ~UniqueNumberGenerator() - { - } + UniqueNumberGenerator() = default; + ~UniqueNumberGenerator() = default; static UniqueNumberGenerator* fUnique32Generator; static boost::mutex fLock; diff --git a/dbcon/joblist/virtualtable.h b/dbcon/joblist/virtualtable.h index 9034feaba..10094f402 100644 --- a/dbcon/joblist/virtualtable.h +++ b/dbcon/joblist/virtualtable.h @@ -31,9 +31,7 @@ class VirtualTable { public: VirtualTable(); - virtual ~VirtualTable() - { - } + virtual ~VirtualTable() = default; virtual void initialize(); void addColumn(const execplan::SRCP& column); diff --git a/dbcon/joblist/windowfunctionstep.h b/dbcon/joblist/windowfunctionstep.h index 1f70fae27..f9367a9fb 100644 --- a/dbcon/joblist/windowfunctionstep.h +++ b/dbcon/joblist/windowfunctionstep.h @@ -58,7 +58,7 @@ struct RowPosition uint64_t fGroupId : 48; uint64_t fRowId : 16; - inline RowPosition(uint64_t i = 0, uint64_t j = 0) : fGroupId(i), fRowId(j){}; + inline explicit RowPosition(uint64_t i = 0, uint64_t j = 0) : fGroupId(i), fRowId(j){}; }; /** @brief class WindowFunctionStep @@ -71,23 +71,23 @@ class WindowFunctionStep : public JobStep, public TupleDeliveryStep * @param * @param */ - WindowFunctionStep(const JobInfo&); + explicit WindowFunctionStep(const JobInfo&); /** @brief WindowFunctionStep destructor */ - virtual ~WindowFunctionStep(); + ~WindowFunctionStep() override; /** @brief virtual methods */ - void run(); - void join(); - const std::string toString() const; - void setOutputRowGroup(const rowgroup::RowGroup&); - const rowgroup::RowGroup& getOutputRowGroup() const; - const rowgroup::RowGroup& getDeliveredRowGroup() const; - void deliverStringTableRowGroup(bool b); - bool deliverStringTableRowGroup() const; - uint32_t nextBand(messageqcpp::ByteStream& bs); + void run() override; + void join() override; + const std::string toString() const override; + void setOutputRowGroup(const rowgroup::RowGroup&) override; + const rowgroup::RowGroup& getOutputRowGroup() const override; + const rowgroup::RowGroup& getDeliveredRowGroup() const override; + void deliverStringTableRowGroup(bool b) override; + bool deliverStringTableRowGroup() const override; + uint32_t nextBand(messageqcpp::ByteStream& bs) override; /** @brief initialize methods */ @@ -146,7 +146,7 @@ class WindowFunctionStep : public JobStep, public TupleDeliveryStep class Runner { public: - Runner(WindowFunctionStep* step) : fStep(step) + explicit Runner(WindowFunctionStep* step) : fStep(step) { } void operator()() @@ -188,7 +188,7 @@ class WindowFunctionStep : public JobStep, public TupleDeliveryStep class WFunction { public: - WFunction(WindowFunctionStep* step) : fStep(step) + explicit WFunction(WindowFunctionStep* step) : fStep(step) { } void operator()() @@ -220,4 +220,3 @@ class WindowFunctionStep : public JobStep, public TupleDeliveryStep }; } // namespace joblist - diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index e93fa46e2..4050356e4 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -58,9 +58,7 @@ class ha_mcs : public handler public: ha_mcs(handlerton* hton, TABLE_SHARE* table_arg); - virtual ~ha_mcs() - { - } + virtual ~ha_mcs() = default; /** @brief The name that will be used for display purposes. @@ -121,22 +119,22 @@ class ha_mcs : public handler /** @brief Called in test_quick_select to determine if indexes should be used. */ - virtual IO_AND_CPU_COST scan_time() override - { - IO_AND_CPU_COST cost; - cost.io= 0.0; - /* - For now, assume all cost is CPU cost. - The numbers are also very inadequate for the new cost model. - */ - cost.cpu= (double)(stats.records + stats.deleted) / 20.0 + 10; - return cost; - } + virtual IO_AND_CPU_COST scan_time() override + { + IO_AND_CPU_COST cost; + cost.io = 0.0; + /* + For now, assume all cost is CPU cost. + The numbers are also very inadequate for the new cost model. + */ + cost.cpu = (double)(stats.records + stats.deleted) / 20.0 + 10; + return cost; + } #else /** @brief Called in test_quick_select to determine if indexes should be used. */ - virtual double scan_time() override + double scan_time() override { return (double)(stats.records + stats.deleted) / 20.0 + 10; } @@ -324,7 +322,7 @@ class ha_mcs_cache : public ha_mcs ha_mcs_cache_share* share; ha_mcs_cache(handlerton* hton, TABLE_SHARE* table_arg, MEM_ROOT* mem_root); - ~ha_mcs_cache(); + ~ha_mcs_cache() override; /* The following functions duplicates calls to derived handler and diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h index 1eb1a04ce..3e588d86c 100644 --- a/dbcon/mysql/ha_mcs_datatype.h +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -113,7 +113,7 @@ class StoreFieldMariaDB : public StoreField return m_field->store(static_cast(val), 1); } - int store_float(float dl) override + int store_float(float dl) { if (dl == std::numeric_limits::infinity()) { @@ -128,7 +128,7 @@ class StoreFieldMariaDB : public StoreField return m_field->store(dl); } - int store_double(double dl) override + int store_double(double dl) { if (dl == std::numeric_limits::infinity()) { @@ -461,11 +461,9 @@ class WriteBatchFieldMariaDB : public WriteBatchField } else { - int32_t tmp = ( - (*const_cast(buf) << 8) | - (*const_cast(buf+1) << 16) | - (*const_cast(buf+2) << 24) - ) >> 8; + int32_t tmp = ((*const_cast(buf) << 8) | (*const_cast(buf + 1) << 16) | + (*const_cast(buf + 2) << 24)) >> + 8; fprintf(ci.filePtr(), "%d%c", tmp, ci.delimiter()); } return 3; @@ -477,11 +475,8 @@ class WriteBatchFieldMariaDB : public WriteBatchField fprintf(ci.filePtr(), "%c", ci.delimiter()); else { - uint32_t tmp = ( - (*const_cast(buf)) | - (*const_cast(buf+1) << 8) | - (*const_cast(buf+2) << 16) - ); + uint32_t tmp = ((*const_cast(buf)) | (*const_cast(buf + 1) << 8) | + (*const_cast(buf + 2) << 16)); fprintf(ci.filePtr(), "%u%c", tmp, ci.delimiter()); } return 3; diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 3ab43564e..d1d29aaa3 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -205,14 +205,14 @@ struct gp_walk_info , internalDecimalScale(4) , thd(0) , subSelectType(uint64_t(-1)) - , subQuery(0) + , subQuery(nullptr) , clauseType(INIT) , implicitExplicitGroupBy(false) , disableWrapping(false) , aggOnSelect(false) , hasWindowFunc(false) , hasSubSelect(false) - , lastSub(0) + , lastSub(nullptr) , derivedTbCnt(0) , recursionLevel(-1) , recursionHWM(0) @@ -277,9 +277,7 @@ struct cal_group_info , groupByDistinct(false) { } - ~cal_group_info() - { - } + ~cal_group_info() = default; List* groupByFields; // MCOL-1052 SELECT TABLE_LIST* groupByTables; // MCOL-1052 FROM @@ -303,7 +301,7 @@ struct cal_connection_info ALTER_FIRST_RENAME }; cal_connection_info() - : cal_conn_hndl(0) + : cal_conn_hndl(nullptr) , queryState(0) , currentTable(0) , traceFlags(0) @@ -313,7 +311,7 @@ struct cal_connection_info , singleInsert(true) , isLoaddataInfile(false) , isCacheInsert(false) - , dmlProc(0) + , dmlProc(nullptr) , rowsHaveInserted(0) , rc(0) , tableOid(0) @@ -322,7 +320,7 @@ struct cal_connection_info , expressionId(0) , mysqld_pid(getpid()) , cpimport_pid(0) - , filePtr(0) + , filePtr(nullptr) , headerLength(0) , useXbit(false) , useCpimport(mcs_use_import_for_batchinsert_mode_t::ON) @@ -429,7 +427,7 @@ void setError(THD* thd, uint32_t errcode, const std::string errmsg); void gp_walk(const Item* item, void* arg); void clearDeleteStacks(gp_walk_info& gwi); void parse_item(Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& parseInfo, - gp_walk_info* gwip = NULL); + gp_walk_info* gwip = nullptr); const std::string bestTableName(const Item_field* ifp); // execution plan util functions prototypes diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index 87ef0b826..d9a73a374 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -39,10 +39,10 @@ enum mcs_handler_types_t struct mcs_handler_info { - mcs_handler_info() : hndl_ptr(NULL), hndl_type(LEGACY){}; - mcs_handler_info(mcs_handler_types_t type) : hndl_ptr(NULL), hndl_type(type){}; + mcs_handler_info() : hndl_ptr(nullptr), hndl_type(LEGACY){}; + mcs_handler_info(mcs_handler_types_t type) : hndl_ptr(nullptr), hndl_type(type){}; mcs_handler_info(void* ptr, mcs_handler_types_t type) : hndl_ptr(ptr), hndl_type(type){}; - ~mcs_handler_info(){}; + ~mcs_handler_info() = default; void* hndl_ptr; mcs_handler_types_t hndl_type; }; diff --git a/dbcon/mysql/ha_subquery.h b/dbcon/mysql/ha_subquery.h index 8195c520e..31e6c29d9 100644 --- a/dbcon/mysql/ha_subquery.h +++ b/dbcon/mysql/ha_subquery.h @@ -26,7 +26,7 @@ #pragma once -//#undef LOG_INFO +// #undef LOG_INFO #include #include "idb_mysql.h" #include "ha_mcs_impl_if.h" @@ -48,9 +48,7 @@ class SubQuery next = *gwip.subQueriesChain; *gwip.subQueriesChain = this; } - virtual ~SubQuery() - { - } + virtual ~SubQuery() = default; virtual gp_walk_info& gwip() const { return fGwip; @@ -111,9 +109,7 @@ class WhereSubQuery : public SubQuery WhereSubQuery(gp_walk_info& gwip, Item_subselect* sub) : SubQuery(gwip), fSub(sub) { } // for exists - virtual ~WhereSubQuery() - { - } + ~WhereSubQuery() override = default; /** Accessors and mutators */ virtual Item_subselect* sub() const @@ -150,8 +146,8 @@ class ScalarSub : public WhereSubQuery ScalarSub(gp_walk_info& gwip, Item_func* func); ScalarSub(gp_walk_info& gwip, const execplan::SRCP& column, Item_subselect* sub, Item_func* func); ScalarSub(const ScalarSub& rhs); - ~ScalarSub(); - execplan::ParseTree* transform(); + ~ScalarSub() override; + execplan::ParseTree* transform() override; execplan::ParseTree* transform_between(); execplan::ParseTree* transform_in(); execplan::ParseTree* buildParseTree(execplan::PredicateOperator* op); @@ -177,10 +173,10 @@ class InSub : public WhereSubQuery InSub(gp_walk_info& gwip); InSub(gp_walk_info& gwip, Item_func* func); InSub(const InSub& rhs); - ~InSub(); - execplan::ParseTree* transform(); + ~InSub() override; + execplan::ParseTree* transform() override; void handleFunc(gp_walk_info* gwip, Item_func* func); - void handleNot(); + void handleNot() override; }; /** @@ -191,9 +187,9 @@ class ExistsSub : public WhereSubQuery public: ExistsSub(gp_walk_info&); // not complete. just for compile ExistsSub(gp_walk_info&, Item_subselect* sub); - ~ExistsSub(); - execplan::ParseTree* transform(); - void handleNot(); + ~ExistsSub() override; + execplan::ParseTree* transform() override; + void handleNot() override; }; /** @@ -211,7 +207,7 @@ class FromSubQuery : public SubQuery public: FromSubQuery(gp_walk_info&); FromSubQuery(gp_walk_info&, SELECT_LEX* fromSub); - ~FromSubQuery(); + ~FromSubQuery() override; const SELECT_LEX* fromSub() const { return fFromSub; @@ -240,7 +236,7 @@ class SelectSubQuery : public SubQuery public: SelectSubQuery(gp_walk_info&); SelectSubQuery(gp_walk_info&, Item_subselect* sel); - ~SelectSubQuery(); + ~SelectSubQuery() override; execplan::SCSEP transform(); Item_subselect* selSub() { diff --git a/dbcon/mysql/ha_view.h b/dbcon/mysql/ha_view.h index 46c6693c2..de395efe6 100644 --- a/dbcon/mysql/ha_view.h +++ b/dbcon/mysql/ha_view.h @@ -25,7 +25,7 @@ #pragma once -//#undef LOG_INFO +// #undef LOG_INFO #include "ha_mcs_impl_if.h" #include "idb_mysql.h" @@ -42,9 +42,7 @@ class View View(SELECT_LEX& select, gp_walk_info* parentGwip) : fSelect(select), fParentGwip(parentGwip) { } - ~View() - { - } + ~View() = default; execplan::CalpontSystemCatalog::TableAliasName& viewName(); void viewName(execplan::CalpontSystemCatalog::TableAliasName& viewName); diff --git a/dbcon/mysql/sm.h b/dbcon/mysql/sm.h index 0e055f8e0..a8d081a1b 100644 --- a/dbcon/mysql/sm.h +++ b/dbcon/mysql/sm.h @@ -60,14 +60,14 @@ const int SQL_NOT_FOUND = -1000; const int SQL_KILLED = -1001; const int CALPONT_INTERNAL_ERROR = -1007; -//#if IDB_SM_DEBUG -// extern std::ofstream smlog; -//#define SMDEBUGLOG smlog -//#else +// #if IDB_SM_DEBUG +// extern std::ofstream smlog; +// #define SMDEBUGLOG smlog +// #else #define SMDEBUGLOG \ if (false) \ std::cout -//#endif +// #endif extern const std::string DEFAULT_SAVE_PATH; typedef uint64_t tableid_t; @@ -84,9 +84,7 @@ typedef struct Column Column() : tableID(-1) { } - ~Column() - { - } + ~Column() = default; int tableID; int colPos; int dataType; @@ -132,7 +130,14 @@ struct Profiler struct cpsm_tplsch_t { cpsm_tplsch_t() - : tableid(0), rowsreturned(0), rowGroup(0), traceFlags(0), bandID(0), saveFlag(0), bandsReturned(0), ctp(0) + : tableid(0) + , rowsreturned(0) + , rowGroup(nullptr) + , traceFlags(0) + , bandID(0) + , saveFlag(0) + , bandsReturned(0) + , ctp(0) { } ~cpsm_tplsch_t() @@ -172,7 +177,7 @@ struct cpsm_tplsch_t uint16_t getStatus() { - idbassert(rowGroup != 0); + idbassert(rowGroup != nullptr); return rowGroup->getStatus(); } @@ -295,7 +300,7 @@ extern status_t sm_cleanup(cpsm_conhdl_t*); extern status_t tpl_open(tableid_t, sp_cpsm_tplh_t&, cpsm_conhdl_t*); extern status_t tpl_scan_open(tableid_t, sp_cpsm_tplsch_t&, cpsm_conhdl_t*); -extern status_t tpl_scan_fetch(sp_cpsm_tplsch_t&, cpsm_conhdl_t*, int* k = 0); +extern status_t tpl_scan_fetch(sp_cpsm_tplsch_t&, cpsm_conhdl_t*, int* k = nullptr); extern status_t tpl_scan_close(sp_cpsm_tplsch_t&); extern status_t tpl_close(sp_cpsm_tplh_t&, cpsm_conhdl_t**, querystats::QueryStats& stats, bool ask_4_stats, bool clear_scan_ctx = false); diff --git a/ddlproc/ddlprocessor.h b/ddlproc/ddlprocessor.h index 82bf22d2c..18a364b1e 100644 --- a/ddlproc/ddlprocessor.h +++ b/ddlproc/ddlprocessor.h @@ -35,6 +35,7 @@ namespace ddlprocessor class DDLProcessor { public: + DDLProcessor() = delete; /** @brief ctor * * @param packageMaxThreads the maximum number of threads to process ddl packages @@ -75,12 +76,7 @@ class DDLProcessor fPackageWorkQueueSize = workQueueSize; } - protected: private: - /** @brief ctor - */ - DDLProcessor(); - /** @brief the thread pool for processing ddl packages */ threadpool::ThreadPool fDdlPackagepool; diff --git a/dmlproc/batchinsertprocessor.h b/dmlproc/batchinsertprocessor.h index a8d9ef82c..a950b0986 100644 --- a/dmlproc/batchinsertprocessor.h +++ b/dmlproc/batchinsertprocessor.h @@ -22,7 +22,7 @@ #pragma once -#include +#include #include #include @@ -45,6 +45,7 @@ class BatchInsertProc typedef std::queue pkg_type; typedef boost::shared_ptr SP_PKG; typedef std::vector BulkSetHWMArgs; + BatchInsertProc() = delete; BatchInsertProc(bool isAutocommitOn, uint32_t tableOid, execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM* aDbrm); BatchInsertProc(const BatchInsertProc& rhs); diff --git a/primitives/blockcache/blockrequestprocessor.h b/primitives/blockcache/blockrequestprocessor.h index 0970c40d5..bbd66cda0 100644 --- a/primitives/blockcache/blockrequestprocessor.h +++ b/primitives/blockcache/blockrequestprocessor.h @@ -31,7 +31,7 @@ #include "fileblockrequestqueue.h" #include "filebuffermgr.h" #include "iomanager.h" -#include +#include /** @author Jason Rodriguez diff --git a/primitives/primproc/activestatementcounter.h b/primitives/primproc/activestatementcounter.h index 87e7163aa..550819123 100644 --- a/primitives/primproc/activestatementcounter.h +++ b/primitives/primproc/activestatementcounter.h @@ -21,7 +21,7 @@ #pragma once -#include +#include #include #include @@ -35,9 +35,7 @@ class ActiveStatementCounter { } - virtual ~ActiveStatementCounter() - { - } + virtual ~ActiveStatementCounter() = default; void incr(bool& counted); void decr(bool& counted); @@ -61,4 +59,3 @@ class ActiveStatementCounter boost::condition condvar; BRM::VSS fVss; }; - diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index 012f84467..f686ecc15 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -66,7 +66,7 @@ typedef boost::shared_ptr SBPP; class scalar_exception : public std::exception { - const char* what() const throw() + const char* what() const noexcept override { return "Not a scalar subquery."; } @@ -231,7 +231,7 @@ class BatchPrimitiveProcessor /* Common space for primitive data */ alignas(utils::MAXCOLUMNWIDTH) uint8_t blockData[BLOCK_SIZE * utils::MAXCOLUMNWIDTH]; uint8_t blockDataAux[BLOCK_SIZE * execplan::AUX_COL_WIDTH]; - std::unique_ptr outputMsg; + std::unique_ptr outputMsg; uint32_t outMsgSize; std::vector filterSteps; diff --git a/primitives/primproc/bppseeder.h b/primitives/primproc/bppseeder.h index c8896e8bc..b3c8eec7e 100644 --- a/primitives/primproc/bppseeder.h +++ b/primitives/primproc/bppseeder.h @@ -46,13 +46,14 @@ namespace primitiveprocessor class BPPSeeder : public threadpool::FairThreadPool::Functor { public: + BPPSeeder() = delete; BPPSeeder(const messageqcpp::SBS&, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& ios, const int pmThreads, const bool trace = false); BPPSeeder(const BPPSeeder& b); - virtual ~BPPSeeder(); + ~BPPSeeder() override; - int operator()(); + int operator()() override; bool isSysCat(); boost::shared_ptr spof; @@ -74,7 +75,6 @@ class BPPSeeder : public threadpool::FairThreadPool::Functor } private: - BPPSeeder(); void catchHandler(const std::string& s, uint32_t uniqueID, uint32_t step); void flushSyscatOIDs(); diff --git a/primitives/primproc/bppsendthread.h b/primitives/primproc/bppsendthread.h index 88f71456c..7f2028917 100644 --- a/primitives/primproc/bppsendthread.h +++ b/primitives/primproc/bppsendthread.h @@ -29,6 +29,7 @@ #include #include #include +#include #include "threadnaming.h" #include "fair_threadpool.h" @@ -49,19 +50,10 @@ class BPPSendThread Msg_t() : sockIndex(0) { } - Msg_t(const Msg_t& m) : msg(m.msg), sock(m.sock), sockLock(m.sockLock), sockIndex(m.sockIndex) - { - } - Msg_t& operator=(const Msg_t& m) - { - msg = m.msg; - sock = m.sock; - sockLock = m.sockLock; - sockIndex = m.sockIndex; - return *this; - } - Msg_t(const messageqcpp::SBS& m, const SP_UM_IOSOCK& so, const SP_UM_MUTEX& sl, int si) - : msg(m), sock(so), sockLock(sl), sockIndex(si) + Msg_t(const Msg_t& m) = default; + Msg_t& operator=(const Msg_t& m) = default; + Msg_t(messageqcpp::SBS m, SP_UM_IOSOCK so, SP_UM_MUTEX sl, int si) + : msg(std::move(m)), sock(std::move(so)), sockLock(std::move(sl)), sockIndex(si) { } }; @@ -87,7 +79,7 @@ class BPPSendThread } void setProcessorPool(boost::shared_ptr processorPool) { - fProcessorPool = processorPool; + fProcessorPool = std::move(processorPool); } private: @@ -127,7 +119,7 @@ class BPPSendThread /* Load balancing structures */ struct Connection_t { - Connection_t(const SP_UM_MUTEX& lock, const SP_UM_IOSOCK& so) : sockLock(lock), sock(so) + Connection_t(SP_UM_MUTEX lock, SP_UM_IOSOCK so) : sockLock(std::move(lock)), sock(std::move(so)) { } SP_UM_MUTEX sockLock; diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index be99261c4..3914debe9 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -45,9 +45,9 @@ class ColumnCommand : public Command public: ColumnCommand(); ColumnCommand(execplan::CalpontSystemCatalog::ColType& aColType); - virtual ~ColumnCommand(); + ~ColumnCommand() override; - inline uint64_t getLBID() + inline uint64_t getLBID() override { return lbid; } @@ -68,12 +68,12 @@ class ColumnCommand : public Command return colType; } - void execute(); + void execute() override; void execute(int64_t* vals); // used by RTSCommand to redirect values - virtual void prep(int8_t outputType, bool absRids); - void project(); - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t pos); - void nextLBID(); + void prep(int8_t outputType, bool absRids) override; + void project() override; + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t pos) override; + void nextLBID() override; bool isScan() { return _isScan; @@ -88,18 +88,18 @@ class ColumnCommand : public Command return lbidAux; } - void createCommand(messageqcpp::ByteStream&); + void createCommand(messageqcpp::ByteStream&) override; void createCommand(execplan::CalpontSystemCatalog::ColType& aColType, messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); - void setMakeAbsRids(bool m) + void resetCommand(messageqcpp::ByteStream&) override; + void setMakeAbsRids(bool m) override { makeAbsRids = m; } bool willPrefetch(); int64_t getLastLbid(); - void getLBIDList(uint32_t loopCount, std::vector* lbids); + void getLBIDList(uint32_t loopCount, std::vector* lbids) override; - virtual SCommand duplicate(); + SCommand duplicate() override; bool operator==(const ColumnCommand&) const; bool operator!=(const ColumnCommand&) const; @@ -111,7 +111,7 @@ class ColumnCommand : public Command void disableFilters(); void enableFilters(); - int getCompType() const + int getCompType() const override { return colType.compressionType; } @@ -321,4 +321,3 @@ inline void ColumnCommand::fillEmptyBlock(uint } } // namespace primitiveprocessor - diff --git a/primitives/primproc/dictstep.h b/primitives/primproc/dictstep.h index 9c5120b98..cb07717bb 100644 --- a/primitives/primproc/dictstep.h +++ b/primitives/primproc/dictstep.h @@ -39,28 +39,28 @@ class DictStep : public Command { public: DictStep(); - virtual ~DictStep(); + ~DictStep() override; - void execute(); - void project(); + void execute() override; + void project() override; void project(int64_t* vals); // used by RTSCommand to redirect input - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t row); + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t row) override; void projectIntoRowGroup(rowgroup::RowGroup& rg, int64_t* vals, uint32_t col); - uint64_t getLBID(); + uint64_t getLBID() override; /* This doesn't do anything for this class... make it column-specific or not? */ - void nextLBID(); - void createCommand(messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); + void nextLBID() override; + void createCommand(messageqcpp::ByteStream&) override; + void resetCommand(messageqcpp::ByteStream&) override; /* Put bootstrap code here (ie, build the template primitive msg) */ - void prep(int8_t outputType, bool makeAbsRids); + void prep(int8_t outputType, bool makeAbsRids) override; - SCommand duplicate(); + SCommand duplicate() override; bool operator==(const DictStep&) const; bool operator!=(const DictStep&) const; - int getCompType() const + int getCompType() const override { return compressionType; } @@ -78,13 +78,11 @@ class DictStep : public Command const uint8_t* ptr; unsigned len; - StringPtr() : ptr(NULL), len(0) + StringPtr() : ptr(nullptr), len(0) { - ; } StringPtr(const uint8_t* p, unsigned l) : ptr(p), len(l) { - ; } utils::ConstString getConstString() const { @@ -111,9 +109,7 @@ class DictStep : public Command OrderedToken() : inResult(false) { } - ~OrderedToken() - { - } + ~OrderedToken() = default; }; struct TokenSorter { diff --git a/primitives/primproc/filtercommand.h b/primitives/primproc/filtercommand.h index 0532196c1..3515c7c02 100644 --- a/primitives/primproc/filtercommand.h +++ b/primitives/primproc/filtercommand.h @@ -38,21 +38,21 @@ class FilterCommand : public Command { public: FilterCommand(); - virtual ~FilterCommand(); + ~FilterCommand() override; // returns a FilterCommand based on column types static Command* makeFilterCommand(messageqcpp::ByteStream&, std::vector& cmds); // virtuals from base class -- Command - void execute(); - void project(); - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); - uint64_t getLBID(); - void nextLBID(); - void createCommand(messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); - SCommand duplicate(); - void prep(int8_t outputType, bool makeAbsRids); + void execute() override; + void project() override; + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col) override; + uint64_t getLBID() override; + void nextLBID() override; + void createCommand(messageqcpp::ByteStream&) override; + void resetCommand(messageqcpp::ByteStream&) override; + SCommand duplicate() override; + void prep(int8_t outputType, bool makeAbsRids) override; void setColTypes(const execplan::CalpontSystemCatalog::ColType& left, const execplan::CalpontSystemCatalog::ColType& right); @@ -61,7 +61,7 @@ class FilterCommand : public Command bool operator==(const FilterCommand&) const; bool operator!=(const FilterCommand&) const; - int getCompType() const + int getCompType() const override { return 0; } @@ -96,8 +96,8 @@ class ScaledFilterCmd : public FilterCommand { public: ScaledFilterCmd(); - virtual ~ScaledFilterCmd(); - SCommand duplicate(); + ~ScaledFilterCmd() override; + SCommand duplicate() override; void setFactor(double); double factor(); @@ -108,7 +108,7 @@ class ScaledFilterCmd : public FilterCommand protected: // compare method, take the indices to the values array - bool compare(uint64_t, uint64_t); + bool compare(uint64_t, uint64_t) override; // value used in comparison; double fFactor; @@ -123,11 +123,11 @@ class StrFilterCmd : public FilterCommand { public: StrFilterCmd(); - virtual ~StrFilterCmd(); + ~StrFilterCmd() override; // override FilterCommand methods - void execute(); - SCommand duplicate(); + void execute() override; + SCommand duplicate() override; void setCompareFunc(uint32_t); void setCharLength(size_t); @@ -139,7 +139,7 @@ class StrFilterCmd : public FilterCommand protected: // compare method, take the indices to the values array - bool compare(uint64_t, uint64_t); + bool compare(uint64_t, uint64_t) override; // compare method for differernt column combination, c--char[], s--string // compare char[]-char[] is not the same as int-int due to endian issue. diff --git a/primitives/primproc/passthrucommand.h b/primitives/primproc/passthrucommand.h index 9dbbc7d2a..87e577872 100644 --- a/primitives/primproc/passthrucommand.h +++ b/primitives/primproc/passthrucommand.h @@ -38,21 +38,21 @@ class PassThruCommand : public Command { public: PassThruCommand(); - virtual ~PassThruCommand(); + ~PassThruCommand() override; - void prep(int8_t outputType, bool makeAbsRids); - void execute(); - void project(); - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); - uint64_t getLBID(); - void nextLBID(); - void createCommand(messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); - SCommand duplicate(); + void prep(int8_t outputType, bool makeAbsRids) override; + void execute() override; + void project() override; + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col) override; + uint64_t getLBID() override; + void nextLBID() override; + void createCommand(messageqcpp::ByteStream&) override; + void resetCommand(messageqcpp::ByteStream&) override; + SCommand duplicate() override; bool operator==(const PassThruCommand&) const; bool operator!=(const PassThruCommand&) const; - int getCompType() const + int getCompType() const override { return 0; } diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index ceac88fc5..1c93f7f29 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -45,6 +45,7 @@ using namespace std; #include #include +#include #include using namespace boost; #include "distributedenginecomm.h" @@ -110,7 +111,7 @@ using namespace threadpool; // make global for blockcache // static const char* statsName = {"pm"}; -dbbc::Stats* gPMStatsPtr = 0; +dbbc::Stats* gPMStatsPtr = nullptr; bool gPMProfOn = false; uint32_t gSession = 0; dbbc::Stats pmstats(statsName); @@ -141,8 +142,6 @@ int noVB = 0; BPPMap bppMap; boost::mutex bppLock; -#define DJLOCK_READ 0 -#define DJLOCK_WRITE 1 boost::mutex djMutex; // lock for djLock, lol. std::map djLock; // djLock synchronizes destroy and joiner msgs, see bug 2619 @@ -160,9 +159,7 @@ struct preFetchCond waiters = 0; } - ~preFetchCond() - { - } + ~preFetchCond() = default; }; typedef preFetchCond preFetchBlock_t; @@ -202,7 +199,7 @@ void waitForRetry(long count) timespec ts; ts.tv_sec = 5L * count / 10L; ts.tv_nsec = (5L * count % 10L) * 100000000L; - nanosleep(&ts, 0); + nanosleep(&ts, nullptr); } void prefetchBlocks(const uint64_t lbid, const int compType, uint32_t* rCount) @@ -234,7 +231,7 @@ void prefetchBlocks(const uint64_t lbid, const int compType, uint32_t* rCount) return; } - preFetchBlock_t* pfb = 0; + preFetchBlock_t* pfb = nullptr; pfb = new preFetchBlock_t(lowlbid); pfBlockMap[lowlbid] = pfb; @@ -304,7 +301,7 @@ void prefetchBlocks(const uint64_t lbid, const int compType, uint32_t* rCount) if (pfBlockMap.erase(lowlbid) > 0) delete pfb; - pfb = 0; + pfb = nullptr; pfbMutex.unlock(); throw; } @@ -326,7 +323,7 @@ cleanup: if (pfBlockMap.erase(lowlbid) > 0) delete pfb; - pfb = 0; + pfb = nullptr; pfbMutex.unlock(); } // prefetchBlocks() @@ -524,15 +521,15 @@ void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bu boost::scoped_array uCmpBufSa; ptrdiff_t alignedBuffer = 0; - void* readBufferPtr = NULL; - char* cmpHdrBuf = NULL; - char* cmpBuf = NULL; - unsigned char* uCmpBuf = NULL; + void* readBufferPtr = nullptr; + char* cmpHdrBuf = nullptr; + char* cmpBuf = nullptr; + unsigned char* uCmpBuf = nullptr; uint64_t cmpBufLen = 0; int blockReadRetryCount = 0; unsigned idx = 0; int pageSize = getpagesize(); - IDBDataFile* fp = 0; + IDBDataFile* fp = nullptr; try { @@ -543,7 +540,7 @@ void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bu int opts = directIOFlag ? IDBDataFile::USE_ODIRECT : 0; fp = IDBDataFile::open(IDBPolicy::getType(fileNamePtr, IDBPolicy::PRIMPROC), fileNamePtr, "r", opts); - if (fp == NULL) + if (fp == nullptr) { int errCode = errno; SUMMARY_INFO2("open failed: ", fileNamePtr); @@ -552,7 +549,7 @@ void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bu // #if STRERROR_R_CHAR_P const char* p; - if ((p = strerror_r(errCode, errbuf, 80)) != 0) + if ((p = strerror_r(errCode, errbuf, 80)) != nullptr) errMsg = p; if (errCode == EINVAL) @@ -595,8 +592,8 @@ void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bu cout << "pread2(" << fd << ", 0x" << hex << (ptrdiff_t)readBufferPtr << dec << ", " << DATA_BLOCK_SIZE << ", " << offset << ") = " << i << endl; } -#endif // IDB_COMP_POC_DEBUG - } // if (compType == 0) +#endif // IDB_COMP_POC_DEBUG + } // if (compType == 0) else // if (compType != 0) { // retry if file is out of sync -- compressed column file only. @@ -692,7 +689,7 @@ void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bu uint64_t cmpBufOff = ptrList[idx].first; uint64_t cmpBufSz = ptrList[idx].second; - if (cmpBufSa.get() == NULL || cmpBufLen < cmpBufSz) + if (cmpBufSa.get() == nullptr || cmpBufLen < cmpBufSz) { cmpBufSa.reset(new char[cmpBufSz + pageSize]); cmpBufLen = cmpBufSz; @@ -759,12 +756,12 @@ void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bu catch (...) { delete fp; - fp = 0; + fp = nullptr; throw; } delete fp; - fp = 0; + fp = nullptr; // log the retries if (blockReadRetryCount > 0) @@ -787,7 +784,7 @@ void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bu return; } - FileBuffer* fbPtr = 0; + FileBuffer* fbPtr = nullptr; bool wasBlockInCache = false; fbPtr = bc.getBlockPtr(lbid, ver, flg); @@ -827,12 +824,12 @@ void loadBlock(uint64_t lbid, QueryContext v, uint32_t t, int compType, void* bu struct AsynchLoader { - AsynchLoader(uint64_t l, const QueryContext& v, uint32_t t, int ct, uint32_t* cCount, uint32_t* rCount, - bool trace, uint32_t sesID, boost::mutex* m, uint32_t* loaderCount, + AsynchLoader(uint64_t l, QueryContext v, uint32_t t, int ct, uint32_t* cCount, uint32_t* rCount, bool trace, + uint32_t sesID, boost::mutex* m, uint32_t* loaderCount, boost::shared_ptr st, // sendThread for abort upon exception. VSSCache* vCache) : lbid(l) - , ver(v) + , ver(std::move(v)) , txn(t) , compType(ct) , LBIDTrace(trace) @@ -841,7 +838,7 @@ struct AsynchLoader , readCount(rCount) , busyLoaders(loaderCount) , mutex(m) - , sendThread(st) + , sendThread(std::move(st)) , vssCache(vCache) { } @@ -1016,10 +1013,10 @@ class DictScanJob : public threadpool::FairThreadPool::Functor { public: DictScanJob(SP_UM_IOSOCK ios, SBS bs, SP_UM_MUTEX writeLock); - virtual ~DictScanJob(); + ~DictScanJob() override; void write(const SBS); - int operator()(); + int operator()() override; void catchHandler(const std::string& ex, uint32_t id, uint16_t code = logging::primitiveServerErr); void sendErrorMsg(uint32_t id, uint16_t code); @@ -1031,14 +1028,12 @@ class DictScanJob : public threadpool::FairThreadPool::Functor }; DictScanJob::DictScanJob(SP_UM_IOSOCK ios, SBS bs, SP_UM_MUTEX writeLock) - : fIos(ios), fByteStream(bs), fWriteLock(writeLock) + : fIos(std::move(ios)), fByteStream(std::move(bs)), fWriteLock(std::move(writeLock)) { dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); } -DictScanJob::~DictScanJob() -{ -} +DictScanJob::~DictScanJob() = default; void DictScanJob::write(const SBS sbs) { @@ -1215,9 +1210,8 @@ struct BPPHandler struct BPPHandlerFunctor : public FairThreadPool::Functor { - BPPHandlerFunctor(boost::shared_ptr r, SBS b) : bs(b) + BPPHandlerFunctor(boost::shared_ptr r, SBS b) : rt(std::move(r)), bs(std::move(b)) { - rt = r; dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); } @@ -1228,10 +1222,10 @@ struct BPPHandler struct LastJoiner : public BPPHandlerFunctor { - LastJoiner(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) + LastJoiner(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(std::move(r), std::move(b)) { } - int operator()() + int operator()() override { utils::setThreadName("PPHandLastJoiner"); return rt->lastJoinerMsg(*bs, dieTime); @@ -1240,10 +1234,10 @@ struct BPPHandler struct Create : public BPPHandlerFunctor { - Create(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) + Create(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(std::move(r), std::move(b)) { } - int operator()() + int operator()() override { utils::setThreadName("PPHandCreate"); rt->createBPP(*bs); @@ -1253,10 +1247,10 @@ struct BPPHandler struct Destroy : public BPPHandlerFunctor { - Destroy(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) + Destroy(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(std::move(r), std::move(b)) { } - int operator()() + int operator()() override { utils::setThreadName("PPHandDestroy"); return rt->destroyBPP(*bs, dieTime); @@ -1265,10 +1259,10 @@ struct BPPHandler struct AddJoiner : public BPPHandlerFunctor { - AddJoiner(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) + AddJoiner(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(std::move(r), std::move(b)) { } - int operator()() + int operator()() override { utils::setThreadName("PPHandAddJoiner"); return rt->addJoinerToBPP(*bs, dieTime); @@ -1280,7 +1274,7 @@ struct BPPHandler Abort(boost::shared_ptr r, SBS b) : BPPHandlerFunctor(r, b) { } - int operator()() + int operator()() override { utils::setThreadName("PPHandAbort"); return rt->doAbort(*bs, dieTime); @@ -1667,12 +1661,12 @@ struct BPPHandler class DictionaryOp : public FairThreadPool::Functor { public: - DictionaryOp(SBS cmd) : bs(cmd) + DictionaryOp(SBS cmd) : bs(std::move(cmd)) { dieTime = posix_time::second_clock::universal_time() + posix_time::seconds(100); } virtual int execute() = 0; - int operator()() + int operator()() override { utils::setThreadName("PPDictOp"); int ret; @@ -1705,7 +1699,7 @@ class CreateEqualityFilter : public DictionaryOp CreateEqualityFilter(SBS cmd) : DictionaryOp(cmd) { } - int execute() + int execute() override { createEqualityFilter(); return 0; @@ -1739,10 +1733,10 @@ class CreateEqualityFilter : public DictionaryOp class DestroyEqualityFilter : public DictionaryOp { public: - DestroyEqualityFilter(SBS cmd) : DictionaryOp(cmd) + DestroyEqualityFilter(SBS cmd) : DictionaryOp(std::move(cmd)) { } - int execute() + int execute() override { return destroyEqualityFilter(); } @@ -2167,9 +2161,7 @@ struct ReadThread mlp->logMessage(logging::M0058, args, false); } - ~ReadThread() - { - } + ~ReadThread() = default; string fServerName; IOSocket fIos; PrimitiveServer* fPrimitiveServerPtr; diff --git a/primitives/primproc/primitiveserver.h b/primitives/primproc/primitiveserver.h index 2d8531d25..40c5f919f 100644 --- a/primitives/primproc/primitiveserver.h +++ b/primitives/primproc/primitiveserver.h @@ -87,16 +87,16 @@ extern BPPMap bppMap; void prefetchBlocks(uint64_t lbid, const int compType, uint32_t* rCount); void prefetchExtent(uint64_t lbid, uint32_t ver, uint32_t txn, uint32_t* rCount); void loadBlock(uint64_t lbid, BRM::QueryContext q, uint32_t txn, int compType, void* bufferPtr, - bool* pWasBlockInCache, uint32_t* rCount = NULL, bool LBIDTrace = false, - uint32_t sessionID = 0, bool doPrefetch = true, VSSCache* vssCache = NULL); + bool* pWasBlockInCache, uint32_t* rCount = nullptr, bool LBIDTrace = false, + uint32_t sessionID = 0, bool doPrefetch = true, VSSCache* vssCache = nullptr); void loadBlockAsync(uint64_t lbid, const BRM::QueryContext& q, uint32_t txn, int CompType, uint32_t* cCount, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID, boost::mutex* m, uint32_t* busyLoaders, boost::shared_ptr sendThread, - VSSCache* vssCache = 0); + VSSCache* vssCache = nullptr); uint32_t loadBlocks(BRM::LBID_t* lbids, BRM::QueryContext q, BRM::VER_t txn, int compType, uint8_t** bufferPtrs, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID, uint32_t blockCount, bool* wasVersioned, bool doPrefetch = true, - VSSCache* vssCache = NULL); + VSSCache* vssCache = nullptr); uint32_t cacheNum(uint64_t lbid); void buildFileName(BRM::OID_t oid, char* fileName); diff --git a/primitives/primproc/primitiveserverthreadpools.h b/primitives/primproc/primitiveserverthreadpools.h index db23d290c..fffdcf9a0 100644 --- a/primitives/primproc/primitiveserverthreadpools.h +++ b/primitives/primproc/primitiveserverthreadpools.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "fair_threadpool.h" class PrimitiveServerThreadPools @@ -25,7 +26,7 @@ class PrimitiveServerThreadPools public: PrimitiveServerThreadPools() = default; PrimitiveServerThreadPools(boost::shared_ptr primServerThreadPool) - : fPrimServerThreadPool(primServerThreadPool) + : fPrimServerThreadPool(std::move(primServerThreadPool)) { } diff --git a/primitives/primproc/primproc.h b/primitives/primproc/primproc.h index 4fb757c1f..c82f32b3a 100644 --- a/primitives/primproc/primproc.h +++ b/primitives/primproc/primproc.h @@ -155,12 +155,12 @@ class ServicePrimProc : public Service, public Opt void LogErrno() override { - cerr << strerror(errno) << endl; + std::cerr << strerror(errno) << std::endl; } void ParentLogChildMessage(const std::string& str) override { - cout << str << endl; + std::cout << str << std::endl; } int Child() override; int Run() diff --git a/primitives/primproc/pseudocc.h b/primitives/primproc/pseudocc.h index b7e220aa9..e95fcf3b0 100644 --- a/primitives/primproc/pseudocc.h +++ b/primitives/primproc/pseudocc.h @@ -27,14 +27,14 @@ class PseudoCC : public ColumnCommand { public: PseudoCC(); - virtual ~PseudoCC(); + ~PseudoCC() override; - virtual SCommand duplicate(); - virtual void createCommand(messageqcpp::ByteStream&); - virtual void resetCommand(messageqcpp::ByteStream&); + SCommand duplicate() override; + void createCommand(messageqcpp::ByteStream&) override; + void resetCommand(messageqcpp::ByteStream&) override; protected: - virtual void loadData(); + void loadData() override; private: template @@ -102,7 +102,7 @@ template void PseudoCC::loadSegmentNum() { uint16_t segNum; - rowgroup::getLocationFromRid(bpp->baseRid, NULL, &segNum, NULL, NULL); + rowgroup::getLocationFromRid(bpp->baseRid, nullptr, &segNum, nullptr, nullptr); loadSingleValue(segNum); } @@ -110,7 +110,7 @@ template void PseudoCC::loadPartitionNum() { uint32_t partNum; - rowgroup::getLocationFromRid(bpp->baseRid, &partNum, NULL, NULL, NULL); + rowgroup::getLocationFromRid(bpp->baseRid, &partNum, nullptr, nullptr, nullptr); loadSingleValue(partNum); } diff --git a/primitives/primproc/rssmonfcn.h b/primitives/primproc/rssmonfcn.h index 341038e91..d43a29aa6 100644 --- a/primitives/primproc/rssmonfcn.h +++ b/primitives/primproc/rssmonfcn.h @@ -30,7 +30,7 @@ class RssMonFcn : public utils::MonitorProcMem { } - void operator()() const; + void operator()() const override; }; -} // namespace \ No newline at end of file +} // namespace exemgr \ No newline at end of file diff --git a/primitives/primproc/rtscommand.h b/primitives/primproc/rtscommand.h index 9ce53b5eb..e5c594a27 100644 --- a/primitives/primproc/rtscommand.h +++ b/primitives/primproc/rtscommand.h @@ -40,23 +40,23 @@ class RTSCommand : public Command { public: RTSCommand(); - virtual ~RTSCommand(); + ~RTSCommand() override; - void execute(); - void project(); - void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col); - uint64_t getLBID(); - void nextLBID(); - void createCommand(messageqcpp::ByteStream&); - void resetCommand(messageqcpp::ByteStream&); - SCommand duplicate(); + void execute() override; + void project() override; + void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col) override; + uint64_t getLBID() override; + void nextLBID() override; + void createCommand(messageqcpp::ByteStream&) override; + void resetCommand(messageqcpp::ByteStream&) override; + SCommand duplicate() override; bool operator==(const RTSCommand&) const; bool operator!=(const RTSCommand&) const; - void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*); + void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*) override; /* Put bootstrap code here (ie, build the template primitive msg) */ - void prep(int8_t outputType, bool makeAbsRids); + void prep(int8_t outputType, bool makeAbsRids) override; uint8_t isPassThru() { return passThru; @@ -65,10 +65,10 @@ class RTSCommand : public Command { absNull = a; } - void getLBIDList(uint32_t loopCount, std::vector* lbids); + void getLBIDList(uint32_t loopCount, std::vector* lbids) override; // TODO: do we need to reference either col or dict? - int getCompType() const + int getCompType() const override { return dict.getCompType(); } diff --git a/primitives/primproc/serviceexemgr.h b/primitives/primproc/serviceexemgr.h index d605d557d..5be609593 100644 --- a/primitives/primproc/serviceexemgr.h +++ b/primitives/primproc/serviceexemgr.h @@ -62,320 +62,324 @@ namespace exemgr { - using SharedPtrEMSock = boost::shared_ptr; - class Opt +using SharedPtrEMSock = boost::shared_ptr; +class Opt +{ + public: + int m_debug; + bool m_e; + bool m_fg; + Opt() : m_debug(0), m_e(false), m_fg(false){}; + Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) { - public: - int m_debug; - bool m_e; - bool m_fg; - Opt() : m_debug(0), m_e(false), m_fg(false) {}; - Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) + int c; + while ((c = getopt(argc, argv, "edf")) != EOF) { - int c; - while ((c = getopt(argc, argv, "edf")) != EOF) + switch (c) { - switch (c) - { - case 'd': m_debug++; break; + case 'd': m_debug++; break; - case 'e': m_e = true; break; + case 'e': m_e = true; break; - case 'f': m_fg = true; break; + case 'f': m_fg = true; break; - case '?': - default: break; - } + case '?': + default: break; } } - int getDebugLevel() const - { - return m_debug; - } - }; - - class ServiceExeMgr : public Service, public Opt + } + int getDebugLevel() const { - using SessionMemMap_t = std::map; - using ThreadCntPerSessionMap_t = std::map; - protected: - void log(logging::LOG_TYPE type, const std::string& str) - { - logging::LoggingID logid(16); - logging::Message::Args args; - logging::Message message(8); - args.add(strerror(errno)); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(type, message, logid); - } + return m_debug; + } +}; - public: - ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) +class ServiceExeMgr : public Service, public Opt +{ + using SessionMemMap_t = std::map; + using ThreadCntPerSessionMap_t = std::map; + + protected: + void log(logging::LOG_TYPE type, const std::string& str) + { + logging::LoggingID logid(16); + logging::Message::Args args; + logging::Message message(8); + args.add(strerror(errno)); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(type, message, logid); + } + + public: + ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) + { + bool runningWithExeMgr = true; + rm_ = joblist::ResourceManager::instance(runningWithExeMgr); + } + ServiceExeMgr(const Opt& opt, config::Config* aConfig) + : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) + { + bool runningWithExeMgr = true; + rm_ = joblist::ResourceManager::instance(runningWithExeMgr, aConfig); + } + void LogErrno() override + { + log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); + } + void ParentLogChildMessage(const std::string& str) override + { + log(logging::LOG_TYPE_INFO, str); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } + static const constexpr unsigned logDefaultMsg = logging::M0000; + static const constexpr unsigned logDbProfStartStatement = logging::M0028; + static const constexpr unsigned logDbProfEndStatement = logging::M0029; + static const constexpr unsigned logStartSql = logging::M0041; + static const constexpr unsigned logEndSql = logging::M0042; + static const constexpr unsigned logRssTooBig = logging::M0044; + static const constexpr unsigned logDbProfQueryStats = logging::M0047; + static const constexpr unsigned logExeMgrExcpt = logging::M0055; + // If any flags other than the table mode flags are set, produce output to screeen + static const constexpr uint32_t flagsWantOutput = + (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & + ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); + logging::Logger& getLogger() + { + return msgLog_; + } + void updateSessionMap(const size_t pct) + { + std::lock_guard lk(sessionMemMapMutex_); + + for (auto mapIter = sessionMemMap_.begin(); mapIter != sessionMemMap_.end(); ++mapIter) { - bool runningWithExeMgr = true; - rm_ = joblist::ResourceManager::instance(runningWithExeMgr); + if (pct > mapIter->second) + { + mapIter->second = pct; + } } - ServiceExeMgr(const Opt& opt, config::Config* aConfig) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) - { - bool runningWithExeMgr = true; - rm_ = joblist::ResourceManager::instance(runningWithExeMgr, aConfig); - } - void LogErrno() override - { - log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); - } - void ParentLogChildMessage(const std::string& str) override - { - log(logging::LOG_TYPE_INFO, str); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } - static const constexpr unsigned logDefaultMsg = logging::M0000; - static const constexpr unsigned logDbProfStartStatement = logging::M0028; - static const constexpr unsigned logDbProfEndStatement = logging::M0029; - static const constexpr unsigned logStartSql = logging::M0041; - static const constexpr unsigned logEndSql = logging::M0042; - static const constexpr unsigned logRssTooBig = logging::M0044; - static const constexpr unsigned logDbProfQueryStats = logging::M0047; - static const constexpr unsigned logExeMgrExcpt = logging::M0055; - // If any flags other than the table mode flags are set, produce output to screeen - static const constexpr uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & - ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); - logging::Logger& getLogger() - { - return msgLog_; - } - void updateSessionMap(const size_t pct) + } + ThreadCntPerSessionMap_t& getThreadCntPerSessionMap() + { + return threadCntPerSessionMap_; + } + std::mutex& getThreadCntPerSessionMapMutex() + { + return threadCntPerSessionMapMutex_; + } + void initMaxMemPct(uint32_t sessionId) + { + if (sessionId < 0x80000000) { std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); - for (auto mapIter = sessionMemMap_.begin(); mapIter != sessionMemMap_.end(); ++mapIter) + if (mapIter == sessionMemMap_.end()) { - if (pct > mapIter->second) - { - mapIter->second = pct; - } + sessionMemMap_[sessionId] = 0; } - } - ThreadCntPerSessionMap_t& getThreadCntPerSessionMap() - { - return threadCntPerSessionMap_; - } - std::mutex& getThreadCntPerSessionMapMutex() - { - return threadCntPerSessionMapMutex_; - } - void initMaxMemPct(uint32_t sessionId) - { - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex_); - auto mapIter = sessionMemMap_.find(sessionId); - - if (mapIter == sessionMemMap_.end()) - { - sessionMemMap_[sessionId] = 0; - } - else - { - mapIter->second = 0; - } - } - } - uint64_t getMaxMemPct(const uint32_t sessionId) - { - uint64_t maxMemoryPct = 0; - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex_); - auto mapIter = sessionMemMap_.find(sessionId); - - if (mapIter != sessionMemMap_.end()) - { - maxMemoryPct = (uint64_t)mapIter->second; - } - } - - return maxMemoryPct; - } - void deleteMaxMemPct(uint32_t sessionId) - { - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex_); - auto mapIter = sessionMemMap_.find(sessionId); - - if (mapIter != sessionMemMap_.end()) - { - sessionMemMap_.erase(sessionId); - } - } - } - //...Increment the number of threads using the specified sessionId - void incThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex_); - auto mapIter = threadCntPerSessionMap_.find(sessionId); - - if (mapIter == threadCntPerSessionMap_.end()) - threadCntPerSessionMap_.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); else - mapIter->second++; - } - //...Decrement the number of threads using the specified sessionId. - //...When the thread count for a sessionId reaches 0, the corresponding - //...CalpontSystemCatalog objects are deleted. - //...The user query and its associated catalog query have a different - //...session Id where the highest bit is flipped. - //...The object with id(sessionId | 0x80000000) cannot be removed before - //...user query session completes because the sysdata may be used for - //...debugging/stats purpose, such as result graph, etc. - void decThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex_); - auto mapIter = threadCntPerSessionMap_.find(sessionId); - - if (mapIter != threadCntPerSessionMap_.end()) { - if (--mapIter->second == 0) + mapIter->second = 0; + } + } + } + uint64_t getMaxMemPct(const uint32_t sessionId) + { + uint64_t maxMemoryPct = 0; + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter != sessionMemMap_.end()) + { + maxMemoryPct = (uint64_t)mapIter->second; + } + } + + return maxMemoryPct; + } + void deleteMaxMemPct(uint32_t sessionId) + { + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter != sessionMemMap_.end()) + { + sessionMemMap_.erase(sessionId); + } + } + } + //...Increment the number of threads using the specified sessionId + void incThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex_); + auto mapIter = threadCntPerSessionMap_.find(sessionId); + + if (mapIter == threadCntPerSessionMap_.end()) + threadCntPerSessionMap_.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); + else + mapIter->second++; + } + //...Decrement the number of threads using the specified sessionId. + //...When the thread count for a sessionId reaches 0, the corresponding + //...CalpontSystemCatalog objects are deleted. + //...The user query and its associated catalog query have a different + //...session Id where the highest bit is flipped. + //...The object with id(sessionId | 0x80000000) cannot be removed before + //...user query session completes because the sysdata may be used for + //...debugging/stats purpose, such as result graph, etc. + void decThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex_); + auto mapIter = threadCntPerSessionMap_.find(sessionId); + + if (mapIter != threadCntPerSessionMap_.end()) + { + if (--mapIter->second == 0) + { + threadCntPerSessionMap_.erase(mapIter); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); + } + } + } + ActiveStatementCounter* getStatementsRunningCount() + { + return statementsRunningCount_; + } + joblist::DistributedEngineComm* getDec() + { + return dec_; + } + int toInt(const std::string& val) + { + if (val.length() == 0) + return -1; + + return static_cast(config::Config::fromText(val)); + } + const std::string prettyPrintMiniInfo(const std::string& in); + + const std::string timeNow() + { + time_t outputTime = time(nullptr); + struct tm ltm; + char buf[32]; // ctime(3) says at least 26 + size_t len = 0; + asctime_r(localtime_r(&outputTime, <m), buf); + len = strlen(buf); + + if (len > 0) + --len; + + if (buf[len] == '\n') + buf[len] = 0; + + return buf; + } + querytele::QueryTeleServerParms& getTeleServerParms() + { + return teleServerParms_; + } + joblist::ResourceManager& getRm() + { + return *rm_; + } + bool isLocalNodeSock(SharedPtrEMSock& sock) const + { + for (auto& sin : localNetIfaceSins_) + { + if (sock->isSameAddr(sin)) + { + return true; + } + } + return false; + } + + private: + void setupSignalHandlers(); + int8_t setupCwd() + { + std::string workdir = rm_->getScWorkingDir(); + int8_t rc = chdir(workdir.c_str()); + + if (rc < 0 || access(".", W_OK) != 0) + rc = chdir("/tmp"); + + return (rc < 0) ? -5 : rc; + } + int setupResources() + { + struct rlimit rlim; + + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -1; + } + rlim.rlim_cur = rlim.rlim_max = 65536; + + if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -2; + } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -3; + } + if (rlim.rlim_cur != 65536) + { + return -4; + } + return 0; + } + void getLocalNetIfacesSins() + { + string ipAddress = "Unable to get IP Address"; + struct ifaddrs* netIfacesList = nullptr; + struct ifaddrs* ifaceListMembPtr = nullptr; + int success = 0; + // retrieve the current interfaces - returns 0 on success + success = getifaddrs(&netIfacesList); + if (success == 0) + { + ifaceListMembPtr = netIfacesList; + for (; ifaceListMembPtr; ifaceListMembPtr = ifaceListMembPtr->ifa_next) + { + if (ifaceListMembPtr->ifa_addr->sa_family == AF_INET) { - threadCntPerSessionMap_.erase(mapIter); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); + localNetIfaceSins_.push_back(((struct sockaddr_in*)ifaceListMembPtr->ifa_addr)->sin_addr); } } } - ActiveStatementCounter* getStatementsRunningCount() - { - return statementsRunningCount_; - } - joblist::DistributedEngineComm* getDec() - { - return dec_; - } - int toInt(const std::string& val) - { - if (val.length() == 0) - return -1; - - return static_cast(config::Config::fromText(val)); - } - const std::string prettyPrintMiniInfo(const std::string& in); - - const std::string timeNow() - { - time_t outputTime = time(0); - struct tm ltm; - char buf[32]; // ctime(3) says at least 26 - size_t len = 0; - asctime_r(localtime_r(&outputTime, <m), buf); - len = strlen(buf); - - if (len > 0) - --len; - - if (buf[len] == '\n') - buf[len] = 0; - - return buf; - } - querytele::QueryTeleServerParms& getTeleServerParms() - { - return teleServerParms_; - } - joblist::ResourceManager& getRm() - { - return *rm_; - } - bool isLocalNodeSock(SharedPtrEMSock& sock) const - { - for (auto& sin : localNetIfaceSins_) - { - if (sock->isSameAddr(sin)) - { - return true; - } - } - return false; - } - private: - void setupSignalHandlers(); - int8_t setupCwd() - { - std::string workdir = rm_->getScWorkingDir(); - int8_t rc = chdir(workdir.c_str()); - - if (rc < 0 || access(".", W_OK) != 0) - rc = chdir("/tmp"); - - return (rc < 0) ? -5 : rc; - } - int setupResources() - { - struct rlimit rlim; - - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -1; - } - rlim.rlim_cur = rlim.rlim_max = 65536; - - if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -2; - } - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -3; - } - if (rlim.rlim_cur != 65536) - { - return -4; - } - return 0; - } - void getLocalNetIfacesSins() - { - string ipAddress = "Unable to get IP Address"; - struct ifaddrs* netIfacesList = nullptr; - struct ifaddrs* ifaceListMembPtr = nullptr; - int success = 0; - // retrieve the current interfaces - returns 0 on success - success = getifaddrs(&netIfacesList); - if (success == 0) - { - ifaceListMembPtr = netIfacesList; - for (; ifaceListMembPtr; ifaceListMembPtr = ifaceListMembPtr->ifa_next) - { - if (ifaceListMembPtr->ifa_addr->sa_family == AF_INET) - { - localNetIfaceSins_.push_back(((struct sockaddr_in*)ifaceListMembPtr->ifa_addr)->sin_addr); - } - } - } - freeifaddrs(netIfacesList); - } - logging::Logger msgLog_; - SessionMemMap_t sessionMemMap_; // track memory% usage during a query - std::mutex sessionMemMapMutex_; - //...The FrontEnd may establish more than 1 connection (which results in - // more than 1 ExeMgr thread) per session. These threads will share - // the same CalpontSystemCatalog object for that session. Here, we - // define a std::map to track how many threads are sharing each session, so - // that we know when we can safely delete a CalpontSystemCatalog object - // shared by multiple threads per session. - ThreadCntPerSessionMap_t threadCntPerSessionMap_; - std::mutex threadCntPerSessionMapMutex_; - ActiveStatementCounter* statementsRunningCount_ = nullptr; - joblist::DistributedEngineComm* dec_ = nullptr; - joblist::ResourceManager* rm_ = nullptr; - // Its attributes are set in Child() - querytele::QueryTeleServerParms teleServerParms_; - std::vector localNetIfaceSins_; - }; - extern ServiceExeMgr* globServiceExeMgr; -} \ No newline at end of file + freeifaddrs(netIfacesList); + } + logging::Logger msgLog_; + SessionMemMap_t sessionMemMap_; // track memory% usage during a query + std::mutex sessionMemMapMutex_; + //...The FrontEnd may establish more than 1 connection (which results in + // more than 1 ExeMgr thread) per session. These threads will share + // the same CalpontSystemCatalog object for that session. Here, we + // define a std::map to track how many threads are sharing each session, so + // that we know when we can safely delete a CalpontSystemCatalog object + // shared by multiple threads per session. + ThreadCntPerSessionMap_t threadCntPerSessionMap_; + std::mutex threadCntPerSessionMapMutex_; + ActiveStatementCounter* statementsRunningCount_ = nullptr; + joblist::DistributedEngineComm* dec_ = nullptr; + joblist::ResourceManager* rm_ = nullptr; + // Its attributes are set in Child() + querytele::QueryTeleServerParms teleServerParms_; + std::vector localNetIfaceSins_; +}; +extern ServiceExeMgr* globServiceExeMgr; +} // namespace exemgr \ No newline at end of file diff --git a/primitives/primproc/umsocketselector.h b/primitives/primproc/umsocketselector.h index 63b042b73..2dd49cbe6 100644 --- a/primitives/primproc/umsocketselector.h +++ b/primitives/primproc/umsocketselector.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -80,7 +81,7 @@ class UmSocketSelector /** @brief UmSocketSelector destructor * */ - ~UmSocketSelector(){}; + ~UmSocketSelector() = default; /** @brief Accessor to total number of UM IP's in Columnstore.xml. * @@ -152,15 +153,15 @@ class UmModuleIPs * * @param moduleName (in) UM module name for this UmModuleIPs object. */ - explicit UmModuleIPs(const std::string& moduleName) - : fUmModuleName(moduleName), fNextUmIPSocketIdx(NEXT_IP_SOCKET_UNASSIGNED) + explicit UmModuleIPs(std::string moduleName) + : fUmModuleName(std::move(moduleName)), fNextUmIPSocketIdx(NEXT_IP_SOCKET_UNASSIGNED) { } /** @brief UmModuleIPs destructor. * */ - ~UmModuleIPs(){}; + ~UmModuleIPs() = default; /** @brief Accessor to number of IP's from Columnstore.xml for this UM. * @@ -258,15 +259,13 @@ class UmIPSocketConns /** @brief UmIPSocketConns destructor. * */ - ~UmIPSocketConns() - { - } + ~UmIPSocketConns() = default; /** @brief Accessor to the IP address for this UmIPSocketConns object. * * @return IP address (in network byte order). */ - in_addr_t ipAddress() + in_addr_t ipAddress() const { return fIpAddress; } diff --git a/storage-manager/src/AppendTask.cpp b/storage-manager/src/AppendTask.cpp index d30211524..19b5a1832 100644 --- a/storage-manager/src/AppendTask.cpp +++ b/storage-manager/src/AppendTask.cpp @@ -18,7 +18,7 @@ #include "AppendTask.h" #include "messageFormat.h" #include "SMLogging.h" -#include +#include using namespace std; @@ -50,7 +50,7 @@ bool AppendTask::run() success = read(cmdbuf, sizeof(append_cmd)); check_error("AppendTask read", false); - append_cmd* cmd = (append_cmd*)cmdbuf; + auto* cmd = (append_cmd*)cmdbuf; if (cmd->flen > 1023 - sizeof(*cmd)) { diff --git a/storage-manager/src/AppendTask.h b/storage-manager/src/AppendTask.h index fa027015b..ac34ab56e 100644 --- a/storage-manager/src/AppendTask.h +++ b/storage-manager/src/AppendTask.h @@ -24,13 +24,11 @@ namespace storagemanager class AppendTask : public PosixTask { public: + AppendTask() = delete; AppendTask(int sock, uint length); - virtual ~AppendTask(); + ~AppendTask() override; - bool run(); - - private: - AppendTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/Cache.h b/storage-manager/src/Cache.h index 01d8b54d8..305465b10 100644 --- a/storage-manager/src/Cache.h +++ b/storage-manager/src/Cache.h @@ -46,7 +46,7 @@ class Cache : public boost::noncopyable, public ConfigListener { public: static Cache* get(); - virtual ~Cache(); + ~Cache() override; // reading fcns // read() marks objects to be read s.t. they do not get flushed. @@ -98,7 +98,7 @@ class Cache : public boost::noncopyable, public ConfigListener void reset(); void validateCacheSize(); - virtual void configListener() override; + void configListener() override; private: Cache(); diff --git a/storage-manager/src/CloudStorage.h b/storage-manager/src/CloudStorage.h index c857f1b67..97d18a210 100644 --- a/storage-manager/src/CloudStorage.h +++ b/storage-manager/src/CloudStorage.h @@ -26,11 +26,12 @@ namespace storagemanager class CloudStorage { public: - virtual ~CloudStorage(){}; + virtual ~CloudStorage() = default; /* These behave like syscalls. return code -1 means an error, and errno is set */ - virtual int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL) = 0; + virtual int getObject(const std::string& sourceKey, const std::string& destFile, + size_t* size = nullptr) = 0; virtual int getObject(const std::string& sourceKey, std::shared_ptr* data, - size_t* size = NULL) = 0; + size_t* size = nullptr) = 0; virtual int putObject(const std::string& sourceFile, const std::string& destKey) = 0; virtual int putObject(const std::shared_ptr data, size_t len, const std::string& destKey) = 0; virtual int deleteObject(const std::string& key) = 0; diff --git a/storage-manager/src/Config.h b/storage-manager/src/Config.h index b7e9a75b1..ba8e85bca 100644 --- a/storage-manager/src/Config.h +++ b/storage-manager/src/Config.h @@ -18,14 +18,14 @@ #pragma once #ifndef __clang__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif #include #ifndef __clang__ - #pragma GCC diagnostic pop +#pragma GCC diagnostic pop #endif #include @@ -41,7 +41,7 @@ namespace storagemanager class ConfigListener { public: - virtual ~ConfigListener(){}; + virtual ~ConfigListener() = default; virtual void configListener() = 0; }; @@ -61,7 +61,7 @@ class Config : public boost::noncopyable private: Config(); - Config(const std::string&); + explicit Config(const std::string&); bool reload(); void reloadThreadFcn(); diff --git a/storage-manager/src/CopyTask.cpp b/storage-manager/src/CopyTask.cpp index 7d6fe980e..4694c9834 100644 --- a/storage-manager/src/CopyTask.cpp +++ b/storage-manager/src/CopyTask.cpp @@ -54,10 +54,10 @@ bool CopyTask::run() success = read(buf, getLength()); check_error("CopyTask read", false); - copy_cmd* cmd = (copy_cmd*)buf; + auto* cmd = (copy_cmd*)buf; string filename1(cmd->file1.filename, cmd->file1.flen); // need to copy this in case it's not null terminated - f_name* filename2 = (f_name*)&buf[sizeof(copy_cmd) + cmd->file1.flen]; + auto* filename2 = (f_name*)&buf[sizeof(copy_cmd) + cmd->file1.flen]; #ifdef SM_TRACE logger->log(LOG_DEBUG, "copy %s to %s.", filename1.c_str(), filename2->filename); @@ -80,7 +80,7 @@ bool CopyTask::run() return true; } - sm_response* resp = (sm_response*)buf; + auto* resp = (sm_response*)buf; resp->returnCode = 0; return write(*resp, 0); } diff --git a/storage-manager/src/CopyTask.h b/storage-manager/src/CopyTask.h index 0745c8064..bbd2240e8 100644 --- a/storage-manager/src/CopyTask.h +++ b/storage-manager/src/CopyTask.h @@ -24,13 +24,11 @@ namespace storagemanager class CopyTask : public PosixTask { public: + CopyTask() = delete; CopyTask(int sock, uint length); - virtual ~CopyTask(); + ~CopyTask() override; - bool run(); - - private: - CopyTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/Downloader.cpp b/storage-manager/src/Downloader.cpp index 1a0897f9f..4e914ce61 100644 --- a/storage-manager/src/Downloader.cpp +++ b/storage-manager/src/Downloader.cpp @@ -19,9 +19,10 @@ #include "Config.h" #include "SMLogging.h" #include -#include +#include #include #include +#include using namespace std; namespace bf = boost::filesystem; @@ -135,14 +136,20 @@ const bf::path& Downloader::getTmpPath() const return tmpPath; } /* The helper fcns */ -Downloader::Download::Download(const string& source, const bf::path& _dlPath, boost::mutex* _lock, - Downloader* _dl) - : dlPath(_dlPath), key(source), dl_errno(0), size(0), lock(_lock), finished(false), itRan(false), dl(_dl) +Downloader::Download::Download(string source, bf::path _dlPath, boost::mutex* _lock, Downloader* _dl) + : dlPath(std::move(_dlPath)) + , key(std::move(source)) + , dl_errno(0) + , size(0) + , lock(_lock) + , finished(false) + , itRan(false) + , dl(_dl) { } -Downloader::Download::Download(const string& source) - : key(source), dl_errno(0), size(0), lock(NULL), finished(false), itRan(false), dl(NULL) +Downloader::Download::Download(string source) + : key(std::move(source)), dl_errno(0), size(0), lock(nullptr), finished(false), itRan(false), dl(nullptr) { } @@ -154,12 +161,12 @@ Downloader::Download::~Download() void Downloader::Download::operator()() { itRan = true; - CloudStorage* storage = CloudStorage::get(); + CloudStorage* stor = CloudStorage::get(); // download to a tmp path if (!bf::exists(dlPath / dl->getTmpPath())) bf::create_directories(dlPath / dl->getTmpPath()); bf::path tmpFile = dlPath / dl->getTmpPath() / key; - int err = storage->getObject(key, tmpFile.string(), &size); + int err = stor->getObject(key, tmpFile.string(), &size); if (err != 0) { dl_errno = errno; @@ -179,8 +186,10 @@ void Downloader::Download::operator()() lock->lock(); finished = true; - for (uint i = 0; i < listeners.size(); i++) - listeners[i]->downloadFinished(); + for (auto* listener : listeners) + { + listener->downloadFinished(); + } lock->unlock(); } @@ -217,7 +226,7 @@ void Downloader::configListener() { maxDownloads = 20; workers.setMaxThreads(maxDownloads); - logger->log(LOG_INFO, "max_concurrent_downloads = %u",maxDownloads); + logger->log(LOG_INFO, "max_concurrent_downloads = %u", maxDownloads); } if (stmp.empty()) { diff --git a/storage-manager/src/Downloader.h b/storage-manager/src/Downloader.h index d4919c6f2..1208ab345 100644 --- a/storage-manager/src/Downloader.h +++ b/storage-manager/src/Downloader.h @@ -38,7 +38,7 @@ class Downloader : public ConfigListener { public: Downloader(); - virtual ~Downloader(); + ~Downloader() override; // caller owns the memory for the strings. // errors are reported through errnos @@ -49,7 +49,7 @@ class Downloader : public ConfigListener void printKPIs() const; - virtual void configListener() override; + void configListener() override; private: uint maxDownloads; @@ -73,10 +73,10 @@ class Downloader : public ConfigListener */ struct Download : public ThreadPool::Job { - Download(const std::string& source, const boost::filesystem::path& _dlPath, boost::mutex*, Downloader*); - Download(const std::string& source); - virtual ~Download(); - void operator()(); + Download(std::string source, boost::filesystem::path _dlPath, boost::mutex*, Downloader*); + explicit Download(std::string source); + ~Download() override; + void operator()() override; boost::filesystem::path dlPath; const std::string key; int dl_errno; // to propagate errors from the download job to the caller diff --git a/storage-manager/src/IOCoordinator.h b/storage-manager/src/IOCoordinator.h index f499aad3e..131c79711 100644 --- a/storage-manager/src/IOCoordinator.h +++ b/storage-manager/src/IOCoordinator.h @@ -18,7 +18,7 @@ #pragma once #include -#include +#include #include #include #include @@ -58,7 +58,7 @@ class IOCoordinator : public boost::noncopyable // The shared logic for merging a journal file with its base file. // len should be set to the length of the data requested std::shared_ptr mergeJournal(const char* objectPath, const char* journalPath, off_t offset, - size_t len, size_t* sizeRead) const; + size_t len, size_t* sizeRead) const; // this version modifies object data in memory, given the journal filename. Processes the whole object // and whole journal file. diff --git a/storage-manager/src/ListDirectoryTask.h b/storage-manager/src/ListDirectoryTask.h index 183c6dad5..feea7c2ff 100644 --- a/storage-manager/src/ListDirectoryTask.h +++ b/storage-manager/src/ListDirectoryTask.h @@ -25,14 +25,13 @@ namespace storagemanager class ListDirectoryTask : public PosixTask { public: + ListDirectoryTask() = delete; ListDirectoryTask(int sock, uint length); - virtual ~ListDirectoryTask(); + ~ListDirectoryTask() override; - bool run(); + bool run() override; private: - ListDirectoryTask(); - bool writeString(uint8_t* buf, int* offset, int size, const std::string& str); }; diff --git a/storage-manager/src/LocalStorage.h b/storage-manager/src/LocalStorage.h index fc1f5a5d3..178efc76b 100644 --- a/storage-manager/src/LocalStorage.h +++ b/storage-manager/src/LocalStorage.h @@ -29,18 +29,19 @@ class LocalStorage : public CloudStorage { public: LocalStorage(); - virtual ~LocalStorage(); + ~LocalStorage() override; - int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL); - int getObject(const std::string& sourceKey, std::shared_ptr* data, size_t* size = NULL); - int putObject(const std::string& sourceFile, const std::string& destKey); - int putObject(const std::shared_ptr data, size_t len, const std::string& destKey); - int deleteObject(const std::string& key); - int copyObject(const std::string& sourceKey, const std::string& destKey); - int exists(const std::string& key, bool* out); + int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = nullptr) override; + int getObject(const std::string& sourceKey, std::shared_ptr* data, + size_t* size = nullptr) override; + int putObject(const std::string& sourceFile, const std::string& destKey) override; + int putObject(const std::shared_ptr data, size_t len, const std::string& destKey) override; + int deleteObject(const std::string& key) override; + int copyObject(const std::string& sourceKey, const std::string& destKey) override; + int exists(const std::string& key, bool* out) override; const boost::filesystem::path& getPrefix() const; - void printKPIs() const; + void printKPIs() const override; protected: size_t bytesRead, bytesWritten; diff --git a/storage-manager/src/MetadataFile.h b/storage-manager/src/MetadataFile.h index f8115ae09..c30092df1 100644 --- a/storage-manager/src/MetadataFile.h +++ b/storage-manager/src/MetadataFile.h @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include @@ -33,7 +33,7 @@ namespace storagemanager struct metadataObject { metadataObject(); - metadataObject(uint64_t offset); // so we can search mObjects by integer + explicit metadataObject(uint64_t offset); // so we can search mObjects by integer metadataObject(uint64_t offset, uint64_t length, const std::string& key); uint64_t offset; mutable uint64_t length; @@ -51,7 +51,7 @@ class MetadataFile { }; MetadataFile(); - MetadataFile(const boost::filesystem::path& filename); + explicit MetadataFile(const boost::filesystem::path& filename); MetadataFile(const boost::filesystem::path& path, no_create_t, bool appendExt); // this one won't create it if it doesn't exist diff --git a/storage-manager/src/OpenTask.cpp b/storage-manager/src/OpenTask.cpp index 267d253ee..983a1c8e3 100644 --- a/storage-manager/src/OpenTask.cpp +++ b/storage-manager/src/OpenTask.cpp @@ -19,8 +19,7 @@ #include "messageFormat.h" #include "SMLogging.h" #include -#include -#include +#include using namespace std; @@ -30,9 +29,7 @@ OpenTask::OpenTask(int sock, uint len) : PosixTask(sock, len) { } -OpenTask::~OpenTask() -{ -} +OpenTask::~OpenTask() = default; bool OpenTask::run() { @@ -58,12 +55,12 @@ bool OpenTask::run() return false; } - open_cmd* cmd = (open_cmd*)buf; + auto* cmd = (open_cmd*)buf; #ifdef SM_TRACE logger->log(LOG_DEBUG, "open filename %s mode %o.", cmd->filename, cmd->openmode); #endif - sm_response* resp = (sm_response*)buf; + auto* resp = (sm_response*)buf; int err; try diff --git a/storage-manager/src/OpenTask.h b/storage-manager/src/OpenTask.h index 658d8e7ec..90403e831 100644 --- a/storage-manager/src/OpenTask.h +++ b/storage-manager/src/OpenTask.h @@ -24,13 +24,11 @@ namespace storagemanager class OpenTask : public PosixTask { public: + OpenTask() = delete; OpenTask(int sock, uint length); - virtual ~OpenTask(); + ~OpenTask() override; - bool run(); - - private: - OpenTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/Ownership.h b/storage-manager/src/Ownership.h index f3b6a2d97..1a229a420 100644 --- a/storage-manager/src/Ownership.h +++ b/storage-manager/src/Ownership.h @@ -62,7 +62,7 @@ class Ownership : public boost::noncopyable struct Monitor { - Monitor(Ownership*); + explicit Monitor(Ownership*); ~Monitor(); boost::thread thread; Ownership* owner; diff --git a/storage-manager/src/PingTask.cpp b/storage-manager/src/PingTask.cpp index 8451b7757..82018340e 100644 --- a/storage-manager/src/PingTask.cpp +++ b/storage-manager/src/PingTask.cpp @@ -17,7 +17,7 @@ #include "PingTask.h" #include "messageFormat.h" -#include +#include namespace storagemanager { @@ -25,9 +25,7 @@ PingTask::PingTask(int sock, uint len) : PosixTask(sock, len) { } -PingTask::~PingTask() -{ -} +PingTask::~PingTask() = default; bool PingTask::run() { @@ -49,7 +47,7 @@ bool PingTask::run() } // send generic success response - sm_response ret; + sm_response ret{}; ret.returnCode = 0; success = write(ret, 0); return success; diff --git a/storage-manager/src/PingTask.h b/storage-manager/src/PingTask.h index b9855e61d..aad7c84a4 100644 --- a/storage-manager/src/PingTask.h +++ b/storage-manager/src/PingTask.h @@ -24,13 +24,11 @@ namespace storagemanager class PingTask : public PosixTask { public: + PingTask() = delete; PingTask(int sock, uint length); - virtual ~PingTask(); + ~PingTask() override; - bool run(); - - private: - PingTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/PosixTask.cpp b/storage-manager/src/PosixTask.cpp index 2bd4dc790..f236653f6 100644 --- a/storage-manager/src/PosixTask.cpp +++ b/storage-manager/src/PosixTask.cpp @@ -18,10 +18,9 @@ #include "PosixTask.h" #include "messageFormat.h" #include "SMLogging.h" -#include #include #include -#include +#include #define min(x, y) (x < y ? x : y) @@ -53,7 +52,7 @@ void PosixTask::handleError(const char* name, int errCode) char buf[80]; // send an error response if possible - sm_response* resp = (sm_response*)buf; + auto* resp = (sm_response*)buf; resp->returnCode = -1; *((int*)resp->payload) = errCode; bool success = write(*resp, 4); @@ -67,7 +66,7 @@ uint PosixTask::getRemainingLength() return remainingLengthForCaller; } -uint PosixTask::getLength() +uint PosixTask::getLength() const { return totalLength; } @@ -152,7 +151,7 @@ void PosixTask::primeBuffer() } } -bool PosixTask::write(const uint8_t* buf, uint len) +bool PosixTask::write(const uint8_t* buf, uint len) const { int err; uint count = 0; @@ -167,11 +166,11 @@ bool PosixTask::write(const uint8_t* buf, uint len) return true; } -bool PosixTask::write(sm_response& resp, uint payloadLength) +bool PosixTask::write(sm_response& resp, uint payloadLength) const { int err; uint count = 0; - uint8_t* buf = (uint8_t*)&resp; + auto* buf = (uint8_t*)&resp; resp.header.type = SM_MSG_START; resp.header.flags = 0; diff --git a/storage-manager/src/PosixTask.h b/storage-manager/src/PosixTask.h index 5b0641a21..ad0d52253 100644 --- a/storage-manager/src/PosixTask.h +++ b/storage-manager/src/PosixTask.h @@ -29,6 +29,7 @@ namespace storagemanager class PosixTask { public: + PosixTask() = delete; PosixTask(int sock, uint length); virtual ~PosixTask(); @@ -39,18 +40,16 @@ class PosixTask protected: int read(uint8_t* buf, uint length); bool write(const std::vector& buf); - bool write(sm_response& resp, uint payloadLength); - bool write(const uint8_t* buf, uint length); + bool write(sm_response& resp, uint payloadLength) const; + bool write(const uint8_t* buf, uint length) const; void consumeMsg(); // drains the remaining portion of the message - uint getLength(); // returns the total length of the msg + uint getLength() const; // returns the total length of the msg uint getRemainingLength(); // returns the remaining length from the caller's perspective void handleError(const char* name, int errCode); IOCoordinator* ioc; private: - PosixTask(); - int sock; int totalLength; uint remainingLengthInStream; diff --git a/storage-manager/src/PrefixCache.h b/storage-manager/src/PrefixCache.h index bd29b513a..351d66e85 100644 --- a/storage-manager/src/PrefixCache.h +++ b/storage-manager/src/PrefixCache.h @@ -39,7 +39,8 @@ namespace storagemanager class PrefixCache : public boost::noncopyable { public: - PrefixCache(const boost::filesystem::path& prefix); + PrefixCache() = delete; + explicit PrefixCache(const boost::filesystem::path& prefix); virtual ~PrefixCache(); // reading fcns @@ -82,8 +83,6 @@ class PrefixCache : public boost::noncopyable void validateCacheSize(); private: - PrefixCache(); - boost::filesystem::path cachePrefix; boost::filesystem::path journalPrefix; boost::filesystem::path firstDir; diff --git a/storage-manager/src/ProcessTask.h b/storage-manager/src/ProcessTask.h index afc91b363..7734d0032 100644 --- a/storage-manager/src/ProcessTask.h +++ b/storage-manager/src/ProcessTask.h @@ -24,14 +24,13 @@ namespace storagemanager class ProcessTask : public ThreadPool::Job { public: + ProcessTask() = delete; ProcessTask(int sock, uint length); // _sock is the socket to read from - virtual ~ProcessTask(); + ~ProcessTask() override; - void operator()(); + void operator()() override; private: - ProcessTask(); - void handleError(int errCode); int sock; uint length; diff --git a/storage-manager/src/ReadTask.h b/storage-manager/src/ReadTask.h index 9bcd7a21c..0bd2b7717 100644 --- a/storage-manager/src/ReadTask.h +++ b/storage-manager/src/ReadTask.h @@ -24,13 +24,11 @@ namespace storagemanager class ReadTask : public PosixTask { public: + ReadTask() = delete; ReadTask(int sock, uint length); - virtual ~ReadTask(); + ~ReadTask() override; - bool run(); - - private: - ReadTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/Replicator.h b/storage-manager/src/Replicator.h index 165c60e45..334f949ea 100644 --- a/storage-manager/src/Replicator.h +++ b/storage-manager/src/Replicator.h @@ -17,11 +17,11 @@ #pragma once -//#include "ThreadPool.h" +// #include "ThreadPool.h" #include "MetadataFile.h" #include #include -#include +#include #define JOURNAL_ENTRY_HEADER_SIZE 16 diff --git a/storage-manager/src/S3Storage.h b/storage-manager/src/S3Storage.h index 96e412edf..cda369a18 100644 --- a/storage-manager/src/S3Storage.h +++ b/storage-manager/src/S3Storage.h @@ -36,8 +36,9 @@ class S3Storage : public CloudStorage ~S3Storage() override; - int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL) override; - int getObject(const std::string& sourceKey, std::shared_ptr* data, size_t* size = NULL) override; + int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = nullptr) override; + int getObject(const std::string& sourceKey, std::shared_ptr* data, + size_t* size = nullptr) override; int putObject(const std::string& sourceFile, const std::string& destKey) override; int putObject(const std::shared_ptr data, size_t len, const std::string& destKey) override; int deleteObject(const std::string& key) override; @@ -77,7 +78,9 @@ class S3Storage : public CloudStorage struct Connection { - Connection(uint64_t id): id(id) {} + explicit Connection(uint64_t id) : id(id) + { + } uint64_t id; ms3_st* conn{nullptr}; timespec touchedAt{}; @@ -96,7 +99,8 @@ class S3Storage : public CloudStorage mutable boost::mutex connMutex; std::deque> freeConns; // using this as a stack to keep lru objects together - std::unordered_map> usedConns; // using this for displaying and killing tasks + std::unordered_map> + usedConns; // using this for displaying and killing tasks uint64_t nextConnId = 0; const time_t maxIdleSecs = 30; }; diff --git a/storage-manager/src/StatTask.cpp b/storage-manager/src/StatTask.cpp index b34e12fe2..82f10fff7 100644 --- a/storage-manager/src/StatTask.cpp +++ b/storage-manager/src/StatTask.cpp @@ -18,11 +18,10 @@ #include "StatTask.h" #include "messageFormat.h" #include "SMLogging.h" -#include +#include #include #include -#include -#include +#include using namespace std; diff --git a/storage-manager/src/StatTask.h b/storage-manager/src/StatTask.h index 6d4a480ac..c7b1da8a2 100644 --- a/storage-manager/src/StatTask.h +++ b/storage-manager/src/StatTask.h @@ -24,13 +24,11 @@ namespace storagemanager class StatTask : public PosixTask { public: + StatTask() = delete; StatTask(int sock, uint length); - virtual ~StatTask(); + ~StatTask() override; - bool run(); - - private: - StatTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/SyncTask.cpp b/storage-manager/src/SyncTask.cpp index b669d12d8..6b54b34e4 100644 --- a/storage-manager/src/SyncTask.cpp +++ b/storage-manager/src/SyncTask.cpp @@ -18,7 +18,7 @@ #include "SyncTask.h" #include "Synchronizer.h" #include "messageFormat.h" -#include +#include namespace storagemanager { @@ -26,9 +26,7 @@ SyncTask::SyncTask(int sock, uint len) : PosixTask(sock, len) { } -SyncTask::~SyncTask() -{ -} +SyncTask::~SyncTask() = default; bool SyncTask::run() { diff --git a/storage-manager/src/SyncTask.h b/storage-manager/src/SyncTask.h index 853d8b4ad..e6623f6b1 100644 --- a/storage-manager/src/SyncTask.h +++ b/storage-manager/src/SyncTask.h @@ -24,13 +24,11 @@ namespace storagemanager class SyncTask : public PosixTask { public: + SyncTask() = delete; SyncTask(int sock, uint length); - virtual ~SyncTask(); + ~SyncTask() override; - bool run(); - - private: - SyncTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/Synchronizer.h b/storage-manager/src/Synchronizer.h index 19244f892..82d21f61c 100644 --- a/storage-manager/src/Synchronizer.h +++ b/storage-manager/src/Synchronizer.h @@ -40,7 +40,7 @@ class Synchronizer : public boost::noncopyable, public ConfigListener { public: static Synchronizer* get(); - virtual ~Synchronizer(); + ~Synchronizer() override; // these take keys as parameters, not full path names, ex, pass in '12345' not // 'cache/12345'. @@ -61,7 +61,7 @@ class Synchronizer : public boost::noncopyable, public ConfigListener boost::filesystem::path getCachePath(); void printKPIs() const; - virtual void configListener() override; + void configListener() override; private: Synchronizer(); @@ -77,7 +77,7 @@ class Synchronizer : public boost::noncopyable, public ConfigListener // this struct kind of got sloppy. Need to clean it at some point. struct PendingOps { - PendingOps(int flags); + explicit PendingOps(int flags); ~PendingOps(); int opFlags; int waiters; @@ -89,9 +89,9 @@ class Synchronizer : public boost::noncopyable, public ConfigListener struct Job : public ThreadPool::Job { - virtual ~Job(){}; + ~Job() override = default; Job(Synchronizer* s, std::list::iterator i); - void operator()(); + void operator()() override; Synchronizer* sync; std::list::iterator it; }; diff --git a/storage-manager/src/ThreadPool.h b/storage-manager/src/ThreadPool.h index c39e5088b..e7d6d8758 100644 --- a/storage-manager/src/ThreadPool.h +++ b/storage-manager/src/ThreadPool.h @@ -37,7 +37,7 @@ class ThreadPool : public boost::noncopyable class Job { public: - virtual ~Job(){}; + virtual ~Job() = default; virtual void operator()() = 0; }; diff --git a/storage-manager/src/TruncateTask.h b/storage-manager/src/TruncateTask.h index 4497b2cc5..a52b14650 100644 --- a/storage-manager/src/TruncateTask.h +++ b/storage-manager/src/TruncateTask.h @@ -24,13 +24,11 @@ namespace storagemanager class TruncateTask : public PosixTask { public: + TruncateTask() = delete; TruncateTask(int sock, uint length); - virtual ~TruncateTask(); + ~TruncateTask() override; - bool run(); - - private: - TruncateTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/UnlinkTask.h b/storage-manager/src/UnlinkTask.h index fcc1e742b..44ee1dd3c 100644 --- a/storage-manager/src/UnlinkTask.h +++ b/storage-manager/src/UnlinkTask.h @@ -24,13 +24,11 @@ namespace storagemanager class UnlinkTask : public PosixTask { public: + UnlinkTask() = delete; UnlinkTask(int sock, uint length); - virtual ~UnlinkTask(); + ~UnlinkTask() override; - bool run(); - - private: - UnlinkTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/Utilities.h b/storage-manager/src/Utilities.h index eeba82059..186692a52 100644 --- a/storage-manager/src/Utilities.h +++ b/storage-manager/src/Utilities.h @@ -42,23 +42,23 @@ struct ScopedFileLock struct ScopedReadLock : public ScopedFileLock { ScopedReadLock(IOCoordinator* i, const std::string& k); - virtual ~ScopedReadLock(); - void lock(); - void unlock(); + ~ScopedReadLock() override; + void lock() override; + void unlock() override; }; struct ScopedWriteLock : public ScopedFileLock { ScopedWriteLock(IOCoordinator* i, const std::string& k); - virtual ~ScopedWriteLock(); - void lock(); - void unlock(); + ~ScopedWriteLock() override; + void lock() override; + void unlock() override; }; struct ScopedCloser { ScopedCloser(); - ScopedCloser(int f); + explicit ScopedCloser(int f); ~ScopedCloser(); int fd; }; diff --git a/storage-manager/src/WriteTask.h b/storage-manager/src/WriteTask.h index f06ebb3f4..d934426d1 100644 --- a/storage-manager/src/WriteTask.h +++ b/storage-manager/src/WriteTask.h @@ -24,13 +24,11 @@ namespace storagemanager class WriteTask : public PosixTask { public: + WriteTask() = delete; WriteTask(int sock, uint length); - virtual ~WriteTask(); + ~WriteTask() override; - bool run(); - - private: - WriteTask(); + bool run() override; }; } // namespace storagemanager diff --git a/storage-manager/src/main.cpp b/storage-manager/src/main.cpp index 6ca9be9df..a8a3e9309 100644 --- a/storage-manager/src/main.cpp +++ b/storage-manager/src/main.cpp @@ -15,15 +15,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include #include #include -#include -#include -#include -#include -#include -#include +#include using namespace std; @@ -54,7 +48,7 @@ class ServiceStorageManager : public Service, public Opt void setupChildSignalHandlers(); public: - ServiceStorageManager(const Opt& opt) : Service("StorageManager"), Opt(opt) + explicit ServiceStorageManager(const Opt& opt) : Service("StorageManager"), Opt(opt) { } void LogErrno() override @@ -74,14 +68,14 @@ class ServiceStorageManager : public Service, public Opt bool signalCaught = false; -void printCacheUsage(int sig) +void printCacheUsage(int /*sig*/) { Cache::get()->validateCacheSize(); cout << "Current cache size = " << Cache::get()->getCurrentCacheSize() << endl; cout << "Cache element count = " << Cache::get()->getCurrentCacheElementCount() << endl; } -void printKPIs(int sig) +void printKPIs(int /*sig*/) { IOCoordinator::get()->printKPIs(); Cache::get()->printKPIs(); @@ -120,29 +114,29 @@ void ServiceStorageManager::setupChildSignalHandlers() sa.sa_handler = shutdownSM; for (int sig : shutdownSignals) - sigaction(sig, &sa, NULL); + sigaction(sig, &sa, nullptr); sa.sa_handler = coreSM; for (int sig : coreSignals) - sigaction(sig, &sa, NULL); + sigaction(sig, &sa, nullptr); sa.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &sa, NULL); + sigaction(SIGPIPE, &sa, nullptr); sa.sa_handler = printCacheUsage; - sigaction(SIGUSR1, &sa, NULL); + sigaction(SIGUSR1, &sa, nullptr); sa.sa_handler = printKPIs; - sigaction(SIGUSR2, &sa, NULL); + sigaction(SIGUSR2, &sa, nullptr); } int ServiceStorageManager::Child() { SMLogging* logger = SMLogging::get(); - IOCoordinator* ioc = NULL; - Cache* cache = NULL; - Synchronizer* sync = NULL; - Replicator* rep = NULL; + IOCoordinator* ioc = nullptr; + Cache* cache = nullptr; + Synchronizer* sync = nullptr; + Replicator* rep = nullptr; /* Instantiate objects to have them verify config settings before continuing */ try diff --git a/utils/cacheutils/cacheutils.cpp b/utils/cacheutils/cacheutils.cpp index a0042e155..c94eb77c5 100644 --- a/utils/cacheutils/cacheutils.cpp +++ b/utils/cacheutils/cacheutils.cpp @@ -25,7 +25,7 @@ #include #include #include -//#define NDEBUG +// #define NDEBUG #include using namespace std; @@ -77,9 +77,7 @@ class CacheOpThread CacheOpThread(const string& svr, const ByteStream& outBs) : fServerName(svr), fOutBs(outBs) { } - ~CacheOpThread() - { - } + ~CacheOpThread() = default; void operator()() { struct timespec ts = {10, 0}; @@ -89,7 +87,7 @@ class CacheOpThread try { cl->write(fOutBs); - rc = extractRespCode(cl->read(&ts)); + rc = extractRespCode(*cl->read(&ts)); } catch (...) { @@ -343,7 +341,7 @@ int purgePrimProcFdCache(const std::vector files, const int pmId) oss << "PMS" << pmId; scoped_ptr cl(new MessageQueueClient(oss.str())); cl->write(bs); - rc = extractRespCode(cl->read(&ts)); + rc = extractRespCode(*cl->read(&ts)); } catch (...) { @@ -353,4 +351,3 @@ int purgePrimProcFdCache(const std::vector files, const int pmId) return rc; } } // namespace cacheutils - diff --git a/utils/cloudio/SMDataFile.h b/utils/cloudio/SMDataFile.h index 6f7965130..6e0539126 100644 --- a/utils/cloudio/SMDataFile.h +++ b/utils/cloudio/SMDataFile.h @@ -27,19 +27,19 @@ namespace idbdatafile class SMDataFile : public IDBDataFile { public: - virtual ~SMDataFile(); + ~SMDataFile() override; - ssize_t pread(void* ptr, off64_t offset, size_t count); - ssize_t read(void* ptr, size_t count); - ssize_t write(const void* ptr, size_t count); - int seek(off64_t offset, int whence); - int truncate(off64_t length); - int fallocate(int mode, off64_t offset, off64_t length); - off64_t size(); - off64_t tell(); - int flush(); - time_t mtime(); - int close(); + ssize_t pread(void* ptr, off64_t offset, size_t count) override; + ssize_t read(void* ptr, size_t count) override; + ssize_t write(const void* ptr, size_t count) override; + int seek(off64_t offset, int whence) override; + int truncate(off64_t length) override; + int fallocate(int mode, off64_t offset, off64_t length) override; + off64_t size() override; + off64_t tell() override; + int flush() override; + time_t mtime() override; + int close() override; // for testing only SMDataFile(const char* fname, int openmode, size_t fake_size); diff --git a/utils/cloudio/SMFileFactory.h b/utils/cloudio/SMFileFactory.h index b1e1313eb..02a1cc20b 100644 --- a/utils/cloudio/SMFileFactory.h +++ b/utils/cloudio/SMFileFactory.h @@ -25,7 +25,7 @@ namespace idbdatafile class SMFileFactory : public FileFactoryBase { public: - IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); + IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth) override; }; } // namespace idbdatafile diff --git a/utils/cloudio/SMFileSystem.h b/utils/cloudio/SMFileSystem.h index 92c26d10c..cb5046ee8 100644 --- a/utils/cloudio/SMFileSystem.h +++ b/utils/cloudio/SMFileSystem.h @@ -28,20 +28,20 @@ class SMFileSystem : public IDBFileSystem, boost::noncopyable { public: SMFileSystem(); - virtual ~SMFileSystem(); + ~SMFileSystem() override; // why are some of these const and some not const in IDBFileSystem? - int mkdir(const char* pathname); - off64_t size(const char* path) const; - off64_t compressedSize(const char* path) const; - int remove(const char* pathname); - int rename(const char* oldpath, const char* newpath); - bool exists(const char* pathname) const; - int listDirectory(const char* pathname, std::list& contents) const; - bool isDir(const char* pathname) const; - int copyFile(const char* srcPath, const char* destPath) const; - bool filesystemIsUp() const; - bool filesystemSync() const; + int mkdir(const char* pathname) override; + off64_t size(const char* path) const override; + off64_t compressedSize(const char* path) const override; + int remove(const char* pathname) override; + int rename(const char* oldpath, const char* newpath) override; + bool exists(const char* pathname) const override; + int listDirectory(const char* pathname, std::list& contents) const override; + bool isDir(const char* pathname) const override; + int copyFile(const char* srcPath, const char* destPath) const override; + bool filesystemIsUp() const override; + bool filesystemSync() const override; }; } // namespace idbdatafile diff --git a/utils/cloudio/sm_exceptions.h b/utils/cloudio/sm_exceptions.h index 96b299f85..fecf55420 100644 --- a/utils/cloudio/sm_exceptions.h +++ b/utils/cloudio/sm_exceptions.h @@ -24,7 +24,7 @@ namespace idbdatafile class NotImplementedYet : public std::logic_error { public: - NotImplementedYet(const std::string& s); + explicit NotImplementedYet(const std::string& s); }; NotImplementedYet::NotImplementedYet(const std::string& s) : std::logic_error(s) diff --git a/utils/common/any.hpp b/utils/common/any.hpp index e511db3ed..fa63279f8 100644 --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -9,7 +9,7 @@ * http://www.boost.org/LICENSE_1_0.txt */ -#include +#include #include #include #include "mcs_basic_types.h" @@ -40,7 +40,7 @@ struct base_any_policy template struct typed_base_any_policy : base_any_policy { - virtual size_t get_size() + size_t get_size() override { return sizeof(T); } @@ -53,23 +53,23 @@ template struct small_any_policy : typed_base_any_policy { virtual ~small_any_policy() = default; - virtual void static_delete(void** x) + void static_delete(void** x) override { - *x = 0; + *x = nullptr; } - virtual void copy_from_value(void const* src, void** dest) + void copy_from_value(void const* src, void** dest) override { new (dest) T(*reinterpret_cast(src)); } - virtual void clone(void* const* src, void** dest) + void clone(void* const* src, void** dest) override { *dest = *src; } - virtual void move(void* const* src, void** dest) + void move(void* const* src, void** dest) override { *dest = *src; } - virtual void* get_value(void** src) + void* get_value(void** src) override { return reinterpret_cast(src); } @@ -79,26 +79,26 @@ template struct big_any_policy : typed_base_any_policy { virtual ~big_any_policy() = default; - virtual void static_delete(void** x) + void static_delete(void** x) override { if (*x) delete (*reinterpret_cast(x)); - *x = NULL; + *x = nullptr; } - virtual void copy_from_value(void const* src, void** dest) + void copy_from_value(void const* src, void** dest) override { *dest = new T(*reinterpret_cast(src)); } - virtual void clone(void* const* src, void** dest) + void clone(void* const* src, void** dest) override { *dest = new T(**reinterpret_cast(src)); } - virtual void move(void* const* src, void** dest) + void move(void* const* src, void** dest) override { (*reinterpret_cast(dest))->~T(); **reinterpret_cast(dest) = **reinterpret_cast(src); } - virtual void* get_value(void** src) + void* get_value(void** src) override { return *src; } @@ -181,24 +181,24 @@ class any public: /// Initializing constructor. template - any(const T& x) : policy(anyimpl::get_policy()), object(NULL) + any(const T& x) : policy(anyimpl::get_policy()), object(nullptr) { assign(x); } /// Empty constructor. - any() : policy(anyimpl::get_policy()), object(NULL) + any() : policy(anyimpl::get_policy()), object(nullptr) { } /// Special initializing constructor for string literals. - any(const char* x) : policy(anyimpl::get_policy()), object(NULL) + any(const char* x) : policy(anyimpl::get_policy()), object(nullptr) { assign(x); } /// Copy constructor. - any(const any& x) : policy(anyimpl::get_policy()), object(NULL) + any(const any& x) : policy(anyimpl::get_policy()), object(nullptr) { assign(x); } diff --git a/utils/funcexp/func_mod.cpp b/utils/funcexp/func_mod.cpp index dff1a4e07..11c542168 100644 --- a/utils/funcexp/func_mod.cpp +++ b/utils/funcexp/func_mod.cpp @@ -405,7 +405,7 @@ int64_t Func_mod::getIntVal(Row& row, FunctionParm& parm, bool& isNull, return mod; } -uint64_t Func_mod::getUIntVal(Row& row, FunctionParm& parm, bool& isNull, +uint64_t Func_mod::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { if (parm.size() < 2) diff --git a/utils/funcexp/funcexpwrapper.cpp b/utils/funcexp/funcexpwrapper.cpp index 9b400f4ca..4a5c572a7 100644 --- a/utils/funcexp/funcexpwrapper.cpp +++ b/utils/funcexp/funcexpwrapper.cpp @@ -60,12 +60,13 @@ FuncExpWrapper::FuncExpWrapper(const FuncExpWrapper& f) rcs[i].reset(f.rcs[i]->clone()); } -FuncExpWrapper::~FuncExpWrapper() -{ -} +FuncExpWrapper::~FuncExpWrapper() = default; -void FuncExpWrapper::operator=(const FuncExpWrapper& f) +FuncExpWrapper& FuncExpWrapper::operator=(const FuncExpWrapper& f) { + if (&f == this) + return *this; + uint32_t i; filters.resize(f.filters.size()); @@ -77,6 +78,7 @@ void FuncExpWrapper::operator=(const FuncExpWrapper& f) for (i = 0; i < f.rcs.size(); i++) rcs[i].reset(f.rcs[i]->clone()); + return *this; } void FuncExpWrapper::serialize(ByteStream& bs) const @@ -101,12 +103,12 @@ void FuncExpWrapper::deserialize(ByteStream& bs) bs >> rcsCount; for (i = 0; i < fCount; i++) - filters.push_back(boost::shared_ptr(ObjectReader::createParseTree(bs))); + filters.emplace_back(ObjectReader::createParseTree(bs)); for (i = 0; i < rcsCount; i++) { - ReturnedColumn* rc = (ReturnedColumn*)ObjectReader::createTreeNode(bs); - rcs.push_back(boost::shared_ptr(rc)); + auto* rc = (ReturnedColumn*)ObjectReader::createTreeNode(bs); + rcs.emplace_back(rc); } } @@ -133,4 +135,4 @@ void FuncExpWrapper::addReturnedColumn(const boost::shared_ptr& rcs.push_back(rc); } -}; // namespace funcexp +} // namespace funcexp diff --git a/utils/funcexp/funcexpwrapper.h b/utils/funcexp/funcexpwrapper.h index cca271fe9..4362f5a5c 100644 --- a/utils/funcexp/funcexpwrapper.h +++ b/utils/funcexp/funcexpwrapper.h @@ -47,12 +47,12 @@ class FuncExpWrapper : public messageqcpp::Serializeable public: FuncExpWrapper(); FuncExpWrapper(const FuncExpWrapper&); - virtual ~FuncExpWrapper(); + ~FuncExpWrapper() override; - void operator=(const FuncExpWrapper&); + FuncExpWrapper& operator=(const FuncExpWrapper&); - void serialize(messageqcpp::ByteStream&) const; - void deserialize(messageqcpp::ByteStream&); + void serialize(messageqcpp::ByteStream&) const override; + void deserialize(messageqcpp::ByteStream&) override; bool evaluate(rowgroup::Row*); inline bool evaluateFilter(uint32_t num, rowgroup::Row* r); diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index be65235d0..c0ef30cbd 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -28,7 +28,7 @@ #define __STDC_FORMAT_MACROS #endif -#include +#include #include #include @@ -40,9 +40,7 @@ #ifndef ULONGLONG_MAX #define ULONGLONG_MAX ulonglong_max #endif -namespace funcexp -{ -namespace helpers +namespace funcexp::helpers { // 10 ** i const int64_t powerOf10_c[] = {1ll, @@ -242,7 +240,7 @@ inline int16_t convert_mysql_mode_to_modeflags(int16_t mode) // // This is a mirror of calc_week, at a later date we should use sql_time.h inline uint32_t calc_mysql_week(uint32_t year, uint32_t month, uint32_t day, int16_t modeflags, - uint32_t* weekyear = 0) + uint32_t* weekyear = nullptr) { // need to make sure that the date is valid if (!dataconvert::isDateValid(day, month, year)) @@ -268,7 +266,7 @@ inline uint32_t calc_mysql_week(uint32_t year, uint32_t month, uint32_t day, int if (!week_year && ((first_weekday && weekday != 0) || (!first_weekday && weekday >= 4))) return 0; - week_year = 1; + week_year = true; if (weekyear) { @@ -360,12 +358,12 @@ inline bool calc_time_diff(int64_t time1, int64_t time2, int l_sign, long long* (long long)(1000000) + (long long)msec1 - l_sign * (long long)msec2; - neg = 0; + neg = false; if (microseconds < 0) { microseconds = -microseconds; - neg = 1; + neg = true; } *seconds_out = microseconds / 1000000L; @@ -485,7 +483,7 @@ inline int getNumbers(const std::string& expr, int64_t* array, execplan::OpType if (funcType == execplan::OP_SUB) funcNeg = -1; - if (expr.size() == 0) + if (expr.empty()) return 0; // @bug 4703 reworked this code to avoid use of incrementally @@ -496,10 +494,8 @@ inline int getNumbers(const std::string& expr, int64_t* array, execplan::OpType int64_t number = 0; int neg = 1; - for (unsigned int i = 0; i < expr.size(); i++) + for (auto value : expr) { - char value = expr[i]; - if ((value >= '0' && value <= '9')) { foundNumber = true; @@ -553,7 +549,7 @@ inline int getNumbers(const std::string& expr, int* array, execplan::OpType func if (funcType == execplan::OP_SUB) funcNeg = -1; - if (expr.size() == 0) + if (expr.empty()) return 0; // @bug 4703 reworked this code to avoid use of incrementally @@ -564,10 +560,8 @@ inline int getNumbers(const std::string& expr, int* array, execplan::OpType func int number = 0; int neg = 1; - for (unsigned int i = 0; i < expr.size(); i++) + for (char value : expr) { - char value = expr[i]; - if ((value >= '0' && value <= '9')) { foundNumber = true; @@ -614,56 +608,51 @@ inline int getNumbers(const std::string& expr, int* array, execplan::OpType func inline int dayOfWeek(std::string day) // Sunday = 0 { - int value = -1; boost::to_lower(day); if (day == "sunday" || day == "sun") { - value = 0; + return 0; } else if (day == "monday" || day == "mon") { - value = 1; + return 1; } else if (day == "tuesday" || day == "tue") { - value = 2; + return 2; } else if (day == "wednesday" || day == "wed") { - value = 3; + return 3; } else if (day == "thursday" || day == "thu") { - value = 4; + return 4; } else if (day == "friday" || day == "fri") { - value = 5; + return 5; } else if (day == "saturday" || day == "sat") { - value = 6; - } - else - { - value = -1; + return 6; } - return value; + return -1; } inline string intToString(int64_t i) { char buf[32]; - snprintf(buf, 32, "%" PRId64 "", i); + snprintf(buf, sizeof(buf), "%" PRId64 "", i); return buf; } inline string uintToString(uint64_t i) { char buf[32]; - snprintf(buf, 32, "%" PRIu64 "", i); + snprintf(buf, sizeof(buf), "%" PRIu64 "", i); return buf; } @@ -672,7 +661,7 @@ inline string doubleToString(double d) // double's can be *really* long to print out. Max mysql // is e308 so allow for 308 + 36 decimal places minimum. char buf[384]; - snprintf(buf, 384, "%f", d); + snprintf(buf, sizeof(buf), "%f", d); return buf; } @@ -681,7 +670,7 @@ inline string longDoubleToString(long double ld) // long double's can be *really* long to print out. Max mysql // is e308 so allow for 308 + 36 decimal places minimum. char buf[384]; - snprintf(buf, 384, "%Lf", ld); + snprintf(buf, sizeof(buf), "%Lf", ld); return buf; } @@ -691,5 +680,4 @@ const std::string IDB_date_format(const dataconvert::DateTime&, const std::strin const std::string timediff(int64_t, int64_t, bool isDateTime = true); const char* convNumToStr(int64_t, char*, int); -} // namespace helpers -} // namespace funcexp +} // namespace funcexp::helpers diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index ca4ad594e..19eea32f7 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -68,10 +68,8 @@ class Func { public: Func(); - Func(const std::string& funcName); - virtual ~Func() - { - } + explicit Func(const std::string& funcName); + virtual ~Func() = default; const std::string funcName() const { @@ -116,7 +114,7 @@ class Func virtual std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) = 0; utils::NullString getNullStrVal(rowgroup::Row& row, FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) { bool isNull; std::string val = getStrVal(row, fp, isNull, op_ct); @@ -220,10 +218,8 @@ class Func class ParmTSInt64 : public datatypes::TSInt64Null { public: - ParmTSInt64() - { - } - ParmTSInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, long timeZone) + ParmTSInt64() = default; + ParmTSInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func&, long) : TSInt64Null(parm->data()->toTSInt64Null(row)) { } @@ -232,10 +228,8 @@ class ParmTSInt64 : public datatypes::TSInt64Null class ParmTUInt64 : public datatypes::TUInt64Null { public: - ParmTUInt64() - { - } - ParmTUInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, long timeZone) + ParmTUInt64() = default; + ParmTUInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func&, long) : TUInt64Null(parm->data()->toTUInt64Null(row)) { } diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index c67297d4f..825f015d0 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -35,15 +35,11 @@ namespace funcexp class Func_All : public Func { public: - Func_All() - { - } - Func_All(const std::string& funcName) : Func(funcName) - { - } - virtual ~Func_All() + Func_All() = default; + explicit Func_All(const std::string& funcName) : Func(funcName) { } + ~Func_All() override = default; /* int64_t getIntVal(rowgroup::Row& row, @@ -71,42 +67,40 @@ class Func_simple_case : public Func_All Func_simple_case() : Func_All("case_simple") { } - virtual ~Func_simple_case() - { - } + ~Func_simple_case() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; class Func_decode_oracle : public Func_All @@ -115,42 +109,40 @@ class Func_decode_oracle : public Func_All Func_decode_oracle() : Func_All("decode_oracle") { } - virtual ~Func_decode_oracle() - { - } + ~Func_decode_oracle() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_searched_case class @@ -161,42 +153,40 @@ class Func_searched_case : public Func_All Func_searched_case() : Func_All("case_searched") { } - virtual ~Func_searched_case() - { - } + ~Func_searched_case() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_if class @@ -207,39 +197,37 @@ class Func_if : public Func_All Func_if() : Func_All("Func_if") { } - virtual ~Func_if() - { - } + ~Func_if() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_ifnull class @@ -250,42 +238,40 @@ class Func_ifnull : public Func_All Func_ifnull() : Func_All("ifnull") { } - virtual ~Func_ifnull() - { - } + ~Func_ifnull() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_greatest class @@ -296,42 +282,40 @@ class Func_greatest : public Func_All Func_greatest() : Func_All("greatest") { } - virtual ~Func_greatest() - { - } + ~Func_greatest() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_least class @@ -342,39 +326,37 @@ class Func_least : public Func_All Func_least() : Func_All("least") { } - virtual ~Func_least() - { - } + ~Func_least() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_coalesce class @@ -385,39 +367,37 @@ class Func_coalesce : public Func_All Func_coalesce() : Func_All("coalesce") { } - virtual ~Func_coalesce() - { - } + ~Func_coalesce() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_nullif class @@ -428,42 +408,40 @@ class Func_nullif : public Func_All Func_nullif() : Func_All("nullif") { } - virtual ~Func_nullif() - { - } + ~Func_nullif() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; } // namespace funcexp diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index 8ce3986b2..b237449bc 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -34,15 +34,11 @@ namespace funcexp class Func_Bool : public Func { public: - Func_Bool() - { - } - Func_Bool(const std::string& funcName) : Func(funcName) - { - } - virtual ~Func_Bool() + Func_Bool() = default; + explicit Func_Bool(const std::string& funcName) : Func(funcName) { } + ~Func_Bool() override = default; /* virtual bool getBoolVal(rowgroup::Row& row, @@ -53,58 +49,58 @@ class Func_Bool : public Func */ int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return (getBoolVal(row, fp, isNull, op_ct) ? 1 : 0); } double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return (getBoolVal(row, fp, isNull, op_ct) ? 1.0 : 0.0); } long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return (getBoolVal(row, fp, isNull, op_ct) ? 1.0 : 0.0); } std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return (getBoolVal(row, fp, isNull, op_ct) ? "1" : "0"); } execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return execplan::IDB_Decimal(getIntVal(row, fp, isNull, op_ct), 0, 0); } int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { isNull = true; return 0; } int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { isNull = true; return 0; } int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { isNull = true; return 0; } int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { isNull = true; return 0; @@ -119,15 +115,13 @@ class Func_between : public Func_Bool Func_between() : Func_Bool("between") { } - virtual ~Func_between() - { - } + ~Func_between() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_notbetween class @@ -138,15 +132,13 @@ class Func_notbetween : public Func_Bool Func_notbetween() : Func_Bool("notbetween") { } - virtual ~Func_notbetween() - { - } + ~Func_notbetween() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_in class @@ -157,15 +149,13 @@ class Func_in : public Func_Bool Func_in() : Func_Bool("in") { } - virtual ~Func_in() - { - } + ~Func_in() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_notin class @@ -176,15 +166,13 @@ class Func_notin : public Func_Bool Func_notin() : Func_Bool("notin") { } - virtual ~Func_notin() - { - } + ~Func_notin() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_regexp class @@ -195,15 +183,13 @@ class Func_regexp : public Func_Bool Func_regexp() : Func_Bool("regexp") { } - virtual ~Func_regexp() - { - } + ~Func_regexp() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_isnull class @@ -217,24 +203,22 @@ class Func_isnull : public Func_Bool Func_isnull() : fIsNotNull(false) { } - Func_isnull(bool isnotnull) : fIsNotNull(isnotnull) + explicit Func_isnull(bool isnotnull) : fIsNotNull(isnotnull) { } /* * Destructor. isnull does not need to do anything here to clean up. */ - virtual ~Func_isnull() - { - } + ~Func_isnull() override = default; /** * Decide on the function's operation type */ - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; private: bool fIsNotNull; @@ -250,19 +234,17 @@ class Func_Truth : public Func_Bool { } - virtual ~Func_Truth() - { - } + ~Func_Truth() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType) + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override { assert(fp.size() == 1); return fp[0]->data()->resultType(); } bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { bool val = fp[0]->data()->getBoolVal(row, isNull); @@ -292,9 +274,7 @@ class Func_IsTrue : public Func_Truth Func_IsTrue() : Func_Truth("istrue", true, true) { } - ~Func_IsTrue() - { - } + ~Func_IsTrue() override = default; }; /** @brief Func_IsNotTrue class @@ -305,9 +285,7 @@ class Func_IsNotTrue : public Func_Truth Func_IsNotTrue() : Func_Truth("isnottrue", true, false) { } - ~Func_IsNotTrue() - { - } + ~Func_IsNotTrue() override = default; }; /** @brief Func_IsFalse class @@ -318,9 +296,7 @@ class Func_IsFalse : public Func_Truth Func_IsFalse() : Func_Truth("isfalse", false, true) { } - ~Func_IsFalse() - { - } + ~Func_IsFalse() override = default; }; /** @brief Func_IsNotFalse class @@ -331,9 +307,7 @@ class Func_IsNotFalse : public Func_Truth Func_IsNotFalse() : Func_Truth("isnotfalse", false, false) { } - ~Func_IsNotFalse() - { - } + ~Func_IsNotFalse() override = default; }; } // namespace funcexp diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index 179687708..1838a6a6f 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -36,15 +36,11 @@ namespace funcexp class Func_Dtm : public Func { public: - Func_Dtm() - { - } - Func_Dtm(const std::string& funcName) : Func(funcName) - { - } - virtual ~Func_Dtm() + Func_Dtm() = default; + explicit Func_Dtm(const std::string& funcName) : Func(funcName) { } + ~Func_Dtm() override = default; /* int64_t getIntVal(rowgroup::Row& row, @@ -54,13 +50,13 @@ class Func_Dtm : public Func */ double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return (double)getIntVal(row, fp, isNull, op_ct); } long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return (long double)getIntVal(row, fp, isNull, op_ct); } @@ -81,18 +77,16 @@ class Func_date : public Func_Dtm Func_date() : Func_Dtm("date") { } - virtual ~Func_date() - { - } + ~Func_date() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_time class @@ -103,21 +97,19 @@ class Func_time : public Func_Dtm Func_time() : Func_Dtm("time") { } - virtual ~Func_time() - { - } + ~Func_time() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_timediff class @@ -128,30 +120,28 @@ class Func_timediff : public Func_Dtm Func_timediff() : Func_Dtm("timediff") { } - virtual ~Func_timediff() - { - } + ~Func_timediff() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_date_add class @@ -162,18 +152,16 @@ class Func_date_add : public Func_Dtm Func_date_add() : Func_Dtm("date_add") { } - virtual ~Func_date_add() - { - } + ~Func_date_add() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cast_date class @@ -184,33 +172,31 @@ class Func_cast_date : public Func_Dtm Func_cast_date() : Func_Dtm("cast_date") { } - virtual ~Func_cast_date() - { - } + ~Func_cast_date() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cast_datetime class @@ -221,33 +207,31 @@ class Func_cast_datetime : public Func_Dtm Func_cast_datetime() : Func_Dtm("cast_datetime") { } - virtual ~Func_cast_datetime() - { - } + ~Func_cast_datetime() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_from_days class @@ -258,24 +242,22 @@ class Func_from_days : public Func_Dtm Func_from_days() : Func_Dtm("from_days") { } - virtual ~Func_from_days() - { - } + ~Func_from_days() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_add_time class @@ -286,30 +268,28 @@ class Func_add_time : public Func_Dtm Func_add_time() : Func_Dtm("add_time") { } - virtual ~Func_add_time() - { - } + ~Func_add_time() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_timestampdiff class @@ -320,30 +300,28 @@ class Func_timestampdiff : public Func_Dtm Func_timestampdiff() : Func_Dtm("timestamp_diff") { } - virtual ~Func_timestampdiff() - { - } + ~Func_timestampdiff() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_sysdate class @@ -354,30 +332,28 @@ class Func_sysdate : public Func_Dtm Func_sysdate() : Func_Dtm("sysdate") { } - virtual ~Func_sysdate() - { - } + ~Func_sysdate() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_from_unixtime @@ -388,32 +364,30 @@ class Func_from_unixtime : public Func_Dtm Func_from_unixtime() : Func_Dtm("from_unixtime") { } - virtual ~Func_from_unixtime() - { - } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + ~Func_from_unixtime() override = default; + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_str_to_date class @@ -424,30 +398,28 @@ class Func_str_to_date : public Func_Dtm Func_str_to_date() : Func_Dtm("str_to_date") { } - virtual ~Func_str_to_date() - { - } + ~Func_str_to_date() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_makedate class @@ -458,18 +430,16 @@ class Func_makedate : public Func_Dtm Func_makedate() : Func_Dtm("makedate") { } - virtual ~Func_makedate() - { - } + ~Func_makedate() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; class Func_convert_tz : public Func_Dtm @@ -478,24 +448,22 @@ class Func_convert_tz : public Func_Dtm Func_convert_tz() : Func_Dtm("convert_tz") { } - virtual ~Func_convert_tz() - { - } + ~Func_convert_tz() override = default; // bool from_tz_cached, to_tz_cached; // Time_zone *from_tz, *to_tz; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct); + execplan::CalpontSystemCatalog::ColType& ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; } // namespace funcexp diff --git a/utils/funcexp/functor_export.h b/utils/funcexp/functor_export.h index 77c45f819..59a104d17 100644 --- a/utils/funcexp/functor_export.h +++ b/utils/funcexp/functor_export.h @@ -38,31 +38,29 @@ class Func_rand : public Func Func_rand() : Func("rand"), fSeed1(0), fSeed2(0), fSeedSet(false) { } - virtual ~Func_rand() - { - } + ~Func_rand() override = default; double getRand(); - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return ((int64_t)getDoubleVal(row, fp, isNull, op_ct)); } double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return (long double)getDoubleVal(row, fp, isNull, op_ct); } std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return doubleToString(getDoubleVal(row, fp, isNull, op_ct)); } diff --git a/utils/funcexp/functor_int.h b/utils/funcexp/functor_int.h index ae5333a9f..d2bc7cd19 100644 --- a/utils/funcexp/functor_int.h +++ b/utils/funcexp/functor_int.h @@ -34,15 +34,11 @@ namespace funcexp class Func_Int : public Func { public: - Func_Int() - { - } - Func_Int(const std::string& funcName) : Func(funcName) - { - } - virtual ~Func_Int() + Func_Int() = default; + explicit Func_Int(const std::string& funcName) : Func(funcName) { } + ~Func_Int() override = default; /* int64_t getIntVal(rowgroup::Row& row, @@ -52,19 +48,19 @@ class Func_Int : public Func */ double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return ((double)getIntVal(row, fp, isNull, op_ct)); } long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return ((long double)getIntVal(row, fp, isNull, op_ct)); } std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return intToString(getIntVal(row, fp, isNull, op_ct)); } @@ -73,7 +69,7 @@ class Func_Int : public Func class Func_BitOp : public Func_Int { public: - Func_BitOp(const std::string& funcName) : Func_Int(funcName) + explicit Func_BitOp(const std::string& funcName) : Func_Int(funcName) { } execplan::CalpontSystemCatalog::ColType operationType( @@ -109,15 +105,13 @@ class Func_instr : public Func_Int Func_instr() : Func_Int("instr") { } - virtual ~Func_instr() - { - } + ~Func_instr() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_length class @@ -128,15 +122,13 @@ class Func_length : public Func_Int Func_length() : Func_Int("length") { } - virtual ~Func_length() - { - } + ~Func_length() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_sign class @@ -147,18 +139,16 @@ class Func_sign : public Func_Int Func_sign() : Func_Int("sign") { } - virtual ~Func_sign() - { - } + ~Func_sign() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_day class @@ -169,15 +159,13 @@ class Func_day : public Func_Int Func_day() : Func_Int("day") { } - virtual ~Func_day() - { - } + ~Func_day() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_minute class @@ -188,15 +176,13 @@ class Func_minute : public Func_Int Func_minute() : Func_Int("minute") { } - virtual ~Func_minute() - { - } + ~Func_minute() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_month class @@ -207,15 +193,13 @@ class Func_month : public Func_Int Func_month() : Func_Int("month") { } - virtual ~Func_month() - { - } + ~Func_month() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_week class @@ -226,15 +210,13 @@ class Func_week : public Func_Int Func_week() : Func_Int("week") { } - virtual ~Func_week() - { - } + ~Func_week() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_year class @@ -245,15 +227,13 @@ class Func_year : public Func_Int Func_year() : Func_Int("year") { } - virtual ~Func_year() - { - } + ~Func_year() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_to_days class @@ -264,15 +244,13 @@ class Func_to_days : public Func_Int Func_to_days() : Func_Int("to_days") { } - virtual ~Func_to_days() - { - } + ~Func_to_days() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_char_length class @@ -283,15 +261,13 @@ class Func_char_length : public Func_Int Func_char_length() : Func_Int("length") { } - virtual ~Func_char_length() - { - } + ~Func_char_length() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_extract class @@ -302,15 +278,13 @@ class Func_extract : public Func_Int Func_extract() : Func_Int("extract") { } - virtual ~Func_extract() - { - } + ~Func_extract() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cast_signed class @@ -321,15 +295,13 @@ class Func_cast_signed : public Func_Int Func_cast_signed() : Func_Int("cast_signed") { } - virtual ~Func_cast_signed() - { - } + ~Func_cast_signed() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cast_unsigned class @@ -340,21 +312,19 @@ class Func_cast_unsigned : public Func_Int Func_cast_unsigned() : Func_Int("cast_unsigned") { } - virtual ~Func_cast_unsigned() - { - } + ~Func_cast_unsigned() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return (int64_t)(getUintVal(row, fp, isNull, op_ct)); } uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_bitand class @@ -365,9 +335,7 @@ class Func_bitand : public Func_BitOp Func_bitand() : Func_BitOp("bitand") { } - virtual ~Func_bitand() - { - } + ~Func_bitand() override = default; bool fix(execplan::FunctionColumn& col) const override; }; @@ -379,9 +347,7 @@ class Func_bitor : public Func_BitOp Func_bitor() : Func_BitOp("bitor") { } - virtual ~Func_bitor() - { - } + ~Func_bitor() override = default; bool fix(execplan::FunctionColumn& col) const override; @@ -397,9 +363,7 @@ class Func_bitxor : public Func_BitOp Func_bitxor() : Func_BitOp("bitxor") { } - virtual ~Func_bitxor() - { - } + ~Func_bitxor() override = default; bool fix(execplan::FunctionColumn& col) const override; }; @@ -411,9 +375,7 @@ class Func_bit_count : public Func_BitOp Func_bit_count() : Func_BitOp("bit_count") { } - virtual ~Func_bit_count() - { - } + ~Func_bit_count() override = default; bool fix(execplan::FunctionColumn& col) const override; }; @@ -425,15 +387,13 @@ class Func_hour : public Func_Int Func_hour() : Func_Int("hour") { } - virtual ~Func_hour() - { - } + ~Func_hour() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_second class @@ -444,15 +404,13 @@ class Func_second : public Func_Int Func_second() : Func_Int("second") { } - virtual ~Func_second() - { - } + ~Func_second() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_dayofweek class @@ -463,15 +421,13 @@ class Func_dayofweek : public Func_Int Func_dayofweek() : Func_Int("dayofweek") { } - virtual ~Func_dayofweek() - { - } + ~Func_dayofweek() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_dayofyear class @@ -482,15 +438,13 @@ class Func_dayofyear : public Func_Int Func_dayofyear() : Func_Int("dayofyear") { } - virtual ~Func_dayofyear() - { - } + ~Func_dayofyear() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_leftshift class @@ -501,9 +455,7 @@ class Func_leftshift : public Func_BitOp Func_leftshift() : Func_BitOp("leftshift") { } - virtual ~Func_leftshift() - { - } + ~Func_leftshift() override = default; bool fix(execplan::FunctionColumn& col) const override; }; @@ -515,9 +467,7 @@ class Func_rightshift : public Func_BitOp Func_rightshift() : Func_BitOp("rightshift") { } - virtual ~Func_rightshift() - { - } + ~Func_rightshift() override = default; bool fix(execplan::FunctionColumn& col) const override; }; @@ -529,15 +479,13 @@ class Func_quarter : public Func_Int Func_quarter() : Func_Int("quarter") { } - virtual ~Func_quarter() - { - } + ~Func_quarter() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_ascii class @@ -548,15 +496,13 @@ class Func_ascii : public Func_Int Func_ascii() : Func_Int("ascii") { } - virtual ~Func_ascii() - { - } + ~Func_ascii() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_dayname class @@ -567,18 +513,16 @@ class Func_dayname : public Func_Int Func_dayname() : Func_Int("dayname") { } - virtual ~Func_dayname() - { - } + ~Func_dayname() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_weekday class @@ -589,15 +533,13 @@ class Func_weekday : public Func_Int Func_weekday() : Func_Int("weekday") { } - virtual ~Func_weekday() - { - } + ~Func_weekday() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_yearweek class @@ -608,15 +550,13 @@ class Func_yearweek : public Func_Int Func_yearweek() : Func_Int("yearweek") { } - virtual ~Func_yearweek() - { - } + ~Func_yearweek() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_last_day class @@ -627,15 +567,13 @@ class Func_last_day : public Func_Int Func_last_day() : Func_Int("last_day") { } - virtual ~Func_last_day() - { - } + ~Func_last_day() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_time_to_sec class @@ -646,15 +584,13 @@ class Func_time_to_sec : public Func_Int Func_time_to_sec() : Func_Int("time_to_sec") { } - virtual ~Func_time_to_sec() - { - } + ~Func_time_to_sec() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_microsecond class @@ -665,15 +601,13 @@ class Func_microsecond : public Func_Int Func_microsecond() : Func_Int("microsecond") { } - virtual ~Func_microsecond() - { - } + ~Func_microsecond() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_crc32 class @@ -684,15 +618,13 @@ class Func_crc32 : public Func_Int Func_crc32() : Func_Int("crc32") { } - virtual ~Func_crc32() - { - } + ~Func_crc32() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_period_add class @@ -703,15 +635,13 @@ class Func_period_add : public Func_Int Func_period_add() : Func_Int("period_add") { } - virtual ~Func_period_add() - { - } + ~Func_period_add() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_period_diff class @@ -722,15 +652,13 @@ class Func_period_diff : public Func_Int Func_period_diff() : Func_Int("period_diff") { } - virtual ~Func_period_diff() - { - } + ~Func_period_diff() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_strcmp class @@ -741,18 +669,16 @@ class Func_strcmp : public Func_Int Func_strcmp() : Func_Int("strcmp") { } - virtual ~Func_strcmp() - { - } + ~Func_strcmp() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_unix_timestamp class @@ -763,18 +689,16 @@ class Func_unix_timestamp : public Func_Int Func_unix_timestamp() : Func_Int("unix_timestamp") { } - virtual ~Func_unix_timestamp() - { - } + ~Func_unix_timestamp() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_strcmp class @@ -785,22 +709,20 @@ class Func_find_in_set : public Func_Int Func_find_in_set() : Func_Int("find_in_set") { } - virtual ~Func_find_in_set() - { - } + ~Func_find_in_set() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; } // namespace funcexp diff --git a/utils/funcexp/functor_json.h b/utils/funcexp/functor_json.h index 1d12df584..0c1b0d9a2 100644 --- a/utils/funcexp/functor_json.h +++ b/utils/funcexp/functor_json.h @@ -28,7 +28,7 @@ struct JSONPath JSONPath() : constant(false), parsed(false), currStep(nullptr) { } - json_path_t p; + json_path_t p{}; bool constant; // check if the argument is constant bool parsed; // check if the argument is parsed json_path_step_t* currStep; @@ -52,10 +52,9 @@ class JSONEgWrapper : public json_engine_t class JSONPathWrapper : public JSONPath { protected: - virtual ~JSONPathWrapper() - { - } + virtual ~JSONPathWrapper() = default; virtual bool checkAndGetValue(JSONEgWrapper* je, std::string& ret, int* error) = 0; + public: bool extract(std::string& ret, rowgroup::Row& row, execplan::SPTP& funcParmJS, execplan::SPTP& funcParmPath); @@ -68,15 +67,13 @@ class Func_json_valid : public Func_Bool Func_json_valid() : Func_Bool("json_valid") { } - ~Func_json_valid() - { - } + ~Func_json_valid() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_depth class @@ -87,15 +84,13 @@ class Func_json_depth : public Func_Int Func_json_depth() : Func_Int("json_depth") { } - virtual ~Func_json_depth() - { - } + ~Func_json_depth() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_length class @@ -109,15 +104,13 @@ class Func_json_length : public Func_Int Func_json_length() : Func_Int("json_length") { } - virtual ~Func_json_length() - { - } + ~Func_json_length() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_equals class @@ -128,15 +121,13 @@ class Func_json_equals : public Func_Bool Func_json_equals() : Func_Bool("json_equals") { } - ~Func_json_equals() - { - } + ~Func_json_equals() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_normalize class @@ -147,15 +138,13 @@ class Func_json_normalize : public Func_Str Func_json_normalize() : Func_Str("json_normalize") { } - virtual ~Func_json_normalize() - { - } + ~Func_json_normalize() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_type class @@ -166,15 +155,13 @@ class Func_json_type : public Func_Str Func_json_type() : Func_Str("json_type") { } - virtual ~Func_json_type() - { - } + ~Func_json_type() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_object class @@ -185,15 +172,13 @@ class Func_json_object : public Func_Str Func_json_object() : Func_Str("json_object") { } - virtual ~Func_json_object() - { - } + ~Func_json_object() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_array class @@ -204,15 +189,13 @@ class Func_json_array : public Func_Str Func_json_array() : Func_Str("json_array") { } - virtual ~Func_json_array() - { - } + ~Func_json_array() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_keys class */ @@ -225,15 +208,13 @@ class Func_json_keys : public Func_Str Func_json_keys() : Func_Str("json_keys") { } - virtual ~Func_json_keys() - { - } + ~Func_json_keys() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_exists class */ @@ -246,15 +227,13 @@ class Func_json_exists : public Func_Bool Func_json_exists() : Func_Bool("json_exists") { } - ~Func_json_exists() - { - } + ~Func_json_exists() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_quote class @@ -268,15 +247,13 @@ class Func_json_quote : public Func_Str Func_json_quote() : Func_Str("json_quote") { } - virtual ~Func_json_quote() - { - } + ~Func_json_quote() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_unquote class @@ -290,15 +267,13 @@ class Func_json_unquote : public Func_Str Func_json_unquote() : Func_Str("json_unquote") { } - virtual ~Func_json_unquote() - { - } + ~Func_json_unquote() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_format class @@ -321,7 +296,7 @@ class Func_json_format : public Func_Str Func_json_format() : Func_Str("json_detailed"), fmt(DETAILED) { } - Func_json_format(FORMATS format) : fmt(format) + explicit Func_json_format(FORMATS format) : fmt(format) { assert(format != NONE); switch (format) @@ -332,15 +307,13 @@ class Func_json_format : public Func_Str default: break; } } - virtual ~Func_json_format() - { - } + ~Func_json_format() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_merge_preserve class */ @@ -350,15 +323,13 @@ class Func_json_merge : public Func_Str Func_json_merge() : Func_Str("json_merge_preserve") { } - virtual ~Func_json_merge() - { - } + ~Func_json_merge() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_merge_patch class @@ -369,15 +340,13 @@ class Func_json_merge_patch : public Func_Str Func_json_merge_patch() : Func_Str("json_merge_patch") { } - virtual ~Func_json_merge_patch() - { - } + ~Func_json_merge_patch() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_value class @@ -388,9 +357,7 @@ class Func_json_value : public Func_Str Func_json_value() : Func_Str("json_value") { } - virtual ~Func_json_value() - { - } + ~Func_json_value() override = default; execplan::CalpontSystemCatalog::ColType operationType( FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; @@ -407,9 +374,7 @@ class Func_json_query : public Func_Str Func_json_query() : Func_Str("json_query") { } - virtual ~Func_json_query() - { - } + ~Func_json_query() override = default; execplan::CalpontSystemCatalog::ColType operationType( FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; @@ -431,15 +396,13 @@ class Func_json_contains : public Func_Bool Func_json_contains() : Func_Bool("json_contains"), arg2Const(false), arg2Parsed(false), arg2Val("") { } - virtual ~Func_json_contains() - { - } + ~Func_json_contains() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_array_append class */ @@ -452,15 +415,13 @@ class Func_json_array_append : public Func_Str Func_json_array_append() : Func_Str("json_array_append") { } - virtual ~Func_json_array_append() - { - } + ~Func_json_array_append() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; private: static const int padding = 8; @@ -476,15 +437,13 @@ class Func_json_array_insert : public Func_Str Func_json_array_insert() : Func_Str("json_array_insert") { } - virtual ~Func_json_array_insert() - { - } + ~Func_json_array_insert() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_insert class @@ -508,7 +467,7 @@ class Func_json_insert : public Func_Str Func_json_insert() : Func_Str("json_insert"), mode(INSERT) { } - Func_json_insert(MODE m) : mode(m) + explicit Func_json_insert(MODE m) : mode(m) { assert(m != NONE); switch (m) @@ -519,20 +478,18 @@ class Func_json_insert : public Func_Str default: break; } } - virtual ~Func_json_insert() - { - } + ~Func_json_insert() override = default; MODE getMode() const { return mode; } - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_remove class */ @@ -545,15 +502,13 @@ class Func_json_remove : public Func_Str Func_json_remove() : Func_Str("json_remove") { } - virtual ~Func_json_remove() - { - } + ~Func_json_remove() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_contains_path class @@ -572,15 +527,13 @@ class Func_json_contains_path : public Func_Bool : Func_Bool("json_contains_path"), isModeOne(false), isModeConst(false), isModeParsed(false) { } - virtual ~Func_json_contains_path() - { - } + ~Func_json_contains_path() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_overlaps class @@ -594,15 +547,13 @@ class Func_json_overlaps : public Func_Bool Func_json_overlaps() : Func_Bool("json_overlaps") { } - virtual ~Func_json_overlaps() - { - } + ~Func_json_overlaps() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; }; /** @brief Func_json_search class */ @@ -620,15 +571,13 @@ class Func_json_search : public Func_Str : Func_Str("json_search"), isModeParsed(false), isModeConst(false), isModeOne(false), escape('\\') { } - virtual ~Func_json_search() - { - } + ~Func_json_search() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; private: int cmpJSValWild(json_engine_t* jsEg, const utils::NullString& cmpStr, const CHARSET_INFO* cs); @@ -644,24 +593,22 @@ class Func_json_extract : public Func_Str Func_json_extract() : Func_Str("json_extract") { } - virtual ~Func_json_extract() - { - } + ~Func_json_extract() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& type); + execplan::CalpontSystemCatalog::ColType& type) override; private: int doExtract(rowgroup::Row& row, FunctionParm& fp, json_value_types* type, std::string& retJS, diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index fbdf25964..a2f0e608d 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -35,36 +35,32 @@ namespace funcexp class Func_Real : public Func { public: - Func_Real() - { - } - Func_Real(const std::string& funcName) : Func(funcName) - { - } - virtual ~Func_Real() + Func_Real() = default; + explicit Func_Real(const std::string& funcName) : Func(funcName) { } + ~Func_Real() override = default; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return ((int64_t)getDoubleVal(row, fp, isNull, op_ct)); } uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return ((uint64_t)getDoubleVal(row, fp, isNull, op_ct)); } long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return ((long double)getDoubleVal(row, fp, isNull, op_ct)); } std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return doubleToString(getDoubleVal(row, fp, isNull, op_ct)); } @@ -78,27 +74,25 @@ class Func_abs : public Func_Real Func_abs() : Func_Real("abs") { } - virtual ~Func_abs() - { - } + ~Func_abs() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_exp class @@ -109,18 +103,16 @@ class Func_exp : public Func_Real Func_exp() : Func_Real("exp") { } - virtual ~Func_exp() - { - } + ~Func_exp() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_pow class @@ -131,18 +123,16 @@ class Func_pow : public Func_Real Func_pow() : Func_Real("pow") { } - virtual ~Func_pow() - { - } + ~Func_pow() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_round class @@ -153,39 +143,37 @@ class Func_round : public Func_Real Func_round() : Func_Real("round") { } - virtual ~Func_round() - { - } + ~Func_round() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); - + execplan::CalpontSystemCatalog::ColType& op_ct) override; + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_truncate class @@ -196,39 +184,37 @@ class Func_truncate : public Func_Real Func_truncate() : Func_Real("truncate") { } - virtual ~Func_truncate() - { - } + ~Func_truncate() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_ceil class @@ -239,30 +225,28 @@ class Func_ceil : public Func_Real Func_ceil() : Func_Real("ceil") { } - virtual ~Func_ceil() - { - } + ~Func_ceil() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_floor class @@ -273,30 +257,28 @@ class Func_floor : public Func_Real Func_floor() : Func_Real("floor") { } - virtual ~Func_floor() - { - } + ~Func_floor() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cast_decimal class @@ -307,24 +289,22 @@ class Func_cast_decimal : public Func_Real Func_cast_decimal() : Func_Real("cast_decimal") { } - virtual ~Func_cast_decimal() - { - } + ~Func_cast_decimal() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cast_double class @@ -335,21 +315,19 @@ class Func_cast_double : public Func_Real Func_cast_double() : Func_Real("cast_double") { } - virtual ~Func_cast_double() - { - } + ~Func_cast_double() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_mod class @@ -360,30 +338,28 @@ class Func_mod : public Func_Real Func_mod() : Func_Real("mod") { } - virtual ~Func_mod() - { - } + ~Func_mod() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - execplan::CalpontSystemCatalog::ColType& operationColType); + execplan::CalpontSystemCatalog::ColType& operationColType) override; - uint64_t getUIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - execplan::CalpontSystemCatalog::ColType& operationColType); + uint64_t getUintVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; private: template @@ -418,15 +394,13 @@ class Func_acos : public Func_Real Func_acos() : Func_Real("acos") { } - virtual ~Func_acos() - { - } + ~Func_acos() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_asin class @@ -437,15 +411,13 @@ class Func_asin : public Func_Real Func_asin() : Func_Real("asin") { } - virtual ~Func_asin() - { - } + ~Func_asin() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_atan class @@ -456,15 +428,13 @@ class Func_atan : public Func_Real Func_atan() : Func_Real("atan") { } - virtual ~Func_atan() - { - } + ~Func_atan() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cos class @@ -475,15 +445,13 @@ class Func_cos : public Func_Real Func_cos() : Func_Real("cos") { } - virtual ~Func_cos() - { - } + ~Func_cos() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cot class @@ -494,15 +462,13 @@ class Func_cot : public Func_Real Func_cot() : Func_Real("cot") { } - virtual ~Func_cot() - { - } + ~Func_cot() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_log class @@ -513,15 +479,13 @@ class Func_log : public Func_Real Func_log() : Func_Real("log") { } - virtual ~Func_log() - { - } + ~Func_log() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_log2 class @@ -532,15 +496,13 @@ class Func_log2 : public Func_Real Func_log2() : Func_Real("log2") { } - virtual ~Func_log2() - { - } + ~Func_log2() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_log10 class @@ -551,15 +513,13 @@ class Func_log10 : public Func_Real Func_log10() : Func_Real("log10") { } - virtual ~Func_log10() - { - } + ~Func_log10() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_sin class @@ -570,15 +530,13 @@ class Func_sin : public Func_Real Func_sin() : Func_Real("sin") { } - virtual ~Func_sin() - { - } + ~Func_sin() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_sqrt class @@ -589,15 +547,13 @@ class Func_sqrt : public Func_Real Func_sqrt() : Func_Real("sqrt") { } - virtual ~Func_sqrt() - { - } + ~Func_sqrt() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_tan class @@ -608,15 +564,13 @@ class Func_tan : public Func_Real Func_tan() : Func_Real("tan") { } - virtual ~Func_tan() - { - } + ~Func_tan() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_radians class @@ -627,15 +581,13 @@ class Func_radians : public Func_Real Func_radians() : Func_Real("radians") { } - virtual ~Func_radians() - { - } + ~Func_radians() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_div class @@ -646,24 +598,22 @@ class Func_div : public Func_Real Func_div() : Func_Real("DIV") { } - virtual ~Func_div() - { - } + ~Func_div() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; uint64_t getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_inet_aton class to convert ascii IP address to big-endian @@ -675,39 +625,37 @@ class Func_inet_aton : public Func_Real Func_inet_aton() : Func_Real("inet_aton") { } - virtual ~Func_inet_aton() - { - } + ~Func_inet_aton() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; private: int64_t convertAton(const std::string& ipString, bool& isNull); @@ -721,15 +669,13 @@ class Func_degrees : public Func_Real Func_degrees() : Func_Real("degrees") { } - virtual ~Func_degrees() - { - } + ~Func_degrees() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; -} // namespace funcexp +} // namespace funcexp \ No newline at end of file diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 17b8adea8..c671c1a65 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -35,32 +35,28 @@ namespace funcexp class Func_Str : public Func { public: - Func_Str() - { - } - Func_Str(const std::string& funcName) : Func(funcName) - { - } - virtual ~Func_Str() + Func_Str() = default; + explicit Func_Str(const std::string& funcName) : Func(funcName) { } + ~Func_Str() override = default; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return atoll(getStrVal(row, fp, isNull, op_ct).c_str()); } double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { - return strtod(getStrVal(row, fp, isNull, op_ct).c_str(), NULL); + return strtod(getStrVal(row, fp, isNull, op_ct).c_str(), nullptr); } long double getLongDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { - return strtold(getStrVal(row, fp, isNull, op_ct).c_str(), NULL); + return strtold(getStrVal(row, fp, isNull, op_ct).c_str(), nullptr); } #if 0 @@ -71,34 +67,34 @@ class Func_Str : public Func #endif execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { return execplan::IDB_Decimal(atoll(getStrVal(row, fp, isNull, op_ct).c_str()), 0, 0); } int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { auto str = getStrVal(row, fp, isNull, op_ct); - return (isNull ? 0 : stringToDate(str)); + return isNull ? 0 : stringToDate(str); } int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { auto str = getStrVal(row, fp, isNull, op_ct); - return (isNull ? 0 : stringToDatetime(str)); + return isNull ? 0 : stringToDatetime(str); } int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { auto str = getStrVal(row, fp, isNull, op_ct); return (isNull ? 0 : stringToTimestamp(str, op_ct.getTimeZone())); } int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + execplan::CalpontSystemCatalog::ColType& op_ct) override { auto str = getStrVal(row, fp, isNull, op_ct); return (isNull ? 0 : stringToTime(str)); @@ -124,10 +120,7 @@ class Func_Str : public Func case execplan::CalpontSystemCatalog::FLOAT: floatVal = fp->data()->getFloatVal(row, isNull); break; - default: - fFloatStr = fp->data()->getStrVal(row, isNull).safeString(""); - return; - break; + default: fFloatStr = fp->data()->getStrVal(row, isNull).safeString(""); return; } if (isNull) @@ -140,14 +133,14 @@ class Func_Str : public Func if (std::isnan(exponent) || std::isnan(base)) { - snprintf(buf, 20, "%Lf", floatVal); - fFloatStr = execplan::removeTrailing0(buf, 20); + snprintf(buf, sizeof(buf), "%Lf", floatVal); + fFloatStr = execplan::removeTrailing0(buf, sizeof(buf)); } else { - snprintf(buf, 20, "%.5Lf", base); - fFloatStr = execplan::removeTrailing0(buf, 20); - snprintf(buf, 20, "e%02d", exponent); + snprintf(buf, sizeof(buf), "%.5Lf", base); + fFloatStr = execplan::removeTrailing0(buf, sizeof(buf)); + snprintf(buf, sizeof(buf), "e%02d", exponent); fFloatStr += buf; } } @@ -161,15 +154,13 @@ class Func_concat : public Func_Str Func_concat() : Func_Str("concat") { } - virtual ~Func_concat() - { - } + ~Func_concat() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_concat_oracle class @@ -180,15 +171,13 @@ class Func_concat_oracle : public Func_Str Func_concat_oracle() : Func_Str("concat_operator_oracle") { } - virtual ~Func_concat_oracle() - { - } + ~Func_concat_oracle() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_substr class @@ -199,15 +188,13 @@ class Func_substr : public Func_Str Func_substr() : Func_Str("substr") { } - virtual ~Func_substr() - { - } + ~Func_substr() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_date_format class @@ -218,24 +205,22 @@ class Func_date_format : public Func_Str Func_date_format() : Func_Str("date_format") { } - virtual ~Func_date_format() - { - } + ~Func_date_format() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_lcase class @@ -246,15 +231,13 @@ class Func_lcase : public Func_Str Func_lcase() : Func_Str("lcase") { } - virtual ~Func_lcase() - { - } + ~Func_lcase() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_ucase class @@ -265,15 +248,13 @@ class Func_ucase : public Func_Str Func_ucase() : Func_Str("ucase") { } - virtual ~Func_ucase() - { - } + ~Func_ucase() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_left class @@ -284,15 +265,13 @@ class Func_left : public Func_Str Func_left() : Func_Str("left") { } - virtual ~Func_left() - { - } + ~Func_left() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_ltrim class @@ -303,15 +282,13 @@ class Func_ltrim : public Func_Str Func_ltrim() : Func_Str("ltrim") { } - virtual ~Func_ltrim() - { - } + ~Func_ltrim() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_rtrim class @@ -322,15 +299,13 @@ class Func_rtrim : public Func_Str Func_rtrim() : Func_Str("rtrim") { } - virtual ~Func_rtrim() - { - } + ~Func_rtrim() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_trim class @@ -341,15 +316,13 @@ class Func_trim : public Func_Str Func_trim() : Func_Str("trim") { } - virtual ~Func_trim() - { - } + ~Func_trim() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_ltrim class @@ -360,15 +333,13 @@ class Func_ltrim_oracle : public Func_Str Func_ltrim_oracle() : Func_Str("ltrim_oracle") { } - virtual ~Func_ltrim_oracle() - { - } + ~Func_ltrim_oracle() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_rtrim class @@ -379,15 +350,13 @@ class Func_rtrim_oracle : public Func_Str Func_rtrim_oracle() : Func_Str("rtrim_oracle") { } - virtual ~Func_rtrim_oracle() - { - } + ~Func_rtrim_oracle() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_trim class @@ -398,15 +367,13 @@ class Func_trim_oracle : public Func_Str Func_trim_oracle() : Func_Str("trim_oracle") { } - virtual ~Func_trim_oracle() - { - } + ~Func_trim_oracle() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_lpad class @@ -419,15 +386,13 @@ class Func_lpad : public Func_Str Func_lpad() : Func_Str("lpad") { } - virtual ~Func_lpad() - { - } + ~Func_lpad() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_rpad class @@ -440,15 +405,13 @@ class Func_rpad : public Func_Str Func_rpad() : Func_Str("rpad") { } - virtual ~Func_rpad() - { - } + ~Func_rpad() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_replace class @@ -459,15 +422,13 @@ class Func_replace : public Func_Str Func_replace() : Func_Str("replace") { } - virtual ~Func_replace() - { - } + ~Func_replace() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; class Func_regexp_replace : public Func_Str @@ -476,70 +437,58 @@ class Func_regexp_replace : public Func_Str Func_regexp_replace() : Func_Str("regexp_replace") { } - virtual ~Func_regexp_replace() - { - } + ~Func_regexp_replace() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; - class Func_regexp_instr : public Func_Str { public: Func_regexp_instr() : Func_Str("regexp_instr") { } - virtual ~Func_regexp_instr() - { - } + ~Func_regexp_instr() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; - class Func_regexp_substr : public Func_Str { public: Func_regexp_substr() : Func_Str("regexp_substr") { } - virtual ~Func_regexp_substr() - { - } + ~Func_regexp_substr() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; - - class Func_replace_oracle : public Func_Str { public: Func_replace_oracle() : Func_Str("replace_oracle") { } - virtual ~Func_replace_oracle() - { - } + ~Func_replace_oracle() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_right class @@ -550,15 +499,13 @@ class Func_right : public Func_Str Func_right() : Func_Str("right") { } - virtual ~Func_right() - { - } + ~Func_right() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_char class @@ -569,15 +516,13 @@ class Func_char : public Func_Str Func_char() : Func_Str("char") { } - virtual ~Func_char() - { - } + ~Func_char() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_cast_char class @@ -588,15 +533,13 @@ class Func_cast_char : public Func_Str Func_cast_char() : Func_Str("char") { } - virtual ~Func_cast_char() - { - } + ~Func_cast_char() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_format class @@ -607,15 +550,13 @@ class Func_format : public Func_Str Func_format() : Func_Str("format") { } - virtual ~Func_format() - { - } + ~Func_format() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_conv class @@ -626,15 +567,13 @@ class Func_conv : public Func_Str Func_conv() : Func_Str("conv") { } - virtual ~Func_conv() - { - } + ~Func_conv() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_md5 class @@ -645,15 +584,13 @@ class Func_md5 : public Func_Str Func_md5() : Func_Str("md5") { } - virtual ~Func_md5() - { - } + ~Func_md5() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_unhex class @@ -664,15 +601,13 @@ class Func_unhex : public Func_Str Func_unhex() : Func_Str("unhex") { } - virtual ~Func_unhex() - { - } + ~Func_unhex() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_concat_ws class @@ -683,15 +618,13 @@ class Func_concat_ws : public Func_Str Func_concat_ws() : Func_Str("concat_ws") { } - virtual ~Func_concat_ws() - { - } + ~Func_concat_ws() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_monthname class @@ -702,35 +635,33 @@ class Func_monthname : public Func_Str Func_monthname() : Func_Str("monthname") { } - virtual ~Func_monthname() - { - } + ~Func_monthname() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getIntValInternal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_time_format class @@ -741,15 +672,13 @@ class Func_time_format : public Func_Str Func_time_format() : Func_Str("time_format") { } - virtual ~Func_time_format() - { - } + ~Func_time_format() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_sec_to_time class @@ -760,24 +689,22 @@ class Func_sec_to_time : public Func_Str Func_sec_to_time() : Func_Str("sec_to_time") { } - virtual ~Func_sec_to_time() - { - } + ~Func_sec_to_time() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_substring_index class @@ -788,15 +715,13 @@ class Func_substring_index : public Func_Str Func_substring_index() : Func_Str("substring_index") { } - virtual ~Func_substring_index() - { - } + ~Func_substring_index() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_hex class @@ -807,15 +732,13 @@ class Func_hex : public Func_Str Func_hex() : Func_Str("hex") { } - virtual ~Func_hex() - { - } + ~Func_hex() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_repeat class @@ -826,15 +749,13 @@ class Func_repeat : public Func_Str Func_repeat() : Func_Str("repeat") { } - virtual ~Func_repeat() - { - } + ~Func_repeat() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_inet_ntoa class to convert big-indian (network ordered) int to @@ -846,39 +767,37 @@ class Func_inet_ntoa : public Func_Str Func_inet_ntoa() : Func_Str("inet_ntoa") { } - virtual ~Func_inet_ntoa() - { - } + ~Func_inet_ntoa() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int32_t getDateIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; private: void convertNtoa(int64_t ipNum, std::string& ipString); @@ -892,15 +811,13 @@ class Func_reverse : public Func_Str Func_reverse() : Func_Str("reverse") { } - virtual ~Func_reverse() - { - } + ~Func_reverse() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_insert class @@ -911,15 +828,13 @@ class Func_insert : public Func_Str Func_insert() : Func_Str("insert") { } - virtual ~Func_insert() - { - } + ~Func_insert() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_maketime class @@ -930,15 +845,13 @@ class Func_maketime : public Func_Str Func_maketime() : Func_Str("maketime") { } - virtual ~Func_maketime() - { - } + ~Func_maketime() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_get_format class @@ -949,15 +862,13 @@ class Func_get_format : public Func_Str Func_get_format() : Func_Str("get_format") { } - virtual ~Func_get_format() - { - } + ~Func_get_format() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_elt class @@ -968,15 +879,13 @@ class Func_elt : public Func_Str Func_elt() : Func_Str("elt") { } - virtual ~Func_elt() - { - } + ~Func_elt() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_sha class @@ -987,15 +896,13 @@ class Func_sha : public Func_Str Func_sha() : Func_Str("sha") { } - virtual ~Func_sha() - { - } + ~Func_sha() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_idbpartition class @@ -1006,15 +913,13 @@ class Func_idbpartition : public Func_Str Func_idbpartition() : Func_Str("idbpartition") { } - virtual ~Func_idbpartition() - { - } + ~Func_idbpartition() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_space class @@ -1025,15 +930,13 @@ class Func_space : public Func_Str Func_space() : Func_Str("space") { } - virtual ~Func_space() - { - } + ~Func_space() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_quote class @@ -1044,15 +947,13 @@ class Func_quote : public Func_Str Func_quote() : Func_Str("quote") { } - virtual ~Func_quote() - { - } + ~Func_quote() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @brief Func_encode class @@ -1064,15 +965,13 @@ class Func_encode : public Func_Str Func_encode() : Func_Str("encode"), fSeeded(false), fSeeds{0, 0} { } - virtual ~Func_encode() - { - } + ~Func_encode() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; void resetSeed() { @@ -1097,15 +996,13 @@ class Func_decode : public Func_Str Func_decode() : Func_Str("decode"), fSeeded(false), fSeeds{0, 0} { } - virtual ~Func_decode() - { - } + ~Func_decode() override = default; - execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; std::string getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::CalpontSystemCatalog::ColType& op_ct) override; void resetSeed() { diff --git a/utils/funcexp/jsonhelpers.h b/utils/funcexp/jsonhelpers.h index ca8b62a66..7e11c574f 100644 --- a/utils/funcexp/jsonhelpers.h +++ b/utils/funcexp/jsonhelpers.h @@ -8,7 +8,7 @@ #include #include #include -//#include +// #include #include "collation.h" #include "functor_json.h" @@ -21,9 +21,7 @@ #include "json_lib.h" -namespace funcexp -{ -namespace helpers +namespace funcexp::helpers { static const int NO_WILDCARD_ALLOWED = 1; @@ -35,7 +33,8 @@ static const int NO_WILDCARD_ALLOWED = 1; int setupJSPath(json_path_t* path, CHARSET_INFO* cs, const string_view& str, bool wildcards); // Return true if err occur, let the outer function handle the exception -bool appendEscapedJS(string& ret, const CHARSET_INFO* retCS, const utils::NullString& js, const CHARSET_INFO* jsCS); +bool appendEscapedJS(string& ret, const CHARSET_INFO* retCS, const utils::NullString& js, + const CHARSET_INFO* jsCS); bool appendJSKeyName(string& ret, const CHARSET_INFO* retCS, rowgroup::Row& row, execplan::SPTP& parm); bool appendJSValue(string& ret, const CHARSET_INFO* retCS, rowgroup::Row& row, execplan::SPTP& parm); @@ -97,12 +96,11 @@ int parseJSPath(JSONPath& path, rowgroup::Row& row, execplan::SPTP& parm, bool w inline void initJSPaths(vector& paths, FunctionParm& fp, const int start, const int step) { - if (paths.size() == 0) + if (paths.empty()) for (size_t i = start; i < fp.size(); i += step) - paths.push_back(JSONPath{}); + paths.emplace_back(); } bool matchJSPath(const vector& paths, const json_path_t* p, json_value_types valType, const int* arrayCounter = nullptr, bool exact = true); -} // namespace helpers -} // namespace funcexp +} // namespace funcexp::helpers diff --git a/utils/funcexp/sql_crypt.h b/utils/funcexp/sql_crypt.h index 8984b7d74..b625b4c67 100644 --- a/utils/funcexp/sql_crypt.h +++ b/utils/funcexp/sql_crypt.h @@ -1,6 +1,6 @@ #pragma once -//#include "my_global.h" +// #include "my_global.h" /* Macros to make switching between C and C++ mode easier */ #ifdef __cplusplus #define C_MODE_START \ @@ -22,16 +22,12 @@ class SQL_CRYPT uint shift; public: - SQL_CRYPT() - { - } - SQL_CRYPT(ulong* seed) + SQL_CRYPT() = default; + explicit SQL_CRYPT(ulong* seed) { init(seed); } - ~SQL_CRYPT() - { - } + ~SQL_CRYPT() = default; void init(ulong* seed); void reinit() { diff --git a/utils/funcexp/timeextract.h b/utils/funcexp/timeextract.h index 181ae5021..96aa21663 100644 --- a/utils/funcexp/timeextract.h +++ b/utils/funcexp/timeextract.h @@ -32,7 +32,6 @@ class TimeExtractor public: TimeExtractor() : dayOfWeek(-1), dayOfYear(-1), weekOfYear(-1), sundayFirst(false) { - ; } /** diff --git a/utils/funcexp/utf8/checked.h b/utils/funcexp/utf8/checked.h index 65a4f8efe..064077bdc 100644 --- a/utils/funcexp/utf8/checked.h +++ b/utils/funcexp/utf8/checked.h @@ -42,10 +42,10 @@ class invalid_code_point : public exception uint32_t cp; public: - invalid_code_point(uint32_t cp) : cp(cp) + explicit invalid_code_point(uint32_t cp) : cp(cp) { } - virtual const char* what() const throw() + const char* what() const noexcept override { return "Invalid code point"; } @@ -60,10 +60,10 @@ class invalid_utf8 : public exception uint8_t u8; public: - invalid_utf8(uint8_t u) : u8(u) + explicit invalid_utf8(uint8_t u) : u8(u) { } - virtual const char* what() const throw() + const char* what() const noexcept override { return "Invalid UTF-8"; } @@ -78,10 +78,10 @@ class invalid_utf16 : public exception uint16_t u16; public: - invalid_utf16(uint16_t u) : u16(u) + explicit invalid_utf16(uint16_t u) : u16(u) { } - virtual const char* what() const throw() + const char* what() const noexcept override { return "Invalid UTF-16"; } @@ -94,7 +94,7 @@ class invalid_utf16 : public exception class not_enough_room : public exception { public: - virtual const char* what() const throw() + const char* what() const noexcept override { return "Not enough space"; } @@ -336,9 +336,7 @@ class iterator : public std::iterator octet_iterator range_end; public: - iterator() - { - } + iterator() = default; explicit iterator(const octet_iterator& octet_it, const octet_iterator& range_start, const octet_iterator& range_end) : it(octet_it), range_start(range_start), range_end(range_end) diff --git a/utils/funcexp/utf8/unchecked.h b/utils/funcexp/utf8/unchecked.h index 4995671b2..2faaa6857 100644 --- a/utils/funcexp/utf8/unchecked.h +++ b/utils/funcexp/utf8/unchecked.h @@ -28,9 +28,7 @@ DEALINGS IN THE SOFTWARE. #include "core.h" -namespace utf8 -{ -namespace unchecked +namespace utf8::unchecked { template octet_iterator append(uint32_t cp, octet_iterator result) @@ -201,9 +199,7 @@ class iterator : public std::iterator octet_iterator it; public: - iterator() - { - } + iterator() = default; explicit iterator(const octet_iterator& octet_it) : it(octet_it) { } @@ -249,5 +245,4 @@ class iterator : public std::iterator } }; // class iterator -} // namespace unchecked -} // namespace utf8 +} // namespace utf8::unchecked diff --git a/utils/idbdatafile/BufferedFile.h b/utils/idbdatafile/BufferedFile.h index e153c524b..ad1100e12 100644 --- a/utils/idbdatafile/BufferedFile.h +++ b/utils/idbdatafile/BufferedFile.h @@ -22,7 +22,6 @@ #include "IDBDataFile.h" #include - namespace idbdatafile { /** @@ -34,22 +33,22 @@ class BufferedFile : public IDBDataFile, boost::noncopyable { public: BufferedFile(const char* fname, const char* mode, unsigned opts); - /* virtual */ ~BufferedFile(); + /* virtual */ ~BufferedFile() override; - /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count); - /* virtual */ ssize_t read(void* ptr, size_t count); - /* virtual */ ssize_t write(const void* ptr, size_t count); - /* virtual */ int seek(off64_t offset, int whence); - /* virtual */ int truncate(off64_t length); - /* virtual */ off64_t size(); - /* virtual */ off64_t tell(); - /* virtual */ int flush(); - /* virtual */ time_t mtime(); - /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); + /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count) override; + /* virtual */ ssize_t read(void* ptr, size_t count) override; + /* virtual */ ssize_t write(const void* ptr, size_t count) override; + /* virtual */ int seek(off64_t offset, int whence) override; + /* virtual */ int truncate(off64_t length) override; + /* virtual */ off64_t size() override; + /* virtual */ off64_t tell() override; + /* virtual */ int flush() override; + /* virtual */ time_t mtime() override; + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length) override; protected: /* virtual */ - int close(); + int close() override; private: void applyOptions(unsigned opts); diff --git a/utils/idbdatafile/BufferedFileFactory.h b/utils/idbdatafile/BufferedFileFactory.h index 74355d61a..a061750dd 100644 --- a/utils/idbdatafile/BufferedFileFactory.h +++ b/utils/idbdatafile/BufferedFileFactory.h @@ -27,7 +27,7 @@ class BufferedFileFactory : public FileFactoryBase { public: /* virtual */ - IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); + IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth) override; }; inline IDBDataFile* BufferedFileFactory::open(const char* fname, const char* mode, unsigned opts, diff --git a/utils/idbdatafile/IDBDataFile.h b/utils/idbdatafile/IDBDataFile.h index c8cd4a210..ece8d6e95 100644 --- a/utils/idbdatafile/IDBDataFile.h +++ b/utils/idbdatafile/IDBDataFile.h @@ -201,7 +201,7 @@ class IDBDataFile * Constructor - takes the filename to be stored in a member variable * for logging purposes */ - IDBDataFile(const char* fname); + explicit IDBDataFile(const char* fname); /** * The close() method closes the file. It is defined as protected @@ -226,9 +226,7 @@ inline IDBDataFile::IDBDataFile(const char* fname) : m_fname(fname), m_fColWidth { } -inline IDBDataFile::~IDBDataFile() -{ -} +inline IDBDataFile::~IDBDataFile() = default; inline const std::string& IDBDataFile::name() const { diff --git a/utils/idbdatafile/IDBFileSystem.h b/utils/idbdatafile/IDBFileSystem.h index 9a9267e2f..78f405230 100644 --- a/utils/idbdatafile/IDBFileSystem.h +++ b/utils/idbdatafile/IDBFileSystem.h @@ -142,7 +142,7 @@ class IDBFileSystem } protected: - IDBFileSystem(Types type); + explicit IDBFileSystem(Types type); private: Types m_type; diff --git a/utils/idbdatafile/PosixFileSystem.h b/utils/idbdatafile/PosixFileSystem.h index 46f9423f3..8a1ceec52 100644 --- a/utils/idbdatafile/PosixFileSystem.h +++ b/utils/idbdatafile/PosixFileSystem.h @@ -25,7 +25,7 @@ class PosixFileSystem : public IDBFileSystem { public: PosixFileSystem(); - ~PosixFileSystem(); + ~PosixFileSystem() override; int mkdir(const char* pathname) override; off64_t size(const char* path) const override; diff --git a/utils/idbdatafile/UnbufferedFile.h b/utils/idbdatafile/UnbufferedFile.h index cd100c95b..3c809464c 100644 --- a/utils/idbdatafile/UnbufferedFile.h +++ b/utils/idbdatafile/UnbufferedFile.h @@ -34,22 +34,22 @@ class UnbufferedFile : public IDBDataFile, boost::noncopyable { public: UnbufferedFile(const char* fname, const char* mode, unsigned opts); - /* virtual */ ~UnbufferedFile(); + /* virtual */ ~UnbufferedFile() override; - /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count); - /* virtual */ ssize_t read(void* ptr, size_t count); - /* virtual */ ssize_t write(const void* ptr, size_t count); - /* virtual */ int seek(off64_t offset, int whence); - /* virtual */ int truncate(off64_t length); - /* virtual */ off64_t size(); - /* virtual */ off64_t tell(); - /* virtual */ int flush(); - /* virtual */ time_t mtime(); - /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); + /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count) override; + /* virtual */ ssize_t read(void* ptr, size_t count) override; + /* virtual */ ssize_t write(const void* ptr, size_t count) override; + /* virtual */ int seek(off64_t offset, int whence) override; + /* virtual */ int truncate(off64_t length) override; + /* virtual */ off64_t size() override; + /* virtual */ off64_t tell() override; + /* virtual */ int flush() override; + /* virtual */ time_t mtime() override; + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length) override; protected: /* virtual */ - int close(); + int close() override; private: int m_fd; diff --git a/utils/idbdatafile/UnbufferedFileFactory.h b/utils/idbdatafile/UnbufferedFileFactory.h index 4933fc469..5e4313994 100644 --- a/utils/idbdatafile/UnbufferedFileFactory.h +++ b/utils/idbdatafile/UnbufferedFileFactory.h @@ -27,7 +27,7 @@ class UnbufferedFileFactory : public FileFactoryBase { public: /* virtual */ - IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth); + IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth) override; }; inline IDBDataFile* UnbufferedFileFactory::open(const char* fname, const char* mode, unsigned opts, diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index ff9606298..84256cf0f 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -81,7 +81,7 @@ class ByteStream : public Serializeable /** * ctor with a uint8_t array and len initializer */ - inline ByteStream(const uint8_t* bp, const BSSizeType len); + inline ByteStream(const uint8_t* bp, BSSizeType len); /** * copy ctor */ @@ -98,40 +98,40 @@ class ByteStream : public Serializeable /** * dtor */ - inline virtual ~ByteStream(); + inline ~ByteStream() override; /** * push a int8_t onto the end of the stream */ - EXPORT ByteStream& operator<<(const int8_t b); + EXPORT ByteStream& operator<<(int8_t b); /** * push a uint8_t onto the end of the stream */ - EXPORT ByteStream& operator<<(const uint8_t b); + EXPORT ByteStream& operator<<(uint8_t b); /** * push a int16_t onto the end of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const int16_t d); + EXPORT ByteStream& operator<<(int16_t d); /** * push a uint16_t onto the end of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const uint16_t d); + EXPORT ByteStream& operator<<(uint16_t d); /** * push a int32_t onto the end of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const int32_t q); + EXPORT ByteStream& operator<<(int32_t q); /** * push a uint32_t onto the end of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const uint32_t q); + EXPORT ByteStream& operator<<(uint32_t q); /** * push an int64_t onto the end of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const int64_t o); + EXPORT ByteStream& operator<<(int64_t o); /** * push an uint64_t onto the end of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const uint64_t o); + EXPORT ByteStream& operator<<(uint64_t o); /** * push an int128_t onto the end of the stream. The byte order is whatever the native byte order is. */ @@ -145,17 +145,17 @@ class ByteStream : public Serializeable * push a float onto the end of the stream. The byte order is * whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const float f); + EXPORT ByteStream& operator<<(float f); /** * push a double onto the end of the stream. The byte order is * whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const double d); + EXPORT ByteStream& operator<<(double d); /** * push a long double onto the end of the stream. The byte * order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const long double d); + EXPORT ByteStream& operator<<(long double d); /** * push a std::string onto the end of the stream. */ @@ -428,12 +428,12 @@ class ByteStream : public Serializeable /** * Serializeable interface */ - EXPORT void serialize(ByteStream& bs) const; + EXPORT void serialize(ByteStream& bs) const override; /** * Serializeable interface */ - EXPORT void deserialize(ByteStream& bs); + EXPORT void deserialize(ByteStream& bs) override; /** * memory allocation chunk size @@ -455,7 +455,7 @@ class ByteStream : public Serializeable /** * pushes one uint8_t onto the end of the stream */ - void add(const uint8_t b); + void add(uint8_t b); /** * adds another BlockSize bytes to the internal buffer */ @@ -527,7 +527,7 @@ static const uint8_t BS_BLOB = 9; static const uint8_t BS_SERIALIZABLE = 10; static const uint8_t BS_UUID = 11; -inline ByteStream::ByteStream(const uint8_t* bp, const BSSizeType len) : fBuf(0), fMaxLen(0) +inline ByteStream::ByteStream(const uint8_t* bp, BSSizeType len) : fBuf(nullptr), fMaxLen(0) { load(bp, len); } @@ -560,7 +560,7 @@ inline void ByteStream::reset() { delete[] fBuf; fMaxLen = 0; - fCurInPtr = fCurOutPtr = fBuf = 0; + fCurInPtr = fCurOutPtr = fBuf = nullptr; } inline void ByteStream::restart() { @@ -669,7 +669,6 @@ void deserializeVector(ByteStream& bs, std::vector& v) } } - template void serializeInlineVector(ByteStream& bs, const std::vector& v) { @@ -703,7 +702,6 @@ void deserializeInlineVector(ByteStream& bs, std::vector& v) } } - inline void deserializeVector(ByteStream& bs, std::vector& v) { deserializeInlineVector(bs, v); diff --git a/utils/messageqcpp/compressed_iss.h b/utils/messageqcpp/compressed_iss.h index e86291414..ff839bb87 100644 --- a/utils/messageqcpp/compressed_iss.h +++ b/utils/messageqcpp/compressed_iss.h @@ -39,16 +39,16 @@ class CompressedInetStreamSocket : public InetStreamSocket public: CompressedInetStreamSocket(); CompressedInetStreamSocket(const CompressedInetStreamSocket&) = default; - virtual ~CompressedInetStreamSocket(){}; + ~CompressedInetStreamSocket() override = default; using InetStreamSocket::operator=; - virtual Socket* clone() const; - virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, - Stats* stats = NULL) const; - virtual void write(const ByteStream& msg, Stats* stats = NULL); - virtual void write(SBS msg, Stats* stats = NULL); - virtual const IOSocket accept(const struct timespec* timeout); - virtual void connect(const sockaddr* addr); + Socket* clone() const override; + const SBS read(const struct timespec* timeout = nullptr, bool* isTimeOut = nullptr, + Stats* stats = nullptr) const override; + void write(const ByteStream& msg, Stats* stats = nullptr) override; + void write(SBS msg, Stats* stats = nullptr) override; + const IOSocket accept(const struct timespec* timeout) override; + void connect(const sockaddr* addr) override; private: std::shared_ptr alg; diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index 7df141192..d42958c9a 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -73,8 +73,6 @@ using namespace std; #include using boost::scoped_array; - - #define INETSTREAMSOCKET_DLLEXPORT #include "inetstreamsocket.h" #undef INETSTREAMSOCKET_DLLEXPORT @@ -97,7 +95,6 @@ namespace // read(), so adding logic to retry after ERESTARTSYS the way we do for EINTR. // const int KERR_ERESTARTSYS = 512; - int in_cksum(unsigned short* buf, int sz) { int nleft = sz; @@ -135,9 +132,7 @@ InetStreamSocket::InetStreamSocket(size_t blocksize) fConnectionTimeout.tv_nsec = 0; } -InetStreamSocket::~InetStreamSocket() -{ -} +InetStreamSocket::~InetStreamSocket() = default; void InetStreamSocket::open() { @@ -172,7 +167,6 @@ void InetStreamSocket::open() throw runtime_error(msg); } - /* XXXPAT: If we have latency problems again, try these... bufferSizeSize = 4; bufferSize = 512000; @@ -379,9 +373,8 @@ bool InetStreamSocket::readToMagic(long msecs, bool* isTimeOut, Stats* stats) co } ostringstream oss; - oss << "InetStreamSocket::readToMagic(): I/O error2.1: " - << "err = " << err << " e = " << e << - ": " << strerror(e); + oss << "InetStreamSocket::readToMagic(): I/O error2.1: " << "err = " << err << " e = " << e << ": " + << strerror(e); throw runtime_error(oss.str()); } @@ -402,7 +395,7 @@ bool InetStreamSocket::readToMagic(long msecs, bool* isTimeOut, Stats* stats) co return true; } -bool InetStreamSocket::readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, const size_t numberOfBytes, +bool InetStreamSocket::readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, size_t numberOfBytes, const struct ::timespec* timeout, bool* isTimeOut, Stats* stats, int64_t msecs) const { @@ -412,7 +405,7 @@ bool InetStreamSocket::readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, co ssize_t currentBytesRead; int err; - if (timeout != NULL) + if (timeout != nullptr) { pfd[0].revents = 0; err = poll(pfd, 1, msecs); @@ -438,7 +431,7 @@ bool InetStreamSocket::readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, co if (currentBytesRead == 0) { - if (timeout == NULL) + if (timeout == nullptr) { logIoError("InetStreamSocket::read: timeout during first read", 0); return false; @@ -482,7 +475,7 @@ const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeO pfd[0].fd = fSocketParms.sd(); pfd[0].events = POLLIN; - if (timeout != 0) + if (timeout != nullptr) msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; if (readToMagic(msecs, isTimeOut, stats) == false) // indicates a timeout or EOF @@ -707,7 +700,7 @@ const IOSocket InetStreamSocket::accept(const struct timespec* timeout) pfd[0].fd = socketParms().sd(); pfd[0].events = POLLIN; - if (timeout != 0) + if (timeout != nullptr) { msecs = timeout->tv_sec * 1000 + timeout->tv_nsec / 1000000; @@ -771,7 +764,7 @@ const IOSocket InetStreamSocket::accept(const struct timespec* timeout) #if STRERROR_R_CHAR_P const char* p; - if ((p = strerror_r(e, blah, 80)) != 0) + if ((p = strerror_r(e, blah, 80)) != nullptr) os << "InetStreamSocket::accept sync: " << p; #else @@ -862,7 +855,7 @@ void InetStreamSocket::connect(const sockaddr* serv_addr) #if STRERROR_R_CHAR_P const char* p; - if ((p = strerror_r(e, blah, 80)) != 0) + if ((p = strerror_r(e, blah, 80)) != nullptr) os << "InetStreamSocket::connect: " << p; #else @@ -883,9 +876,9 @@ const string InetStreamSocket::toString() const ostringstream oss; char buf[INET_ADDRSTRLEN]; const SocketParms& sp = fSocketParms; - oss << "InetStreamSocket: sd: " << sp.sd() << - " inet: " << inet_ntop(AF_INET, &fSa.sin_addr, buf, INET_ADDRSTRLEN) << - " port: " << ntohs(fSa.sin_port); + oss << "InetStreamSocket: sd: " << sp.sd() + << " inet: " << inet_ntop(AF_INET, &fSa.sin_addr, buf, INET_ADDRSTRLEN) + << " port: " << ntohs(fSa.sin_port); return oss.str(); } @@ -1030,7 +1023,7 @@ int InetStreamSocket::ping(const std::string& ipaddr, const struct timespec* tim return -1; } - len = ::recvfrom(pingsock, pkt, pktlen, 0, 0, 0); + len = ::recvfrom(pingsock, pkt, pktlen, 0, nullptr, nullptr); if (len < 76) { @@ -1050,7 +1043,6 @@ int InetStreamSocket::ping(const std::string& ipaddr, const struct timespec* tim ::close(pingsock); - return 0; } diff --git a/utils/messageqcpp/inetstreamsocket.h b/utils/messageqcpp/inetstreamsocket.h index fa9a94915..e09b71a6e 100644 --- a/utils/messageqcpp/inetstreamsocket.h +++ b/utils/messageqcpp/inetstreamsocket.h @@ -58,7 +58,7 @@ class InetStreamSocket : public Socket /** dtor * */ - virtual ~InetStreamSocket(); + ~InetStreamSocket() override; /** copy ctor * @@ -68,37 +68,37 @@ class InetStreamSocket : public Socket /** assign op * */ - virtual InetStreamSocket& operator=(const InetStreamSocket& rhs); + InetStreamSocket& operator=(const InetStreamSocket& rhs); /** fSocket mutator * */ - inline virtual void socketParms(const SocketParms& socket); + inline void socketParms(const SocketParms& socket) override; /** fSocket accessor * */ - inline virtual const SocketParms socketParms() const; + inline const SocketParms socketParms() const override; /** sockaddr mutator * */ - inline virtual void sa(const sockaddr* sa); + inline void sa(const sockaddr* sa) override; /** call socket() to get a sd * */ - virtual void open(); + void open() override; /** close the sd * */ - virtual void close(); + void close() override; /** test if this socket is open * */ - inline virtual bool isOpen() const; + inline bool isOpen() const override; /** read a message from the socket * @@ -114,44 +114,44 @@ class InetStreamSocket : public Socket * The behavior will be unpredictable and possibly fatal. * @note A fix is being reviewed but this is low-priority. */ - virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, - Stats* stats = NULL) const; + const SBS read(const struct timespec* timeout = nullptr, bool* isTimeOut = nullptr, + Stats* stats = nullptr) const override; /** write a message to the socket * * write a message to the socket */ - virtual void write(const ByteStream& msg, Stats* stats = NULL); - virtual void write_raw(const ByteStream& msg, Stats* stats = NULL) const; + void write(const ByteStream& msg, Stats* stats = nullptr) override; + void write_raw(const ByteStream& msg, Stats* stats = nullptr) const override; /** this version of write takes ownership of the bytestream */ - virtual void write(SBS msg, Stats* stats = NULL); + void write(SBS msg, Stats* stats = nullptr) override; /** bind to a port * */ - virtual void bind(const sockaddr* serv_addr); + void bind(const sockaddr* serv_addr) override; /** listen for connections * */ - virtual void listen(int backlog = 5); + void listen(int backlog = 5) override; /** return an (accepted) IOSocket ready for I/O * */ - virtual const IOSocket accept(const struct timespec* timeout = 0); + const IOSocket accept(const struct timespec* timeout = nullptr) override; /** connect to a server socket * */ - virtual void connect(const sockaddr* serv_addr); + void connect(const sockaddr* serv_addr) override; /** dynamically allocate a copy of this object * */ - virtual Socket* clone() const; + Socket* clone() const override; /** get a string rep of the object * @@ -161,7 +161,7 @@ class InetStreamSocket : public Socket /** set the connection timeout (in ms) * */ - virtual void connectionTimeout(const struct ::timespec* timeout) + void connectionTimeout(const struct ::timespec* timeout) override { if (timeout) fConnectionTimeout = *timeout; @@ -170,12 +170,12 @@ class InetStreamSocket : public Socket /** set the connection protocol to be synchronous * */ - virtual void syncProto(bool use) + void syncProto(bool use) override { fSyncProto = use; } - int getConnectionNum() const + int getConnectionNum() const override { return fSocketParms.sd(); } @@ -189,25 +189,25 @@ class InetStreamSocket : public Socket /** return the address as a string * */ - virtual const std::string addr2String() const; + const std::string addr2String() const override; /** compare 2 addresses * */ - virtual bool isSameAddr(const Socket* rhs) const; - virtual bool isSameAddr(const struct in_addr& ipv4Addr) const; + bool isSameAddr(const Socket* rhs) const override; + bool isSameAddr(const struct in_addr& ipv4Addr) const override; /** ping an ip address * */ - EXPORT static int ping(const std::string& ipaddr, const struct timespec* timeout = 0); + EXPORT static int ping(const std::string& ipaddr, const struct timespec* timeout = nullptr); // Check if we are still connected - virtual bool isConnected() const; + bool isConnected() const override; // Check if the socket still has data pending - virtual bool hasData() const; + bool hasData() const override; /* * allow test suite access to private data for OOB test @@ -231,9 +231,9 @@ class InetStreamSocket : public Socket */ virtual bool readToMagic(long msecs, bool* isTimeOut, Stats* stats) const; - void do_write(const ByteStream& msg, uint32_t magic, Stats* stats = NULL) const; + void do_write(const ByteStream& msg, uint32_t magic, Stats* stats = nullptr) const; ssize_t written(int fd, const uint8_t* ptr, size_t nbytes) const; - bool readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, const size_t numberOfBytes, + bool readFixedSizeData(struct pollfd* pfd, uint8_t* buffer, size_t numberOfBytes, const struct ::timespec* timeout, bool* isTimeOut, Stats* stats, int64_t msec) const; SocketParms fSocketParms; /// The socket parms diff --git a/utils/messageqcpp/messagequeue.h b/utils/messageqcpp/messagequeue.h index 197a27e3b..c089070f2 100644 --- a/utils/messageqcpp/messagequeue.h +++ b/utils/messageqcpp/messagequeue.h @@ -76,7 +76,7 @@ class MessageQueueServer * * construct a server queue for thisEnd. Optionally specify a Config object to use. */ - EXPORT explicit MessageQueueServer(const std::string& thisEnd, config::Config* config = 0, + EXPORT explicit MessageQueueServer(const std::string& thisEnd, config::Config* config = nullptr, size_t blocksize = ByteStream::BlockSize, int backlog = 5, bool syncProto = true); @@ -101,7 +101,7 @@ class MessageQueueServer * is then free to wait again for another connection. The IOSocket is already open and ready for * read() and/or write(). The caller is responsible for calling close() when it is done. */ - EXPORT const IOSocket accept(const struct timespec* timeout = 0) const; + EXPORT const IOSocket accept(const struct timespec* timeout = nullptr) const; /** * @brief get a mutable pointer to the client IOSocket @@ -165,7 +165,7 @@ class MessageQueueClient * * construct a queue from this process to otherEnd. Optionally specify a Config object to use. */ - EXPORT explicit MessageQueueClient(const std::string& otherEnd, config::Config* config = 0, + EXPORT explicit MessageQueueClient(const std::string& otherEnd, config::Config* config = nullptr, bool syncProto = true); /** @@ -196,8 +196,8 @@ class MessageQueueClient * wait for and return a message from otherEnd. The deafult timeout waits forever. Note that * eventhough struct timespec has nanosecond resolution, this method only has milisecond resolution. */ - EXPORT const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, - Stats* stats = NULL) const; + EXPORT const SBS read(const struct timespec* timeout = nullptr, bool* isTimeOut = nullptr, + Stats* stats = nullptr) const; /** * @brief write a message to the queue @@ -205,7 +205,8 @@ class MessageQueueClient * write a message to otherEnd. If the socket is not open, the timeout parm (in ms) will be used * to establish a sync connection w/ the server */ - EXPORT void write(const ByteStream& msg, const struct timespec* timeout = 0, Stats* stats = NULL) const; + EXPORT void write(const ByteStream& msg, const struct timespec* timeout = nullptr, + Stats* stats = nullptr) const; /** * @brief shutdown the connection to the server diff --git a/utils/messageqcpp/samenodepseudosocket.h b/utils/messageqcpp/samenodepseudosocket.h index f977f02a2..df8ff72e3 100644 --- a/utils/messageqcpp/samenodepseudosocket.h +++ b/utils/messageqcpp/samenodepseudosocket.h @@ -33,56 +33,56 @@ class SameNodePseudoSocket : public Socket { public: explicit SameNodePseudoSocket(joblist::DistributedEngineComm* exeMgrDecPtr); - virtual ~SameNodePseudoSocket(); - virtual void write(SBS msg, Stats* stats = NULL); + ~SameNodePseudoSocket() override; + void write(SBS msg, Stats* stats = nullptr) override; private: - virtual void bind(const sockaddr* serv_addr); + void bind(const sockaddr* serv_addr) override; SameNodePseudoSocket(const SameNodePseudoSocket& rhs); virtual SameNodePseudoSocket& operator=(const SameNodePseudoSocket& rhs); - virtual void connectionTimeout(const struct ::timespec* timeout) + void connectionTimeout(const struct ::timespec* timeout) override { } - virtual void syncProto(bool use) + void syncProto(bool use) override { } - int getConnectionNum() const + int getConnectionNum() const override { return 1; } - inline virtual void socketParms(const SocketParms& socket) + inline void socketParms(const SocketParms& socket) override { } - inline virtual const SocketParms socketParms() const + inline const SocketParms socketParms() const override { return SocketParms(); } // all these virtual methods are to stay inaccessable. - inline virtual void sa(const sockaddr* sa); - virtual void open(); - virtual void close(); - inline virtual bool isOpen() const; - virtual const SBS read(const struct timespec* timeout = 0, bool* isTimeOut = NULL, - Stats* stats = NULL) const; - virtual void write(const ByteStream& msg, Stats* stats = NULL); - virtual void write_raw(const ByteStream& msg, Stats* stats = NULL) const; - virtual void listen(int backlog = 5); - virtual const IOSocket accept(const struct timespec* timeout = 0); - virtual void connect(const sockaddr* serv_addr); - virtual Socket* clone() const; + inline void sa(const sockaddr* sa) override; + void open() override; + void close() override; + inline bool isOpen() const override; + const SBS read(const struct timespec* timeout = nullptr, bool* isTimeOut = nullptr, + Stats* stats = nullptr) const override; + void write(const ByteStream& msg, Stats* stats = nullptr) override; + void write_raw(const ByteStream& msg, Stats* stats = nullptr) const override; + void listen(int backlog = 5) override; + const IOSocket accept(const struct timespec* timeout = nullptr) override; + void connect(const sockaddr* serv_addr) override; + Socket* clone() const override; virtual const std::string toString() const; - virtual const std::string addr2String() const; - virtual bool isSameAddr(const Socket* rhs) const; - virtual bool isSameAddr(const struct in_addr& ipv4Addr) const; - static int ping(const std::string& ipaddr, const struct timespec* timeout = 0); - virtual bool isConnected() const; - virtual bool hasData() const; + const std::string addr2String() const override; + bool isSameAddr(const Socket* rhs) const override; + bool isSameAddr(const struct in_addr& ipv4Addr) const override; + static int ping(const std::string& ipaddr, const struct timespec* timeout = nullptr); + bool isConnected() const override; + bool hasData() const override; joblist::DistributedEngineComm* dec_ = nullptr; }; diff --git a/utils/messageqcpp/socketclosed.h b/utils/messageqcpp/socketclosed.h index 4bc6e11e0..fb1a61fe6 100644 --- a/utils/messageqcpp/socketclosed.h +++ b/utils/messageqcpp/socketclosed.h @@ -24,6 +24,7 @@ #include #include +#include #pragma once @@ -35,23 +36,21 @@ namespace messageqcpp */ class SocketClosed : public std::exception { - std::string _M_msg; + std::string M_msg; public: /** Takes a character string describing the error. */ - explicit SocketClosed(const std::string& __arg) : _M_msg(__arg) + explicit SocketClosed(std::string _arg) : M_msg(std::move(_arg)) { } - virtual ~SocketClosed() throw() - { - } + ~SocketClosed() noexcept override = default; /** Returns a C-style character string describing the general cause of * the current error (the same string passed to the ctor). */ - virtual const char* what() const throw() + const char* what() const noexcept override { - return _M_msg.c_str(); + return M_msg.c_str(); } }; diff --git a/utils/querytele/QueryTeleService.h b/utils/querytele/QueryTeleService.h index ee02bb45e..a24068be3 100644 --- a/utils/querytele/QueryTeleService.h +++ b/utils/querytele/QueryTeleService.h @@ -9,15 +9,12 @@ #include #include "querytele_types.h" - namespace querytele { class QueryTeleServiceIf { public: - virtual ~QueryTeleServiceIf() - { - } + virtual ~QueryTeleServiceIf() = default; virtual void postQuery(const QueryTele& query) = 0; virtual void postStep(const StepTele& query) = 0; virtual void postImport(const ImportTele& query) = 0; @@ -28,9 +25,7 @@ class QueryTeleServiceIfFactory public: typedef QueryTeleServiceIf Handler; - virtual ~QueryTeleServiceIfFactory() - { - } + virtual ~QueryTeleServiceIfFactory() = default; virtual QueryTeleServiceIf* getHandler(const ::apache::thrift::TConnectionInfo& connInfo) = 0; virtual void releaseHandler(QueryTeleServiceIf* /* handler */) = 0; @@ -39,18 +34,17 @@ class QueryTeleServiceIfFactory class QueryTeleServiceIfSingletonFactory : virtual public QueryTeleServiceIfFactory { public: - QueryTeleServiceIfSingletonFactory(const std::shared_ptr& iface) : iface_(iface) - { - } - virtual ~QueryTeleServiceIfSingletonFactory() + explicit QueryTeleServiceIfSingletonFactory(const std::shared_ptr& iface) + : iface_(iface) { } + ~QueryTeleServiceIfSingletonFactory() override = default; - virtual QueryTeleServiceIf* getHandler(const ::apache::thrift::TConnectionInfo&) + QueryTeleServiceIf* getHandler(const ::apache::thrift::TConnectionInfo&) override { return iface_.get(); } - virtual void releaseHandler(QueryTeleServiceIf* /* handler */) + void releaseHandler(QueryTeleServiceIf* /* handler */) override { } @@ -61,18 +55,16 @@ class QueryTeleServiceIfSingletonFactory : virtual public QueryTeleServiceIfFact class QueryTeleServiceNull : virtual public QueryTeleServiceIf { public: - virtual ~QueryTeleServiceNull() - { - } - void postQuery(const QueryTele& /* query */) + ~QueryTeleServiceNull() override = default; + void postQuery(const QueryTele& /* query */) override { return; } - void postStep(const StepTele& /* query */) + void postStep(const StepTele& /* query */) override { return; } - void postImport(const ImportTele& /* query */) + void postImport(const ImportTele& /* query */) override { return; } @@ -89,13 +81,9 @@ typedef struct _QueryTeleService_postQuery_args__isset class QueryTeleService_postQuery_args { public: - QueryTeleService_postQuery_args() - { - } + QueryTeleService_postQuery_args() = default; - virtual ~QueryTeleService_postQuery_args() throw() - { - } + virtual ~QueryTeleService_postQuery_args() throw() = default; QueryTele query; @@ -127,9 +115,7 @@ class QueryTeleService_postQuery_args class QueryTeleService_postQuery_pargs { public: - virtual ~QueryTeleService_postQuery_pargs() throw() - { - } + virtual ~QueryTeleService_postQuery_pargs() throw() = default; const QueryTele* query; @@ -139,13 +125,9 @@ class QueryTeleService_postQuery_pargs class QueryTeleService_postQuery_result { public: - QueryTeleService_postQuery_result() - { - } + QueryTeleService_postQuery_result() = default; - virtual ~QueryTeleService_postQuery_result() throw() - { - } + virtual ~QueryTeleService_postQuery_result() throw() = default; bool operator==(const QueryTeleService_postQuery_result& /* rhs */) const { @@ -165,9 +147,7 @@ class QueryTeleService_postQuery_result class QueryTeleService_postQuery_presult { public: - virtual ~QueryTeleService_postQuery_presult() throw() - { - } + virtual ~QueryTeleService_postQuery_presult() throw() = default; uint32_t read(::apache::thrift::protocol::TProtocol* iprot); }; @@ -183,13 +163,9 @@ typedef struct _QueryTeleService_postStep_args__isset class QueryTeleService_postStep_args { public: - QueryTeleService_postStep_args() - { - } + QueryTeleService_postStep_args() = default; - virtual ~QueryTeleService_postStep_args() throw() - { - } + virtual ~QueryTeleService_postStep_args() throw() = default; StepTele query; @@ -221,9 +197,7 @@ class QueryTeleService_postStep_args class QueryTeleService_postStep_pargs { public: - virtual ~QueryTeleService_postStep_pargs() throw() - { - } + virtual ~QueryTeleService_postStep_pargs() throw() = default; const StepTele* query; @@ -233,13 +207,9 @@ class QueryTeleService_postStep_pargs class QueryTeleService_postStep_result { public: - QueryTeleService_postStep_result() - { - } + QueryTeleService_postStep_result() = default; - virtual ~QueryTeleService_postStep_result() throw() - { - } + virtual ~QueryTeleService_postStep_result() throw() = default; bool operator==(const QueryTeleService_postStep_result& /* rhs */) const { @@ -259,9 +229,7 @@ class QueryTeleService_postStep_result class QueryTeleService_postStep_presult { public: - virtual ~QueryTeleService_postStep_presult() throw() - { - } + virtual ~QueryTeleService_postStep_presult() throw() = default; uint32_t read(::apache::thrift::protocol::TProtocol* iprot); }; @@ -277,13 +245,9 @@ typedef struct _QueryTeleService_postImport_args__isset class QueryTeleService_postImport_args { public: - QueryTeleService_postImport_args() - { - } + QueryTeleService_postImport_args() = default; - virtual ~QueryTeleService_postImport_args() throw() - { - } + virtual ~QueryTeleService_postImport_args() throw() = default; ImportTele query; @@ -315,9 +279,7 @@ class QueryTeleService_postImport_args class QueryTeleService_postImport_pargs { public: - virtual ~QueryTeleService_postImport_pargs() throw() - { - } + virtual ~QueryTeleService_postImport_pargs() throw() = default; const ImportTele* query; @@ -327,13 +289,9 @@ class QueryTeleService_postImport_pargs class QueryTeleService_postImport_result { public: - QueryTeleService_postImport_result() - { - } + QueryTeleService_postImport_result() = default; - virtual ~QueryTeleService_postImport_result() throw() - { - } + virtual ~QueryTeleService_postImport_result() throw() = default; bool operator==(const QueryTeleService_postImport_result& /* rhs */) const { @@ -353,9 +311,7 @@ class QueryTeleService_postImport_result class QueryTeleService_postImport_presult { public: - virtual ~QueryTeleService_postImport_presult() throw() - { - } + virtual ~QueryTeleService_postImport_presult() throw() = default; uint32_t read(::apache::thrift::protocol::TProtocol* iprot); }; @@ -363,7 +319,7 @@ class QueryTeleService_postImport_presult class QueryTeleServiceClient : virtual public QueryTeleServiceIf { public: - QueryTeleServiceClient(std::shared_ptr< ::apache::thrift::protocol::TProtocol> prot) + explicit QueryTeleServiceClient(std::shared_ptr< ::apache::thrift::protocol::TProtocol> prot) : piprot_(prot), poprot_(prot) { iprot_ = prot.get(); @@ -384,13 +340,13 @@ class QueryTeleServiceClient : virtual public QueryTeleServiceIf { return poprot_; } - void postQuery(const QueryTele& query); + void postQuery(const QueryTele& query) override; void send_postQuery(const QueryTele& query); void recv_postQuery(); - void postStep(const StepTele& query); + void postStep(const StepTele& query) override; void send_postStep(const StepTele& query); void recv_postStep(); - void postImport(const ImportTele& query); + void postImport(const ImportTele& query) override; void send_postImport(const ImportTele& query); void recv_postImport(); @@ -405,9 +361,9 @@ class QueryTeleServiceProcessor : public ::apache::thrift::TDispatchProcessor { protected: std::shared_ptr iface_; - virtual bool dispatchCall(::apache::thrift::protocol::TProtocol* iprot, - ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, - int32_t seqid, void* callContext); + bool dispatchCall(::apache::thrift::protocol::TProtocol* iprot, + ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, int32_t seqid, + void* callContext) override; private: typedef void (QueryTeleServiceProcessor::*ProcessFunction)(int32_t, ::apache::thrift::protocol::TProtocol*, @@ -422,28 +378,27 @@ class QueryTeleServiceProcessor : public ::apache::thrift::TDispatchProcessor ::apache::thrift::protocol::TProtocol* oprot, void* callContext); public: - QueryTeleServiceProcessor(std::shared_ptr iface) : iface_(iface) + explicit QueryTeleServiceProcessor(std::shared_ptr iface) : iface_(iface) { processMap_["postQuery"] = &QueryTeleServiceProcessor::process_postQuery; processMap_["postStep"] = &QueryTeleServiceProcessor::process_postStep; processMap_["postImport"] = &QueryTeleServiceProcessor::process_postImport; } - virtual ~QueryTeleServiceProcessor() - { - } + ~QueryTeleServiceProcessor() override = default; }; class QueryTeleServiceProcessorFactory : public ::apache::thrift::TProcessorFactory { public: - QueryTeleServiceProcessorFactory(const ::std::shared_ptr& handlerFactory) + explicit QueryTeleServiceProcessorFactory( + const ::std::shared_ptr& handlerFactory) : handlerFactory_(handlerFactory) { } ::std::shared_ptr< ::apache::thrift::TProcessor> getProcessor( - const ::apache::thrift::TConnectionInfo& connInfo); + const ::apache::thrift::TConnectionInfo& connInfo) override; protected: ::std::shared_ptr handlerFactory_; @@ -452,25 +407,22 @@ class QueryTeleServiceProcessorFactory : public ::apache::thrift::TProcessorFact class QueryTeleServiceMultiface : virtual public QueryTeleServiceIf { public: - QueryTeleServiceMultiface(std::vector >& ifaces) : ifaces_(ifaces) - { - } - virtual ~QueryTeleServiceMultiface() + explicit QueryTeleServiceMultiface(std::vector >& ifaces) + : ifaces_(ifaces) { } + ~QueryTeleServiceMultiface() override = default; protected: std::vector > ifaces_; - QueryTeleServiceMultiface() - { - } + QueryTeleServiceMultiface() = default; void add(std::shared_ptr iface) { ifaces_.push_back(iface); } public: - void postQuery(const QueryTele& query) + void postQuery(const QueryTele& query) override { size_t sz = ifaces_.size(); size_t i = 0; @@ -483,7 +435,7 @@ class QueryTeleServiceMultiface : virtual public QueryTeleServiceIf ifaces_[i]->postQuery(query); } - void postStep(const StepTele& query) + void postStep(const StepTele& query) override { size_t sz = ifaces_.size(); size_t i = 0; @@ -496,7 +448,7 @@ class QueryTeleServiceMultiface : virtual public QueryTeleServiceIf ifaces_[i]->postStep(query); } - void postImport(const ImportTele& query) + void postImport(const ImportTele& query) override { size_t sz = ifaces_.size(); size_t i = 0; diff --git a/utils/querytele/querystepparms.h b/utils/querytele/querystepparms.h index 656f86ede..222ae7b75 100644 --- a/utils/querytele/querystepparms.h +++ b/utils/querytele/querystepparms.h @@ -27,9 +27,7 @@ class QueryStepParms explicit QueryStepParms(StepTeleStats::StepType st = StepTeleStats::T_INVALID) : stepType(st) { } - ~QueryStepParms() - { - } + ~QueryStepParms() = default; StepTeleStats::StepType stepType; diff --git a/utils/querytele/querytele_types.h b/utils/querytele/querytele_types.h index e4d52e695..dfd2816e9 100644 --- a/utils/querytele/querytele_types.h +++ b/utils/querytele/querytele_types.h @@ -11,7 +11,6 @@ #include #include - namespace querytele { struct QTType @@ -176,9 +175,7 @@ class QueryTele { } - virtual ~QueryTele() throw() - { - } + virtual ~QueryTele() throw() = default; std::string query_uuid; QTType::type msg_type; @@ -569,9 +566,7 @@ class StepTele { } - virtual ~StepTele() throw() - { - } + virtual ~StepTele() throw() = default; std::string query_uuid; STType::type msg_type; @@ -807,9 +802,7 @@ class ImportTele { } - virtual ~ImportTele() throw() - { - } + virtual ~ImportTele() throw() = default; std::string job_uuid; std::string import_uuid; diff --git a/utils/querytele/queryteleclient.h b/utils/querytele/queryteleclient.h index c2d0fa9d5..9f9670be1 100644 --- a/utils/querytele/queryteleclient.h +++ b/utils/querytele/queryteleclient.h @@ -35,7 +35,7 @@ class QueryTeleProtoImpl; class QueryTeleClient { public: - QueryTeleClient() : fProtoImpl(0) + QueryTeleClient() : fProtoImpl(nullptr) { } EXPORT explicit QueryTeleClient(const QueryTeleServerParms&); diff --git a/utils/querytele/queryteleprotoimpl.cpp b/utils/querytele/queryteleprotoimpl.cpp index 0ced86620..d5b5e9e7e 100644 --- a/utils/querytele/queryteleprotoimpl.cpp +++ b/utils/querytele/queryteleprotoimpl.cpp @@ -23,12 +23,15 @@ using namespace std; #define BOOST_DISABLE_ASSERTS #include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" #include "thrift/transport/TSocket.h" #include "thrift/transport/TBufferTransports.h" namespace att = apache::thrift::transport; #include "thrift/protocol/TBinaryProtocol.h" namespace atp = apache::thrift::protocol; +#pragma GCC diagnostic pop #include "atomicops.h" @@ -92,8 +95,7 @@ string get_trace_file() void log_query(const querytele::QueryTele& qtdata) { ofstream trace(get_trace_file().c_str(), ios::out | ios::app); - trace << "Query," << qtdata.query_uuid << "," - << ","; // skip step uuid + trace << "Query," << qtdata.query_uuid << "," << ","; // skip step uuid if (qtdata.msg_type == querytele::QTType::QT_SUMMARY) trace << "SUMMARY,"; diff --git a/utils/querytele/queryteleprotoimpl.h b/utils/querytele/queryteleprotoimpl.h index a5309eec0..34f31e452 100644 --- a/utils/querytele/queryteleprotoimpl.h +++ b/utils/querytele/queryteleprotoimpl.h @@ -25,9 +25,7 @@ class QueryTeleProtoImpl { public: explicit QueryTeleProtoImpl(const QueryTeleServerParms&); - ~QueryTeleProtoImpl() - { - } + ~QueryTeleProtoImpl() = default; int enqStepTele(const StepTele&); int enqImportTele(const ImportTele&); diff --git a/utils/querytele/queryteleserverparms.h b/utils/querytele/queryteleserverparms.h index f5c268816..fe09891b8 100644 --- a/utils/querytele/queryteleserverparms.h +++ b/utils/querytele/queryteleserverparms.h @@ -30,9 +30,7 @@ class QueryTeleServerParms QueryTeleServerParms(const std::string h, int p) : host(h), port(p) { } - ~QueryTeleServerParms() - { - } + ~QueryTeleServerParms() = default; std::string host; int port; diff --git a/utils/querytele/telestats.h b/utils/querytele/telestats.h index 23d207c34..026027c00 100644 --- a/utils/querytele/telestats.h +++ b/utils/querytele/telestats.h @@ -60,9 +60,7 @@ struct QueryTeleStats query_uuid = boost::uuids::nil_generator()(); } - ~QueryTeleStats() - { - } + ~QueryTeleStats() = default; boost::uuids::uuid query_uuid; QTType msg_type; @@ -138,9 +136,7 @@ struct StepTeleStats step_uuid = boost::uuids::nil_generator()(); } - ~StepTeleStats() - { - } + ~StepTeleStats() = default; boost::uuids::uuid query_uuid; STType msg_type; diff --git a/utils/regr/corr.h b/utils/regr/corr.h index 7a8bba083..7900c5444 100644 --- a/utils/regr/corr.h +++ b/utils/regr/corr.h @@ -50,19 +50,19 @@ class corr : public mcsv1_UDAF public: // Defaults OK corr() : mcsv1_UDAF(){}; - virtual ~corr(){}; + ~corr() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/covar_pop.h b/utils/regr/covar_pop.h index 43a9daecc..b8d5a0a50 100644 --- a/utils/regr/covar_pop.h +++ b/utils/regr/covar_pop.h @@ -50,19 +50,19 @@ class covar_pop : public mcsv1_UDAF public: // Defaults OK covar_pop() : mcsv1_UDAF(){}; - virtual ~covar_pop(){}; + ~covar_pop() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/covar_samp.h b/utils/regr/covar_samp.h index 0da83f2e2..d78d662c0 100644 --- a/utils/regr/covar_samp.h +++ b/utils/regr/covar_samp.h @@ -50,19 +50,19 @@ class covar_samp : public mcsv1_UDAF public: // Defaults OK covar_samp() : mcsv1_UDAF(){}; - virtual ~covar_samp(){}; + ~covar_samp() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/moda.h b/utils/regr/moda.h index 6d5916ac0..7f3641de7 100644 --- a/utils/regr/moda.h +++ b/utils/regr/moda.h @@ -55,7 +55,9 @@ namespace mcsv1sdk template struct hasher { - hasher(uint32_t cs_num){} + explicit hasher(uint32_t cs_num) + { + } inline size_t operator()(T val) const { @@ -70,7 +72,9 @@ struct hasher template <> struct hasher { - hasher(uint32_t cs_num){} + explicit hasher(uint32_t cs_num) + { + } inline size_t operator()(long double val) const { #ifdef MASK_LONGDOUBLE @@ -86,23 +90,27 @@ struct hasher }; // A collation aware hasher for strings -template<> +template <> struct hasher { - hasher(uint32_t cs_num) : fHasher(cs_num){} + explicit hasher(uint32_t cs_num) : fHasher(cs_num) + { + } inline size_t operator()(string val) const { return fHasher(val.c_str(), val.size()); } -private: - datatypes::CollationAwareHasher fHasher; + private: + datatypes::CollationAwareHasher fHasher; }; -template +template struct comparator { - comparator(uint32_t cs_num){} + explicit comparator(uint32_t cs_num) + { + } bool operator()(const T& lhs, const T& rhs) const { @@ -113,35 +121,38 @@ struct comparator template <> struct comparator { - comparator(uint32_t cs_num) : fCs(cs_num) {} + explicit comparator(uint32_t cs_num) : fCs(cs_num) + { + } bool operator()(const std::string lhs, const std::string rhs) const { return fCs.eq(lhs, rhs); } + private: datatypes::Charset fCs; }; - - // Override UserData for data storage struct ModaData : public UserData { - ModaData(uint32_t cs_num = 8) - : fMap(NULL) + explicit ModaData(uint32_t cs_num = 8) + : fMap(nullptr) , fReturnType((uint32_t)execplan::CalpontSystemCatalog::UNDEFINED) , fColWidth(0) - , modaImpl(NULL) - , fCs_num(cs_num){} + , modaImpl(nullptr) + , fCs_num(cs_num) + { + } - virtual ~ModaData() + ~ModaData() override { cleanup(); } - virtual void serialize(messageqcpp::ByteStream& bs) const; - virtual void unserialize(messageqcpp::ByteStream& bs); + void serialize(messageqcpp::ByteStream& bs) const override; + void unserialize(messageqcpp::ByteStream& bs) override; template std::unordered_map, comparator >* getMap() @@ -149,8 +160,8 @@ struct ModaData : public UserData if (!fMap) { // Just in time creation - fMap = new std::unordered_map, comparator >( - 10, hasher(fCs_num), comparator(fCs_num)); + fMap = new std::unordered_map, comparator >(10, hasher(fCs_num), + comparator(fCs_num)); } return (std::unordered_map, comparator >*)fMap; } @@ -169,7 +180,7 @@ struct ModaData : public UserData if (fMap) { delete (std::unordered_map, comparator >*)fMap; - fMap = NULL; + fMap = nullptr; } } @@ -192,7 +203,7 @@ struct ModaData : public UserData private: // For now, copy construction is unwanted - ModaData(UserData&); + explicit ModaData(UserData&); void cleanup(); @@ -243,45 +254,46 @@ class Moda_impl_T : public mcsv1_UDAF { public: // Defaults OK - Moda_impl_T(){}; - virtual ~Moda_impl_T(){}; + Moda_impl_T() = default; + ~Moda_impl_T() override = default; - virtual mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual mcsv1_UDAF::ReturnCode reset(mcsv1Context* context); - virtual mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + mcsv1_UDAF::ReturnCode reset(mcsv1Context* context) override; + mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; + mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; + mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; + mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; // Dummy: not used - virtual mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) + mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) override { return mcsv1_UDAF::SUCCESS; } }; -template<> // string specialization +template <> // string specialization class Moda_impl_T : public mcsv1_UDAF { public: // Defaults OK - Moda_impl_T() : cs(8) {}; - virtual ~Moda_impl_T() {}; + Moda_impl_T() : cs(8){}; + ~Moda_impl_T() override = default; - virtual mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual mcsv1_UDAF::ReturnCode reset(mcsv1Context* context); - virtual mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); - virtual mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); - virtual mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - virtual mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + mcsv1_UDAF::ReturnCode reset(mcsv1Context* context) override; + mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; + mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; + mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; + mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; // Dummy: not used - virtual mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) + mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) override { return mcsv1_UDAF::SUCCESS; } + private: datatypes::Charset cs; }; @@ -294,36 +306,36 @@ class moda : public mcsv1_UDAF public: // Defaults OK moda() : mcsv1_UDAF(){}; - virtual ~moda(){}; + ~moda() override = default; - virtual mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context) + ReturnCode reset(mcsv1Context* context) override { return getImpl(context)->reset(context); } - virtual mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) + mcsv1_UDAF::ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override { return getImpl(context)->nextValue(context, valsIn); } - virtual mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) + mcsv1_UDAF::ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override { return getImpl(context)->subEvaluate(context, valIn); } - virtual mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) + mcsv1_UDAF::ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override { return getImpl(context)->evaluate(context, valOut); } - virtual mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) + mcsv1_UDAF::ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override { return getImpl(context)->dropValue(context, valsDropped); } - mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) + mcsv1_UDAF::ReturnCode createUserData(UserData*& userData, int32_t& length) override { userData = new ModaData; length = sizeof(ModaData); diff --git a/utils/regr/regr_avgx.h b/utils/regr/regr_avgx.h index 6dd8e268d..b6b4e1190 100644 --- a/utils/regr/regr_avgx.h +++ b/utils/regr/regr_avgx.h @@ -60,19 +60,19 @@ class regr_avgx : public mcsv1_UDAF public: // Defaults OK regr_avgx() : mcsv1_UDAF(){}; - virtual ~regr_avgx(){}; + ~regr_avgx() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/regr_avgy.h b/utils/regr/regr_avgy.h index 483b0f2db..849c86f35 100644 --- a/utils/regr/regr_avgy.h +++ b/utils/regr/regr_avgy.h @@ -50,19 +50,19 @@ class regr_avgy : public mcsv1_UDAF public: // Defaults OK regr_avgy() : mcsv1_UDAF(){}; - virtual ~regr_avgy(){}; + ~regr_avgy() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/regr_count.h b/utils/regr/regr_count.h index 1286a5c11..feccfd2b9 100644 --- a/utils/regr/regr_count.h +++ b/utils/regr/regr_count.h @@ -50,19 +50,19 @@ class regr_count : public mcsv1_UDAF public: // Defaults OK regr_count() : mcsv1_UDAF(){}; - virtual ~regr_count(){}; + ~regr_count() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/regr_intercept.h b/utils/regr/regr_intercept.h index ffbc55534..2a8f1d3ac 100644 --- a/utils/regr/regr_intercept.h +++ b/utils/regr/regr_intercept.h @@ -50,19 +50,19 @@ class regr_intercept : public mcsv1_UDAF public: // Defaults OK regr_intercept() : mcsv1_UDAF(){}; - virtual ~regr_intercept(){}; + ~regr_intercept() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/regr_r2.h b/utils/regr/regr_r2.h index 23f3d2fd5..4d7fb4d74 100644 --- a/utils/regr/regr_r2.h +++ b/utils/regr/regr_r2.h @@ -50,19 +50,19 @@ class regr_r2 : public mcsv1_UDAF public: // Defaults OK regr_r2() : mcsv1_UDAF(){}; - virtual ~regr_r2(){}; + ~regr_r2() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/regr_slope.h b/utils/regr/regr_slope.h index 1a69f99ef..541571a75 100644 --- a/utils/regr/regr_slope.h +++ b/utils/regr/regr_slope.h @@ -50,19 +50,19 @@ class regr_slope : public mcsv1_UDAF public: // Defaults OK regr_slope() : mcsv1_UDAF(){}; - virtual ~regr_slope(){}; + ~regr_slope() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/regr_sxx.h b/utils/regr/regr_sxx.h index 96f14a250..883e57d3e 100644 --- a/utils/regr/regr_sxx.h +++ b/utils/regr/regr_sxx.h @@ -50,19 +50,19 @@ class regr_sxx : public mcsv1_UDAF public: // Defaults OK regr_sxx() : mcsv1_UDAF(){}; - virtual ~regr_sxx(){}; + ~regr_sxx() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/regr_sxy.h b/utils/regr/regr_sxy.h index 91418bec5..da1262c55 100644 --- a/utils/regr/regr_sxy.h +++ b/utils/regr/regr_sxy.h @@ -50,19 +50,19 @@ class regr_sxy : public mcsv1_UDAF public: // Defaults OK regr_sxy() : mcsv1_UDAF(){}; - virtual ~regr_sxy(){}; + ~regr_sxy() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/regr/regr_syy.h b/utils/regr/regr_syy.h index 7e017c637..2b8e0c035 100644 --- a/utils/regr/regr_syy.h +++ b/utils/regr/regr_syy.h @@ -50,19 +50,19 @@ class regr_syy : public mcsv1_UDAF public: // Defaults OK regr_syy() : mcsv1_UDAF(){}; - virtual ~regr_syy(){}; + ~regr_syy() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index ebbd433bb..3c6ac0695 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -357,11 +357,11 @@ class GroupConcatAg explicit GroupConcatAg(SP_GroupConcat&); virtual ~GroupConcatAg(); - virtual void initialize(){}; - virtual void processRow(const rowgroup::Row&){}; - virtual void merge(const rowgroup::Row&, uint64_t){}; + virtual void initialize() {}; + virtual void processRow(const rowgroup::Row&) {}; + virtual void merge(const rowgroup::Row&, uint64_t) {}; - uint8_t* getResult() + virtual uint8_t* getResult() { return nullptr; } @@ -392,7 +392,8 @@ class RowAggregation : public messageqcpp::Serializeable RowAggregation(); RowAggregation(const std::vector& rowAggGroupByCols, const std::vector& rowAggFunctionCols, - joblist::ResourceManager* rm = nullptr, boost::shared_ptr sessMemLimit = {}, bool withRollup = false); + joblist::ResourceManager* rm = nullptr, boost::shared_ptr sessMemLimit = {}, + bool withRollup = false); RowAggregation(const RowAggregation& rhs); /** @brief RowAggregation default destructor @@ -426,9 +427,15 @@ class RowAggregation : public messageqcpp::Serializeable initialize(); } - void clearRollup() { fRollupFlag = false; } + void clearRollup() + { + fRollupFlag = false; + } - bool hasRollup() const { return fRollupFlag; } + bool hasRollup() const + { + return fRollupFlag; + } /** @brief Define content of data to be joined * @@ -640,9 +647,9 @@ class RowAggregation : public messageqcpp::Serializeable std::unique_ptr fCurRGData; bool fRollupFlag = false; - std::string fTmpDir = config::Config::makeConfig()->getTempFileDir(config::Config::TempDirPurpose::Aggregates); + std::string fTmpDir = + config::Config::makeConfig()->getTempFileDir(config::Config::TempDirPurpose::Aggregates); std::string fCompStr = config::Config::makeConfig()->getConfig("RowAggregation", "Compression"); - }; //------------------------------------------------------------------------------ @@ -655,9 +662,7 @@ class RowAggregationUM : public RowAggregation public: /** @brief RowAggregationUM constructor */ - RowAggregationUM() - { - } + RowAggregationUM() = default; RowAggregationUM(const std::vector& rowAggGroupByCols, const std::vector& rowAggFunctionCols, joblist::ResourceManager*, boost::shared_ptr sessionMemLimit, bool withRollup); @@ -831,9 +836,7 @@ class RowAggregationUMP2 : public RowAggregationUM public: /** @brief RowAggregationUM constructor */ - RowAggregationUMP2() - { - } + RowAggregationUMP2() = default; RowAggregationUMP2(const std::vector& rowAggGroupByCols, const std::vector& rowAggFunctionCols, joblist::ResourceManager*, boost::shared_ptr sessionMemLimit, bool withRollup); @@ -873,9 +876,7 @@ class RowAggregationDistinct : public RowAggregationUMP2 public: /** @brief RowAggregationDistinct constructor */ - RowAggregationDistinct() - { - } + RowAggregationDistinct() = default; RowAggregationDistinct(const std::vector& rowAggGroupByCols, const std::vector& rowAggFunctionCols, joblist::ResourceManager*, boost::shared_ptr sessionMemLimit); @@ -940,9 +941,7 @@ class RowAggregationSubDistinct : public RowAggregationUM public: /** @brief RowAggregationSubDistinct constructor */ - RowAggregationSubDistinct() - { - } + RowAggregationSubDistinct() = default; RowAggregationSubDistinct(const std::vector& rowAggGroupByCols, const std::vector& rowAggFunctionCols, joblist::ResourceManager*, boost::shared_ptr sessionMemLimit); @@ -981,9 +980,7 @@ class RowAggregationMultiDistinct : public RowAggregationDistinct public: /** @brief RowAggregationMultiDistinct constructor */ - RowAggregationMultiDistinct() - { - } + RowAggregationMultiDistinct() = default; RowAggregationMultiDistinct(const std::vector& rowAggGroupByCols, const std::vector& rowAggFunctionCols, joblist::ResourceManager*, boost::shared_ptr sessionMemLimit); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 927dc3c7e..d8d7cedab 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -39,7 +39,6 @@ #include #include - #include "hasher.h" #include "joblisttypes.h" @@ -54,7 +53,7 @@ #include "collation.h" #include "common/hashfamily.h" -#include "stdlib.h" +#include #include "execinfo.h" // Workaround for my_global.h #define of isnan(X) causing a std::std namespace @@ -144,12 +143,12 @@ class StringStore // returns the offset. // it may receive nullptr as data and it is proper way to store NULL values. uint64_t storeString(const uint8_t* data, uint32_t length); - //please note getPointer can return nullptr. + // please note getPointer can return nullptr. inline const uint8_t* getPointer(uint64_t offset) const; inline uint32_t getStringLength(uint64_t offset) const; inline utils::ConstString getConstString(uint64_t offset) const { - return utils::ConstString((const char*)getPointer(offset), getStringLength(offset)); + return {(const char*)getPointer(offset), getStringLength(offset)}; } inline bool isEmpty() const; inline uint64_t getSize() const; @@ -223,7 +222,6 @@ class UserDataStore UserDataStore& operator=(const UserDataStore&) = delete; UserDataStore& operator=(UserDataStore&&) = delete; - void serialize(messageqcpp::ByteStream&) const; void deserialize(messageqcpp::ByteStream&); @@ -244,14 +242,12 @@ class UserDataStore boost::shared_ptr getUserData(uint32_t offset) const; private: - std::vector vStoreData; bool fUseUserDataMutex = false; boost::mutex fMutex; }; - class RowGroup; class Row; @@ -268,8 +264,6 @@ class RGData RGData(RGData&&) = default; virtual ~RGData() = default; - - // amount should be the # returned by RowGroup::getDataSize() void serialize(messageqcpp::ByteStream&, RGDataSizeType amount) const; @@ -321,8 +315,8 @@ class RGData } private: - uint32_t rowSize = 0; // can't be. - uint32_t columnCount = 0; // shouldn't be, but... + uint32_t rowSize = 0; // can't be. + uint32_t columnCount = 0; // shouldn't be, but... std::shared_ptr rowData; std::shared_ptr strings; std::shared_ptr userDataStore; @@ -372,7 +366,7 @@ class Row inline uint32_t getColumnWidth(uint32_t colIndex) const; inline uint32_t getColumnCount() const; inline uint32_t getInternalSize() const; // this is only accurate if there is no string table - inline uint32_t getSize() const; // this is only accurate if there is no string table + inline uint32_t getSize() const; // this is only accurate if there is no string table // if a string table is being used, getRealSize() takes into account variable-length strings inline uint32_t getRealSize() const; inline uint32_t getOffset(uint32_t colIndex) const; @@ -484,7 +478,7 @@ class Row inline void setDoubleField(double val, uint32_t colIndex); inline void setFloatField(float val, uint32_t colIndex); - inline void setDecimalField(double val, uint32_t colIndex){}; // TODO: Do something here + inline void setDecimalField(double val, uint32_t colIndex) {}; // TODO: Do something here inline void setLongDoubleField(const long double& val, uint32_t colIndex); inline void setInt128Field(const int128_t& val, uint32_t colIndex); @@ -610,10 +604,10 @@ class Row const CHARSET_INFO* getCharset(uint32_t col) const; -private: - inline bool inStringTable(uint32_t col) const; + private: + inline bool inStringTable(uint32_t col) const; -private: + private: uint32_t columnCount = 0; uint64_t baseRid = 0; @@ -653,7 +647,7 @@ inline void Row::setPointer(const Pointer& p) { data = p.data; strings = p.strings; - bool hasStrings = (strings != 0); + bool hasStrings = (strings != nullptr); if (useStringTable != hasStrings) { @@ -699,7 +693,7 @@ inline uint32_t Row::getRealSize() const if (!useStringTable) return getSize(); - uint32_t ret = columnCount; // account for NULL flags. + uint32_t ret = columnCount; // account for NULL flags. for (uint32_t i = 0; i < columnCount; i++) { @@ -866,8 +860,7 @@ inline int64_t Row::getIntField(uint32_t colIndex) const case 8: return *((int64_t*)&data[offsets[colIndex]]); - default: - idbassert(0); throw std::logic_error("Row::getIntField(): bad length."); + default: idbassert(0); throw std::logic_error("Row::getIntField(): bad length."); } } @@ -1049,12 +1042,13 @@ inline void Row::setStringField(const utils::ConstString& str, uint32_t colIndex else { uint8_t* buf = &data[offsets[colIndex]]; - memset(buf + length, 0, offsets[colIndex + 1] - (offsets[colIndex] + length)); // needed for memcmp in equals(). + memset(buf + length, 0, + offsets[colIndex + 1] - (offsets[colIndex] + length)); // needed for memcmp in equals(). if (str.str()) { memcpy(buf, str.str(), length); } - else if (colWidth <= 8) // special magic value. + else if (colWidth <= 8) // special magic value. { setToNull(colIndex); } @@ -1481,7 +1475,7 @@ class RowGroup : public messageqcpp::Serializeable explicit RowGroup(messageqcpp::ByteStream& bs); - ~RowGroup(); + ~RowGroup() override; inline void initRow(Row*, bool forceInlineData = false) const; inline uint32_t getRowCount() const; @@ -1517,8 +1511,8 @@ class RowGroup : public messageqcpp::Serializeable void resetRowGroup(uint64_t baseRid); /* The Serializeable interface */ - void serialize(messageqcpp::ByteStream&) const; - void deserialize(messageqcpp::ByteStream&); + void serialize(messageqcpp::ByteStream&) const override; + void deserialize(messageqcpp::ByteStream&) override; uint32_t getColumnWidth(uint32_t col) const; uint32_t getColumnCount() const; @@ -1604,7 +1598,7 @@ class RowGroup : public messageqcpp::Serializeable std::vector oldOffsets; // inline data offsets std::vector stOffsets; // string table offsets - uint32_t* offsets = nullptr; // offsets either points to oldOffsets or stOffsets + uint32_t* offsets = nullptr; // offsets either points to oldOffsets or stOffsets std::vector colWidths; // oids: the real oid of the column, may have duplicates with alias. // This oid is necessary for front-end to decide the real column width. @@ -1714,7 +1708,7 @@ inline void RowGroup::setUseStringTable(bool b) { useStringTable = (b && hasLongStringField); // offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - offsets = 0; + offsets = nullptr; if (useStringTable && !stOffsets.empty()) offsets = &stOffsets[0]; @@ -2053,7 +2047,6 @@ inline void copyRowInline(const Row& in, Row* out, uint32_t colCount) copyRow(in, out, colCount); } - inline utils::NullString StringStore::getString(uint64_t off) const { uint32_t length; @@ -2133,7 +2126,7 @@ inline const uint8_t* StringStore::getPointer(uint64_t off) const inline bool StringStore::isNullValue(uint64_t off) const { - if (off == std::numeric_limits::max()) + if (off == std::numeric_limits::max()) return true; return false; } diff --git a/utils/rwlock/rwlock.h b/utils/rwlock/rwlock.h index 575c8854f..6fafdc4ee 100644 --- a/utils/rwlock/rwlock.h +++ b/utils/rwlock/rwlock.h @@ -42,7 +42,13 @@ namespace rwlock { const std::array RWLockNames = {{ - "all", "VSS", "ExtentMap", "FreeList", "VBBM", "CopyLocks", "ExtentMapIndex", + "all", + "VSS", + "ExtentMap", + "FreeList", + "VBBM", + "CopyLocks", + "ExtentMapIndex", }}; /// the layout of the shmseg @@ -70,7 +76,9 @@ struct LockState class RWLockShmImpl { public: - static RWLockShmImpl* makeRWLockShmImpl(int key, bool* excl = 0); + ~RWLockShmImpl() = delete; + + static RWLockShmImpl* makeRWLockShmImpl(int key, bool* excl = nullptr); boost::interprocess::shared_memory_object fStateShm; boost::interprocess::mapped_region fRegion; @@ -83,7 +91,6 @@ class RWLockShmImpl private: explicit RWLockShmImpl(int key, bool excl = false); - ~RWLockShmImpl(); RWLockShmImpl(const RWLockShmImpl& rhs); RWLockShmImpl& operator=(const RWLockShmImpl& rhs); std::string fKeyString; @@ -92,7 +99,7 @@ class RWLockShmImpl class not_excl : public std::exception { public: - virtual const char* what() const throw() + const char* what() const noexcept override { return "not_excl"; } @@ -101,7 +108,7 @@ class not_excl : public std::exception class wouldblock : public std::exception { public: - virtual const char* what() const throw() + const char* what() const noexcept override { return "wouldblock"; } @@ -143,7 +150,7 @@ class RWLock * this is not the first instance, it will throw not_excl. The intent * is similar to the IPC_EXCL flag in the sem/shm implementations. */ - EXPORT explicit RWLock(int key, bool* excl = 0); + EXPORT explicit RWLock(int key, bool* excl = nullptr); EXPORT ~RWLock(); @@ -184,7 +191,7 @@ class RWLock * a non-NULL LockState struct. This is a specialization for supporting * the RWLockMonitor class. */ - EXPORT bool timed_write_lock(const struct timespec& ts, struct LockState* state = 0); + EXPORT bool timed_write_lock(const struct timespec& ts, struct LockState* state = nullptr); /** @brief Release a write lock. * @@ -262,4 +269,3 @@ class RWLock } // namespace rwlock #undef EXPORT - diff --git a/utils/rwlock/rwlock_local.h b/utils/rwlock/rwlock_local.h index bce1b7c88..c02fa6b98 100644 --- a/utils/rwlock/rwlock_local.h +++ b/utils/rwlock/rwlock_local.h @@ -57,7 +57,7 @@ class RWLock_local class not_excl : public std::exception { public: - virtual const char* what() const throw() + const char* what() const noexcept override { return "not_excl"; } @@ -66,7 +66,7 @@ class RWLock_local class wouldblock : public std::exception { public: - virtual const char* what() const throw() + const char* what() const noexcept override { return "wouldblock"; } @@ -172,10 +172,9 @@ class ScopedRWLock_local void unlock(); private: - explicit ScopedRWLock_local() - { - } - explicit ScopedRWLock_local(const ScopedRWLock_local&) + explicit ScopedRWLock_local() = default; + + ScopedRWLock_local(const ScopedRWLock_local&) { } ScopedRWLock_local& operator=(const ScopedRWLock_local&) diff --git a/utils/udfsdk/allnull.h b/utils/udfsdk/allnull.h index 497cf985a..31cbff06d 100644 --- a/utils/udfsdk/allnull.h +++ b/utils/udfsdk/allnull.h @@ -71,7 +71,7 @@ class allnull : public mcsv1_UDAF public: // Defaults OK allnull() : mcsv1_UDAF(){}; - virtual ~allnull(){}; + ~allnull() override = default; /** * init() @@ -91,7 +91,7 @@ class allnull : public mcsv1_UDAF * colTypes or wrong number of arguments. Else return * mcsv1_UDAF::SUCCESS. */ - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; /** * reset() @@ -106,7 +106,7 @@ class allnull : public mcsv1_UDAF * * Use this opportunity to initialize the userData. */ - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; /** * nextValue() @@ -126,7 +126,7 @@ class allnull : public mcsv1_UDAF * * valsIn (in) - a vector of the parameters from the row. */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; /** * subEvaluate() @@ -153,7 +153,7 @@ class allnull : public mcsv1_UDAF * as seen in the last call to NextValue for a given PM. * */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn) override; /** * evaluate() @@ -174,11 +174,9 @@ class allnull : public mcsv1_UDAF * * To return a NULL value, don't assign to valOut. */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); - - protected: + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; }; -}; // namespace mcsv1sdk +} // namespace mcsv1sdk #undef EXPORT diff --git a/utils/udfsdk/avg_mode.h b/utils/udfsdk/avg_mode.h index 023522bbd..ca858f39e 100644 --- a/utils/udfsdk/avg_mode.h +++ b/utils/udfsdk/avg_mode.h @@ -71,20 +71,18 @@ typedef std::tr1::unordered_map MODE_DATA; // Override UserData for data storage struct ModeData : public UserData { - ModeData(){}; + ModeData() = default; - virtual ~ModeData() - { - } + ~ModeData() override = default; - virtual void serialize(messageqcpp::ByteStream& bs) const; - virtual void unserialize(messageqcpp::ByteStream& bs); + void serialize(messageqcpp::ByteStream& bs) const override; + void unserialize(messageqcpp::ByteStream& bs) override; MODE_DATA mData; private: // For now, copy construction is unwanted - ModeData(UserData&); + explicit ModeData(UserData&); }; // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or @@ -104,7 +102,7 @@ class avg_mode : public mcsv1_UDAF public: // Defaults OK avg_mode() : mcsv1_UDAF(){}; - virtual ~avg_mode(){}; + ~avg_mode() override = default; /** * init() @@ -124,7 +122,7 @@ class avg_mode : public mcsv1_UDAF * colTypes or wrong number of arguments. Else return * mcsv1_UDAF::SUCCESS. */ - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; /** * reset() @@ -139,7 +137,7 @@ class avg_mode : public mcsv1_UDAF * * Use this opportunity to initialize the userData. */ - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; /** * nextValue() @@ -159,7 +157,7 @@ class avg_mode : public mcsv1_UDAF * * valsIn (in) - a vector of the parameters from the row. */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; /** * subEvaluate() @@ -186,7 +184,7 @@ class avg_mode : public mcsv1_UDAF * as seen in the last call to NextValue for a given PM. * */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; /** * evaluate() @@ -207,7 +205,7 @@ class avg_mode : public mcsv1_UDAF * * To return a NULL value, don't assign to valOut. */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; /** * dropValue() @@ -235,7 +233,7 @@ class avg_mode : public mcsv1_UDAF * dropValue() will not be called for unbounded/current row type * frames, as those are already optimized. */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; /** * createUserData() @@ -260,11 +258,11 @@ class avg_mode : public mcsv1_UDAF * memory leaks are your fault. * */ - virtual ReturnCode createUserData(UserData*& data, int32_t& length); + ReturnCode createUserData(UserData*& data, int32_t& length) override; protected: }; -}; // namespace mcsv1sdk +} // namespace mcsv1sdk #undef EXPORT diff --git a/utils/udfsdk/avgx.h b/utils/udfsdk/avgx.h index 7b2129c1f..6ef970787 100644 --- a/utils/udfsdk/avgx.h +++ b/utils/udfsdk/avgx.h @@ -61,19 +61,19 @@ class avgx : public mcsv1_UDAF public: // Defaults OK avgx() : mcsv1_UDAF(){}; - virtual ~avgx(){}; + ~avgx() override = default; - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/udfsdk/distinct_count.h b/utils/udfsdk/distinct_count.h index 695d99ff0..b88e677d3 100644 --- a/utils/udfsdk/distinct_count.h +++ b/utils/udfsdk/distinct_count.h @@ -70,7 +70,7 @@ class distinct_count : public mcsv1_UDAF public: // Defaults OK distinct_count() : mcsv1_UDAF(){}; - virtual ~distinct_count(){}; + ~distinct_count() override = default; /** * init() @@ -90,7 +90,7 @@ class distinct_count : public mcsv1_UDAF * colTypes or wrong number of arguments. Else return * mcsv1_UDAF::SUCCESS. */ - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; /** * reset() @@ -105,7 +105,7 @@ class distinct_count : public mcsv1_UDAF * * Use this opportunity to initialize the userData. */ - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; /** * nextValue() @@ -125,7 +125,7 @@ class distinct_count : public mcsv1_UDAF * * valsIn (in) - a vector of the parameters from the row. */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; /** * subEvaluate() @@ -152,7 +152,7 @@ class distinct_count : public mcsv1_UDAF * as seen in the last call to NextValue for a given PM. * */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn) override; /** * evaluate() @@ -173,7 +173,7 @@ class distinct_count : public mcsv1_UDAF * * To return a NULL value, don't assign to valOut. */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; /** * dropValue() @@ -201,7 +201,7 @@ class distinct_count : public mcsv1_UDAF * dropValue() will not be called for unbounded/current row type * frames, as those are already optimized. */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/udfsdk/mcsv1_udaf.h b/utils/udfsdk/mcsv1_udaf.h index 9fbc56547..ea949b71e 100644 --- a/utils/udfsdk/mcsv1_udaf.h +++ b/utils/udfsdk/mcsv1_udaf.h @@ -100,9 +100,9 @@ typedef std::tr1::unordered_map UDAF_MAP; class UDAFMap { public: - EXPORT UDAFMap(){}; + EXPORT UDAFMap() = default; - EXPORT ~UDAFMap(){}; + EXPORT ~UDAFMap() = default; static EXPORT UDAF_MAP& getMap(); @@ -129,8 +129,8 @@ class mcsv1Context; struct UserData { - UserData() : size(0), data(NULL){}; - UserData(size_t sz) + UserData() : size(0), data(nullptr){}; + explicit UserData(size_t sz) { size = sz; data = new uint8_t[sz]; @@ -468,8 +468,8 @@ class mcsv1_UDAF NOT_IMPLEMENTED = 2 // User UDA(n)F shouldn't return this }; // Defaults OK - mcsv1_UDAF(){}; - virtual ~mcsv1_UDAF(){}; + mcsv1_UDAF() = default; + virtual ~mcsv1_UDAF() = default; /** * init() @@ -667,19 +667,19 @@ inline mcsv1Context::mcsv1Context() , fColWidth(0) , fResultscale(0) , fResultPrecision(18) - , dataFlags(NULL) - , bInterrupted(NULL) + , dataFlags(nullptr) + , bInterrupted(nullptr) , fStartFrame(execplan::WF_UNBOUNDED_PRECEDING) , fEndFrame(execplan::WF_CURRENT_ROW) , fStartConstant(0) , fEndConstant(0) - , func(NULL) + , func(nullptr) , fParamCount(0) , fCharsetNumber(8) // Latin1 { } -inline mcsv1Context::mcsv1Context(const mcsv1Context& rhs) : dataFlags(NULL) +inline mcsv1Context::mcsv1Context(const mcsv1Context& rhs) : dataFlags(nullptr) { copy(rhs); } @@ -703,13 +703,11 @@ inline mcsv1Context& mcsv1Context::copy(const mcsv1Context& rhs) return *this; } -inline mcsv1Context::~mcsv1Context() -{ -} +inline mcsv1Context::~mcsv1Context() = default; inline mcsv1Context& mcsv1Context::operator=(const mcsv1Context& rhs) { - dataFlags = NULL; + dataFlags = nullptr; return copy(rhs); } diff --git a/utils/udfsdk/median.h b/utils/udfsdk/median.h index 710411f95..bb8b544a9 100644 --- a/utils/udfsdk/median.h +++ b/utils/udfsdk/median.h @@ -71,20 +71,18 @@ typedef std::map MEDIAN_DATA; // Override UserData for data storage struct MedianData : public UserData { - MedianData(){}; + MedianData() = default; - virtual ~MedianData() - { - } + ~MedianData() override = default; - virtual void serialize(messageqcpp::ByteStream& bs) const; - virtual void unserialize(messageqcpp::ByteStream& bs); + void serialize(messageqcpp::ByteStream& bs) const override; + void unserialize(messageqcpp::ByteStream& bs) override; MEDIAN_DATA mData; private: // For now, copy construction is unwanted - MedianData(UserData&); + explicit MedianData(UserData&); }; // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or @@ -104,7 +102,7 @@ class median : public mcsv1_UDAF public: // Defaults OK median() : mcsv1_UDAF(){}; - virtual ~median(){}; + ~median() override = default; /** * init() @@ -124,7 +122,7 @@ class median : public mcsv1_UDAF * colTypes or wrong number of arguments. Else return * mcsv1_UDAF::SUCCESS. */ - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; /** * reset() @@ -139,7 +137,7 @@ class median : public mcsv1_UDAF * * Use this opportunity to initialize the userData. */ - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; /** * nextValue() @@ -159,7 +157,7 @@ class median : public mcsv1_UDAF * * valsIn (in) - a vector of the parameters from the row. */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; /** * subEvaluate() @@ -186,7 +184,7 @@ class median : public mcsv1_UDAF * as seen in the last call to NextValue for a given PM. * */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* valIn) override; /** * evaluate() @@ -207,7 +205,7 @@ class median : public mcsv1_UDAF * * To return a NULL value, don't assign to valOut. */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; /** * dropValue() @@ -235,7 +233,7 @@ class median : public mcsv1_UDAF * dropValue() will not be called for unbounded/current row type * frames, as those are already optimized. */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; /** * createUserData() @@ -260,7 +258,7 @@ class median : public mcsv1_UDAF * memory leaks are your fault. * */ - virtual ReturnCode createUserData(UserData*& data, int32_t& length); + ReturnCode createUserData(UserData*& data, int32_t& length) override; protected: }; diff --git a/utils/udfsdk/ssq.h b/utils/udfsdk/ssq.h index de6327edc..2b20dbc2b 100644 --- a/utils/udfsdk/ssq.h +++ b/utils/udfsdk/ssq.h @@ -81,7 +81,7 @@ class ssq : public mcsv1_UDAF public: // Defaults OK ssq() : mcsv1_UDAF(){}; - virtual ~ssq(){}; + ~ssq() override = default; /** * init() @@ -101,7 +101,7 @@ class ssq : public mcsv1_UDAF * colTypes or wrong number of arguments. Else return * mcsv1_UDAF::SUCCESS. */ - virtual ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes); + ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; /** * reset() @@ -114,7 +114,7 @@ class ssq : public mcsv1_UDAF * the next aggregation. May be called multiple times on * different modules. */ - virtual ReturnCode reset(mcsv1Context* context); + ReturnCode reset(mcsv1Context* context) override; /** * nextValue() @@ -134,7 +134,7 @@ class ssq : public mcsv1_UDAF * * valsIn (in) - a vector of the parameters from the row. */ - virtual ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn); + ReturnCode nextValue(mcsv1Context* context, ColumnDatum* valsIn) override; /** * subEvaluate() @@ -161,7 +161,7 @@ class ssq : public mcsv1_UDAF * as seen in the last call to NextValue for a given PM. * */ - virtual ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn); + ReturnCode subEvaluate(mcsv1Context* context, const UserData* userDataIn) override; /** * evaluate() @@ -182,7 +182,7 @@ class ssq : public mcsv1_UDAF * * To return a NULL value, don't assign to valOut. */ - virtual ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut); + ReturnCode evaluate(mcsv1Context* context, static_any::any& valOut) override; /** * dropValue() @@ -210,7 +210,7 @@ class ssq : public mcsv1_UDAF * dropValue() will not be called for unbounded/current row type * frames, as those are already optimized. */ - virtual ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped); + ReturnCode dropValue(mcsv1Context* context, ColumnDatum* valsDropped) override; protected: }; diff --git a/utils/udfsdk/udfsdk.h b/utils/udfsdk/udfsdk.h index 131b9459a..fc21d1b14 100644 --- a/utils/udfsdk/udfsdk.h +++ b/utils/udfsdk/udfsdk.h @@ -108,9 +108,7 @@ class MCS_add : public funcexp::Func /* * Destructor. MCS_add does not need to do anything here to clean up. */ - virtual ~MCS_add() - { - } + ~MCS_add() override = default; /** * Decide on the function's operation type @@ -135,8 +133,8 @@ class MCS_add : public funcexp::Func * This function is called only one from the connector. Once it's determined, it * will be passed to the getXXXval() APIs during function evaluation. */ - execplan::CalpontSystemCatalog::ColType operationType(funcexp::FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + funcexp::FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; /** * Returns an integer result of this function. @@ -154,47 +152,47 @@ class MCS_add : public funcexp::Func * @parm op_ct the operation type that is determined in operationType(). * */ - virtual int64_t getIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a double result of this function. */ - virtual double getDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a long double result of this function. */ - virtual long double getLongDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + long double getLongDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a float result of this function. */ - virtual float getFloatVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + float getFloatVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a string result of this function. */ - virtual std::string getStrVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a bool result of this function. */ - virtual bool getBoolVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a decimal result of this function. * * IDB_Decimal is defined in ~/execplan/treenode.h */ - virtual execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns an integer representation of a date result of the function. @@ -202,8 +200,8 @@ class MCS_add : public funcexp::Func * Check the date/time functions in ~/utils/funcexp for implementation * example of this API. */ - virtual int32_t getDateIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns an integer representation of a datetime result of the function. @@ -211,8 +209,8 @@ class MCS_add : public funcexp::Func * Check the date/time functions in ~/utils/funcexp for implementation * example of this API. */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; /** @@ -233,59 +231,57 @@ class MCS_isnull : public funcexp::Func /* * Destructor. MCS_add does not need to do anything here to clean up. */ - virtual ~MCS_isnull() - { - } + ~MCS_isnull() override = default; /** * Decide on the function's operation type */ - execplan::CalpontSystemCatalog::ColType operationType(funcexp::FunctionParm& fp, - execplan::CalpontSystemCatalog::ColType& resultType); + execplan::CalpontSystemCatalog::ColType operationType( + funcexp::FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) override; /** * Returns an integer result of this function. */ - virtual int64_t getIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a double result of this function. */ - virtual double getDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + double getDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a double result of this function. */ - virtual long double getLongDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + long double getLongDoubleVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a float result of this function. */ - virtual float getFloatVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + float getFloatVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a string result of this function. */ - virtual std::string getStrVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + std::string getStrVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a bool result of this function. */ - virtual bool getBoolVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns a decimal result of this function. * * IDB_Decimal is defined in ~/execplan/treenode.h */ - virtual execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns an integer representation of a date result of the function. @@ -293,8 +289,8 @@ class MCS_isnull : public funcexp::Func * Check the date/time functions in ~/utils/funcexp for implementation * example of this API. */ - virtual int32_t getDateIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int32_t getDateIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; /** * Returns an integer representation of a datetime result of the function. @@ -302,11 +298,10 @@ class MCS_isnull : public funcexp::Func * Check the date/time functions in ~/utils/funcexp for implementation * example of this API. */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getDatetimeIntVal(rowgroup::Row& row, funcexp::FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) override; }; } // namespace udfsdk -#undef EXPORT - +#undef EXPORT \ No newline at end of file diff --git a/utils/windowfunction/framebound.cpp b/utils/windowfunction/framebound.cpp index 56b50d2e0..d41c380f9 100644 --- a/utils/windowfunction/framebound.cpp +++ b/utils/windowfunction/framebound.cpp @@ -17,18 +17,11 @@ // $Id: framebound.cpp 3828 2013-05-22 17:58:14Z xlou $ -//#define NDEBUG -#include +// #define NDEBUG #include #include -#include using namespace std; -#include "idberrorinfo.h" -#include "errorids.h" -#include "exceptclasses.h" -using namespace logging; - #include "idborderby.h" using namespace ordering; diff --git a/utils/windowfunction/framebound.h b/utils/windowfunction/framebound.h index 7dd4ad179..7787fc518 100644 --- a/utils/windowfunction/framebound.h +++ b/utils/windowfunction/framebound.h @@ -52,11 +52,11 @@ class FrameBound /** @brief FrameBound constructor * @param t, frame type */ - FrameBound(int t = 0) : fBoundType(t), fStart(true){}; + explicit FrameBound(int t = 0) : fBoundType(t), fStart(true){}; /** @brief FrameBound destructor */ - virtual ~FrameBound(){}; + virtual ~FrameBound() = default; /** @brief clone */ @@ -144,4 +144,3 @@ class FrameBound extern std::map colType2String; } // namespace windowfunction - diff --git a/utils/windowfunction/frameboundrange.cpp b/utils/windowfunction/frameboundrange.cpp index e884d9303..694ee6a10 100644 --- a/utils/windowfunction/frameboundrange.cpp +++ b/utils/windowfunction/frameboundrange.cpp @@ -18,11 +18,9 @@ // $Id: frameboundrange.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG -#include +// #define NDEBUG #include #include -#include using namespace std; #include "idberrorinfo.h" diff --git a/utils/windowfunction/frameboundrange.h b/utils/windowfunction/frameboundrange.h index 5526a0b04..35515b0d2 100644 --- a/utils/windowfunction/frameboundrange.h +++ b/utils/windowfunction/frameboundrange.h @@ -35,25 +35,25 @@ class FrameBoundRange : public FrameBound * @param n, order by sort spec: null first | null last * @param v, order by column data type */ - FrameBoundRange(int t = 0, bool a = true, bool n = true) + explicit FrameBoundRange(int t = 0, bool a = true, bool n = true) : FrameBound(t), fAsc(a), fNullFirst(n), fIsZero(false){}; /** @brief FrameBoundRange destructor */ - virtual ~FrameBoundRange(){}; + ~FrameBoundRange() override = default; /** @brief clone */ - virtual FrameBound* clone() + FrameBound* clone() override { - return NULL; // abstract class + return nullptr; // abstract class } /** @brief virtual void getBound */ - int64_t getBound(int64_t, int64_t, int64_t); + int64_t getBound(int64_t, int64_t, int64_t) override; - const std::string toString() const; + const std::string toString() const override; void setTupleId(std::vector ids) { @@ -125,9 +125,10 @@ class FrameBoundConstantRange : public FrameBoundRange * @param n, order by sort spec: null first | null last * @param c, constant value. !! caller need to check NULL or negative value !! */ - FrameBoundConstantRange(int t = 0, bool a = true, bool n = true, void* c = NULL) : FrameBoundRange(t, a, n) + explicit FrameBoundConstantRange(int t = 0, bool a = true, bool n = true, void* c = nullptr) + : FrameBoundRange(t, a, n) { - fValue.fIsNull = (c == NULL); + fValue.fIsNull = (c == nullptr); if (!fValue.fIsNull) fValue.fValue = *((T*)c); @@ -135,20 +136,20 @@ class FrameBoundConstantRange : public FrameBoundRange /** @brief FrameBoundConstantRange destructor */ - virtual ~FrameBoundConstantRange(){}; + ~FrameBoundConstantRange() override = default; /** @brief clone */ - virtual FrameBound* clone() + FrameBound* clone() override { return new FrameBoundConstantRange(*this); } /** @brief virtual void getBound */ - int64_t getBound(int64_t, int64_t, int64_t); + int64_t getBound(int64_t, int64_t, int64_t) override; - const std::string toString() const; + const std::string toString() const override; protected: // find the range offset @@ -183,15 +184,16 @@ class FrameBoundExpressionRange : public FrameBoundConstantRange * @param a, order by sort spec: asc | desc * @param n, order by sort spec: null first | null last */ - FrameBoundExpressionRange(int t = 0, bool a = true, bool n = true) : FrameBoundConstantRange(t, a, n){}; + explicit FrameBoundExpressionRange(int t = 0, bool a = true, bool n = true) + : FrameBoundConstantRange(t, a, n){}; /** @brief FrameBoundExpressionRange destructor */ - virtual ~FrameBoundExpressionRange(){}; + ~FrameBoundExpressionRange() override = default; /** @brief clone */ - virtual FrameBound* clone() + FrameBound* clone() override { return new FrameBoundExpressionRange(*this); } @@ -200,16 +202,15 @@ class FrameBoundExpressionRange : public FrameBoundConstantRange */ // int64_t getBound(int64_t, int64_t, int64_t); - const std::string toString() const; + const std::string toString() const override; protected: // virtual in FrameBoundRange - int64_t getPrecedingOffset(int64_t j, int64_t i); - int64_t getFollowingOffset(int64_t j, int64_t k); + int64_t getPrecedingOffset(int64_t j, int64_t i) override; + int64_t getFollowingOffset(int64_t j, int64_t k) override; // validate the expression is not negative - void validate(); + void validate() override; }; } // namespace windowfunction - diff --git a/utils/windowfunction/frameboundrow.cpp b/utils/windowfunction/frameboundrow.cpp index 3d4427413..b157a6b5b 100644 --- a/utils/windowfunction/frameboundrow.cpp +++ b/utils/windowfunction/frameboundrow.cpp @@ -17,11 +17,9 @@ // $Id: frameboundrow.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG -#include +// #define NDEBUG #include #include -#include using namespace std; #include "idberrorinfo.h" @@ -32,7 +30,6 @@ using namespace logging; #include "idborderby.h" using namespace ordering; -#include "treenode.h" #include "frameboundrow.h" namespace windowfunction diff --git a/utils/windowfunction/frameboundrow.h b/utils/windowfunction/frameboundrow.h index 79c071e49..f97e3d929 100644 --- a/utils/windowfunction/frameboundrow.h +++ b/utils/windowfunction/frameboundrow.h @@ -32,24 +32,24 @@ class FrameBoundRow : public FrameBound /** @brief FrameBoundRow constructor * @param t, frame type */ - FrameBoundRow(int t = 0) : FrameBound(t){}; + explicit FrameBoundRow(int t = 0) : FrameBound(t){}; /** @brief FrameBoundRow destructor */ - virtual ~FrameBoundRow(){}; + ~FrameBoundRow() override = default; /** @brief clone */ - virtual FrameBound* clone() + FrameBound* clone() override { return new FrameBoundRow(*this); } /** @brief virtual void getBound */ - int64_t getBound(int64_t, int64_t, int64_t); + int64_t getBound(int64_t, int64_t, int64_t) override; - const std::string toString() const; + const std::string toString() const override; protected: }; @@ -64,24 +64,24 @@ class FrameBoundConstantRow : public FrameBoundRow * @param t, frame type * @param c, constant value. !! caller need to check NULL or negative value !! */ - FrameBoundConstantRow(int t = 0, int c = 0) : FrameBoundRow(t), fOffset(c){}; + explicit FrameBoundConstantRow(int t = 0, int c = 0) : FrameBoundRow(t), fOffset(c){}; /** @brief FrameBoundConstantRow destructor */ - virtual ~FrameBoundConstantRow(){}; + ~FrameBoundConstantRow() override = default; /** @brief clone */ - virtual FrameBound* clone() + FrameBound* clone() override { return new FrameBoundConstantRow(*this); } /** @brief virtual void getBound */ - int64_t getBound(int64_t, int64_t, int64_t); + int64_t getBound(int64_t, int64_t, int64_t) override; - const std::string toString() const; + const std::string toString() const override; protected: // constant offset @@ -98,25 +98,25 @@ class FrameBoundExpressionRow : public FrameBoundConstantRow /** @brief FrameBoundExpressionRow constructor * @param t, frame type */ - FrameBoundExpressionRow(int t, uint64_t id = -1, int idx = -1) + explicit FrameBoundExpressionRow(int t, uint64_t id = -1, int idx = -1) : FrameBoundConstantRow(t), fExprTupleId(id), fExprIdx(idx){}; /** @brief FrameBoundExpressionRow destructor */ - virtual ~FrameBoundExpressionRow(){}; + ~FrameBoundExpressionRow() override = default; /** @brief clone */ - virtual FrameBound* clone() + FrameBound* clone() override { return new FrameBoundExpressionRow(*this); } /** @brief virtual void getBound */ - int64_t getBound(int64_t, int64_t, int64_t); + int64_t getBound(int64_t, int64_t, int64_t) override; - const std::string toString() const; + const std::string toString() const override; void setExprTupleId(int id) { @@ -145,4 +145,3 @@ class FrameBoundExpressionRow : public FrameBoundConstantRow }; } // namespace windowfunction - diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 1c6fd9528..ad52b0f37 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -19,7 +19,6 @@ // $Id: idborderby.cpp 3932 2013-06-25 16:08:10Z xlou $ #include -#include #include #include using namespace std; @@ -28,7 +27,6 @@ using namespace std; #include "calpontselectexecutionplan.h" #include "rowgroup.h" - using namespace boost; #include "errorids.h" @@ -41,7 +39,6 @@ using namespace execplan; #include "resourcemanager.h" using namespace joblist; -#include "rowgroup.h" using namespace rowgroup; #include "idborderby.h" @@ -531,9 +528,9 @@ int TimeCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) bool CompareRule::less(Row::Pointer r1, Row::Pointer r2) { - for (vector::iterator i = fCompares.begin(); i != fCompares.end(); i++) + for (auto& compare : fCompares) { - int c = ((*(*i))(fIdbCompare, r1, r2)); + int c = ((*compare)(fIdbCompare, r1, r2)); if (c < 0) return true; @@ -546,7 +543,7 @@ bool CompareRule::less(Row::Pointer r1, Row::Pointer r2) void CompareRule::revertRules() { - std::vector::iterator fCompareIter = fCompares.begin(); + auto fCompareIter = fCompares.begin(); for (; fCompareIter != fCompares.end(); fCompareIter++) { (*fCompareIter)->revertSortSpec(); @@ -558,32 +555,32 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr const vector& types = rg.getColTypes(); const auto& offsets = rg.getOffsets(); - for (vector::const_iterator i = spec.begin(); i != spec.end(); i++) + for (auto spec_el : spec) { - switch (types[i->fIndex]) + switch (types[spec_el.fIndex]) { case CalpontSystemCatalog::TINYINT: { - Compare* c = new TinyIntCompare(*i); + Compare* c = new TinyIntCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::SMALLINT: { - Compare* c = new SmallIntCompare(*i); + Compare* c = new SmallIntCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: { - Compare* c = new IntCompare(*i); + Compare* c = new IntCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::BIGINT: { - Compare* c = new BigIntCompare(*i); + Compare* c = new BigIntCompare(spec_el); fCompares.push_back(c); break; } @@ -591,14 +588,16 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::UDECIMAL: { Compare* c; - uint32_t len = rg.getColumnWidth(i->fIndex); + uint32_t len = rg.getColumnWidth(spec_el.fIndex); switch (len) { - case datatypes::MAXDECIMALWIDTH: c = new WideDecimalCompare(*i, offsets[i->fIndex]); break; - case datatypes::MAXLEGACYWIDTH: c = new BigIntCompare(*i); break; - case 1: c = new TinyIntCompare(*i); break; - case 2: c = new SmallIntCompare(*i); break; - case 4: c = new IntCompare(*i); break; + case datatypes::MAXDECIMALWIDTH: + c = new WideDecimalCompare(spec_el, offsets[spec_el.fIndex]); + break; + case datatypes::MAXLEGACYWIDTH: c = new BigIntCompare(spec_el); break; + case 1: c = new TinyIntCompare(spec_el); break; + case 2: c = new SmallIntCompare(spec_el); break; + case 4: c = new IntCompare(spec_el); break; } fCompares.push_back(c); @@ -607,26 +606,26 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::UTINYINT: { - Compare* c = new UTinyIntCompare(*i); + Compare* c = new UTinyIntCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::USMALLINT: { - Compare* c = new USmallIntCompare(*i); + Compare* c = new USmallIntCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: { - Compare* c = new UIntCompare(*i); + Compare* c = new UIntCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::UBIGINT: { - Compare* c = new UBigIntCompare(*i); + Compare* c = new UBigIntCompare(spec_el); fCompares.push_back(c); break; } @@ -635,7 +634,7 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: { - Compare* c = new StringCompare(*i); + Compare* c = new StringCompare(spec_el); fCompares.push_back(c); break; } @@ -643,7 +642,7 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { - Compare* c = new DoubleCompare(*i); + Compare* c = new DoubleCompare(spec_el); fCompares.push_back(c); break; } @@ -651,34 +650,34 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: { - Compare* c = new FloatCompare(*i); + Compare* c = new FloatCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::LONGDOUBLE: { - Compare* c = new LongDoubleCompare(*i); + Compare* c = new LongDoubleCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::DATE: { - Compare* c = new DateCompare(*i); + Compare* c = new DateCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: { - Compare* c = new DatetimeCompare(*i); + Compare* c = new DatetimeCompare(spec_el); fCompares.push_back(c); break; } case CalpontSystemCatalog::TIME: { - Compare* c = new TimeCompare(*i); + Compare* c = new TimeCompare(spec_el); fCompares.push_back(c); break; } @@ -717,7 +716,7 @@ OrderByData::OrderByData(const std::vector& spec, const rowgroup::R OrderByData::~OrderByData() { // delete compare objects - vector::iterator i = fRule.fCompares.begin(); + auto i = fRule.fCompares.begin(); while (i != fRule.fCompares.end()) { @@ -732,7 +731,7 @@ OrderByData::~OrderByData() // IdbOrderBy class implementation IdbOrderBy::IdbOrderBy() - : fDistinct(false), fMemSize(0), fRowsPerRG(rowgroup::rgCommonSize), fErrorCode(0), fRm(NULL) + : fDistinct(false), fMemSize(0), fRowsPerRG(rowgroup::rgCommonSize), fErrorCode(0), fRm(nullptr) { } diff --git a/utils/windowfunction/idborderby.h b/utils/windowfunction/idborderby.h index 3ea9f892f..600a0f3da 100644 --- a/utils/windowfunction/idborderby.h +++ b/utils/windowfunction/idborderby.h @@ -49,7 +49,7 @@ class reservablePQ : private std::priority_queue<_Tp, _Sequence, _Compare> { public: typedef typename std::priority_queue<_Tp, _Sequence, _Compare>::size_type size_type; - reservablePQ(size_type capacity = 0) + explicit reservablePQ(size_type capacity = 0) { reserve(capacity); }; @@ -98,12 +98,10 @@ struct IdbSortSpec class Compare { public: - Compare(const IdbSortSpec& spec) : fSpec(spec) - { - } - virtual ~Compare() + explicit Compare(const IdbSortSpec& spec) : fSpec(spec) { } + virtual ~Compare() = default; virtual int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) = 0; void revertSortSpec() @@ -121,41 +119,41 @@ class Compare class TinyIntCompare : public Compare { public: - TinyIntCompare(const IdbSortSpec& spec) : Compare(spec) + explicit TinyIntCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class SmallIntCompare : public Compare { public: - SmallIntCompare(const IdbSortSpec& spec) : Compare(spec) + explicit SmallIntCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class IntCompare : public Compare { public: - IntCompare(const IdbSortSpec& spec) : Compare(spec) + explicit IntCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class BigIntCompare : public Compare { public: - BigIntCompare(const IdbSortSpec& spec) : Compare(spec) + explicit BigIntCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class WideDecimalCompare : public Compare @@ -167,7 +165,7 @@ class WideDecimalCompare : public Compare { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; // End of comparators for signed types @@ -176,41 +174,41 @@ class WideDecimalCompare : public Compare class UTinyIntCompare : public Compare { public: - UTinyIntCompare(const IdbSortSpec& spec) : Compare(spec) + explicit UTinyIntCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class USmallIntCompare : public Compare { public: - USmallIntCompare(const IdbSortSpec& spec) : Compare(spec) + explicit USmallIntCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class UIntCompare : public Compare { public: - UIntCompare(const IdbSortSpec& spec) : Compare(spec) + explicit UIntCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class UBigIntCompare : public Compare { public: - UBigIntCompare(const IdbSortSpec& spec) : Compare(spec) + explicit UBigIntCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; // end of comparators for unsigned types @@ -220,31 +218,31 @@ class UBigIntCompare : public Compare class DoubleCompare : public Compare { public: - DoubleCompare(const IdbSortSpec& spec) : Compare(spec) + explicit DoubleCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class LongDoubleCompare : public Compare { public: - LongDoubleCompare(const IdbSortSpec& spec) : Compare(spec) + explicit LongDoubleCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class FloatCompare : public Compare { public: - FloatCompare(const IdbSortSpec& spec) : Compare(spec) + explicit FloatCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; // End of comparators for float types @@ -253,31 +251,31 @@ class FloatCompare : public Compare class DateCompare : public Compare { public: - DateCompare(const IdbSortSpec& spec) : Compare(spec) + explicit DateCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class DatetimeCompare : public Compare { public: - DatetimeCompare(const IdbSortSpec& spec) : Compare(spec) + explicit DatetimeCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; class TimeCompare : public Compare { public: - TimeCompare(const IdbSortSpec& spec) : Compare(spec) + explicit TimeCompare(const IdbSortSpec& spec) : Compare(spec) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; }; // End of comparators for temporal types @@ -287,11 +285,11 @@ class TimeCompare : public Compare class StringCompare : public Compare { public: - StringCompare(const IdbSortSpec& spec) : Compare(spec), cs(NULL) + explicit StringCompare(const IdbSortSpec& spec) : Compare(spec), cs(nullptr) { } - int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) override; CHARSET_INFO* cs; }; @@ -301,7 +299,7 @@ class StringCompare : public Compare class CompareRule { public: - CompareRule(IdbCompare* c = NULL) : fIdbCompare(c) + explicit CompareRule(IdbCompare* c = nullptr) : fIdbCompare(c) { } @@ -317,8 +315,8 @@ class CompareRule class IdbCompare { public: - IdbCompare(){}; - virtual ~IdbCompare(){}; + IdbCompare() = default; + virtual ~IdbCompare() = default; virtual void initialize(const rowgroup::RowGroup&); void setStringTable(bool b); @@ -362,7 +360,7 @@ class OrderByRow class EqualCompData : public IdbCompare { public: - EqualCompData(std::vector& v) : fIndex(v) + explicit EqualCompData(std::vector& v) : fIndex(v) { } EqualCompData(std::vector& v, const rowgroup::RowGroup& rg) : fIndex(v) @@ -370,7 +368,7 @@ class EqualCompData : public IdbCompare initialize(rg); } - ~EqualCompData(){}; + ~EqualCompData() override = default; bool operator()(rowgroup::Row::Pointer, rowgroup::Row::Pointer); @@ -381,7 +379,7 @@ class OrderByData : public IdbCompare { public: OrderByData(const std::vector&, const rowgroup::RowGroup&); - virtual ~OrderByData(); + ~OrderByData() override; bool operator()(rowgroup::Row::Pointer p1, rowgroup::Row::Pointer p2) { @@ -401,9 +399,9 @@ class IdbOrderBy : public IdbCompare { public: IdbOrderBy(); - virtual ~IdbOrderBy(); + ~IdbOrderBy() override; - virtual void initialize(const rowgroup::RowGroup&); + void initialize(const rowgroup::RowGroup&) override; virtual void processRow(const rowgroup::Row&) = 0; virtual uint64_t getKeyLength() const = 0; virtual const std::string toString() const = 0; diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index 342baf426..ee80030e7 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -17,29 +17,19 @@ // $Id: wf_count.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG -#include -#include +// #define NDEBUG #include -#include -#include using namespace std; #include using namespace boost; #include "loggingid.h" -#include "errorcodes.h" -#include "idberrorinfo.h" using namespace logging; -#include "rowgroup.h" -using namespace rowgroup; - #include "idborderby.h" using namespace ordering; -#include "joblisttypes.h" #include "calpontsystemcatalog.h" using namespace execplan; diff --git a/utils/windowfunction/wf_count.h b/utils/windowfunction/wf_count.h index cb0844788..ab65b75dc 100644 --- a/utils/windowfunction/wf_count.h +++ b/utils/windowfunction/wf_count.h @@ -34,9 +34,9 @@ class WF_count : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -46,4 +46,3 @@ class WF_count : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index dd1d4e93e..d0e886e54 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -18,7 +18,7 @@ // $Id: wf_lead_lag.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG +// #define NDEBUG #include #include #include @@ -159,7 +159,7 @@ void WF_lead_lag::parseParms(const std::vector& parms) // parms[1]: offset ConstantColumn* cc = dynamic_cast(parms[1].get()); - if (cc != NULL) + if (cc != nullptr) { fOffsetNull = false; fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. @@ -168,7 +168,7 @@ void WF_lead_lag::parseParms(const std::vector& parms) // parms[2]: default value cc = dynamic_cast(parms[2].get()); - if (cc != NULL) + if (cc != nullptr) { fDefNull = false; getConstValue(cc, fDefault, fDefNull); @@ -176,7 +176,7 @@ void WF_lead_lag::parseParms(const std::vector& parms) // parms[3]: respect null | ignore null cc = dynamic_cast(parms[3].get()); - if (cc != NULL) + if (cc != nullptr) { bool isNull = false; // dummy. Return not used fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); diff --git a/utils/windowfunction/wf_lead_lag.h b/utils/windowfunction/wf_lead_lag.h index 7ccb41062..7f2219eae 100644 --- a/utils/windowfunction/wf_lead_lag.h +++ b/utils/windowfunction/wf_lead_lag.h @@ -33,10 +33,10 @@ class WF_lead_lag : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; + void parseParms(const std::vector&) override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -51,4 +51,3 @@ class WF_lead_lag : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index fbc2ffe18..7f6af5aab 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -18,11 +18,8 @@ // $Id: wf_min_max.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG -#include +// #define NDEBUG #include -#include -#include using namespace std; #include @@ -30,13 +27,8 @@ using namespace boost; #include "loggingid.h" #include "errorcodes.h" -#include "idberrorinfo.h" using namespace logging; -#include "rowgroup.h" -using namespace rowgroup; - -#include "joblisttypes.h" #include "calpontsystemcatalog.h" using namespace execplan; diff --git a/utils/windowfunction/wf_min_max.h b/utils/windowfunction/wf_min_max.h index caf615191..b0381f7e6 100644 --- a/utils/windowfunction/wf_min_max.h +++ b/utils/windowfunction/wf_min_max.h @@ -33,9 +33,9 @@ class WF_min_max : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -45,4 +45,3 @@ class WF_min_max : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index a79f2a402..2ee196eb3 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -18,7 +18,7 @@ // $Id: wf_nth_value.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG +// #define NDEBUG #include #include #include @@ -149,7 +149,7 @@ void WF_nth_value::parseParms(const std::vector& parms) // parms[1]: nth value ConstantColumn* cc = dynamic_cast(parms[1].get()); - if (cc != NULL) + if (cc != nullptr) { fNthNull = false; fNth = cc->getIntVal(fRow, fNthNull); // row not used, no need to setData. @@ -166,12 +166,12 @@ void WF_nth_value::parseParms(const std::vector& parms) // parms[2]: from first | from last bool isNull = false; cc = dynamic_cast(parms[2].get()); - idbassert(cc != NULL); + idbassert(cc != nullptr); fFromFirst = (cc->getIntVal(fRow, isNull) > 0); // parms[3]: respect null | ignore null cc = dynamic_cast(parms[3].get()); - idbassert(cc != NULL); + idbassert(cc != nullptr); fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); } diff --git a/utils/windowfunction/wf_nth_value.h b/utils/windowfunction/wf_nth_value.h index d4a2247ff..d12835030 100644 --- a/utils/windowfunction/wf_nth_value.h +++ b/utils/windowfunction/wf_nth_value.h @@ -33,10 +33,10 @@ class WF_nth_value : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; + void parseParms(const std::vector&) override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -49,4 +49,3 @@ class WF_nth_value : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_ntile.cpp b/utils/windowfunction/wf_ntile.cpp index 4b6634603..001c79288 100644 --- a/utils/windowfunction/wf_ntile.cpp +++ b/utils/windowfunction/wf_ntile.cpp @@ -17,24 +17,18 @@ // $Id: wf_ntile.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG -#include +// #define NDEBUG #include #include -#include using namespace std; #include using namespace boost; #include "loggingid.h" -#include "errorcodes.h" #include "idberrorinfo.h" using namespace logging; -#include "rowgroup.h" -using namespace rowgroup; - #include "idborderby.h" using namespace ordering; @@ -50,8 +44,8 @@ using namespace joblist; namespace windowfunction { -boost::shared_ptr WF_ntile::makeFunction(int id, const string& name, int ct, - WindowFunctionColumn* wc) +boost::shared_ptr WF_ntile::makeFunction(int id, const string& name, int, + WindowFunctionColumn*) { boost::shared_ptr func(new WF_ntile(id, name)); return func; @@ -70,9 +64,9 @@ void WF_ntile::resetData() void WF_ntile::parseParms(const std::vector& parms) { // parms[0]: nt - ConstantColumn* cc = dynamic_cast(parms[0].get()); + auto* cc = dynamic_cast(parms[0].get()); - if (cc != NULL) + if (cc != nullptr) { fNtileNull = false; fNtile = cc->getIntVal(fRow, fNtileNull); // row not used, no need to setData. diff --git a/utils/windowfunction/wf_ntile.h b/utils/windowfunction/wf_ntile.h index ed5ee6b77..e85524e4e 100644 --- a/utils/windowfunction/wf_ntile.h +++ b/utils/windowfunction/wf_ntile.h @@ -33,10 +33,10 @@ class WF_ntile : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; + void parseParms(const std::vector&) override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -46,4 +46,3 @@ class WF_ntile : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 2b7a34ab5..71c9728de 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -18,11 +18,10 @@ // $Id: wf_percentile.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG +// #define NDEBUG #include #include #include -#include using namespace std; #include @@ -34,13 +33,9 @@ using namespace boost; #include "idberrorinfo.h" using namespace logging; -#include "rowgroup.h" -using namespace rowgroup; - #include "idborderby.h" using namespace ordering; -#include "joblisttypes.h" #include "calpontsystemcatalog.h" #include "constantcolumn.h" using namespace execplan; @@ -206,7 +201,7 @@ void WF_percentile::parseParms(const std::vector& parms) // parms[0]: nve ConstantColumn* cc = dynamic_cast(parms[0].get()); - if (cc != NULL) + if (cc != nullptr) { fNveNull = false; fNve = cc->getDoubleVal(fRow, fNveNull); // row not used, no need to setData. @@ -339,7 +334,7 @@ void WF_percentile::operator()(int64_t b, int64_t e, int64_t c) tempvd[1] = 0; v = *(reinterpret_cast(&tempvd[0])); - //v = *(reinterpret_cast(&vd)); // XXX old code that produced out-of-bounds difference. + // v = *(reinterpret_cast(&vd)); // XXX old code that produced out-of-bounds difference. p = &v; } else // (fFunctionId == WF__PERCENTILE_DISC) diff --git a/utils/windowfunction/wf_percentile.h b/utils/windowfunction/wf_percentile.h index db08c08d2..64b092b9b 100644 --- a/utils/windowfunction/wf_percentile.h +++ b/utils/windowfunction/wf_percentile.h @@ -34,10 +34,10 @@ class WF_percentile : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; + void parseParms(const std::vector&) override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -47,4 +47,3 @@ class WF_percentile : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_ranking.cpp b/utils/windowfunction/wf_ranking.cpp index 540ca8082..0dce1399a 100644 --- a/utils/windowfunction/wf_ranking.cpp +++ b/utils/windowfunction/wf_ranking.cpp @@ -17,29 +17,19 @@ // $Id: wf_ranking.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG -#include +// #define NDEBUG #include -#include -#include using namespace std; #include using namespace boost; #include "loggingid.h" -#include "errorcodes.h" -#include "idberrorinfo.h" using namespace logging; -#include "rowgroup.h" -using namespace rowgroup; - #include "idborderby.h" using namespace ordering; -#include "joblisttypes.h" -#include "calpontsystemcatalog.h" using namespace execplan; #include "windowfunctionstep.h" diff --git a/utils/windowfunction/wf_ranking.h b/utils/windowfunction/wf_ranking.h index ecfdae97a..063d1795d 100644 --- a/utils/windowfunction/wf_ranking.h +++ b/utils/windowfunction/wf_ranking.h @@ -33,9 +33,9 @@ class WF_ranking : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -45,4 +45,3 @@ class WF_ranking : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_row_number.cpp b/utils/windowfunction/wf_row_number.cpp index 6dd58ce31..3dead0920 100644 --- a/utils/windowfunction/wf_row_number.cpp +++ b/utils/windowfunction/wf_row_number.cpp @@ -17,29 +17,19 @@ // $Id: wf_row_number.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG -#include +// #define NDEBUG #include -#include -#include using namespace std; #include using namespace boost; #include "loggingid.h" -#include "errorcodes.h" -#include "idberrorinfo.h" using namespace logging; -#include "rowgroup.h" -using namespace rowgroup; - #include "idborderby.h" using namespace ordering; -#include "joblisttypes.h" -#include "calpontsystemcatalog.h" using namespace execplan; #include "windowfunctionstep.h" @@ -49,8 +39,8 @@ using namespace joblist; namespace windowfunction { -boost::shared_ptr WF_row_number::makeFunction(int id, const string& name, int ct, - WindowFunctionColumn* wc) +boost::shared_ptr WF_row_number::makeFunction(int id, const string& name, int, + WindowFunctionColumn*) { boost::shared_ptr func(new WF_row_number(id, name)); return func; diff --git a/utils/windowfunction/wf_row_number.h b/utils/windowfunction/wf_row_number.h index 953a68c35..dcd20e718 100644 --- a/utils/windowfunction/wf_row_number.h +++ b/utils/windowfunction/wf_row_number.h @@ -33,9 +33,9 @@ class WF_row_number : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -44,4 +44,3 @@ class WF_row_number : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index 73a668acf..c24abdf47 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -18,11 +18,9 @@ // $Id: wf_stats.cpp 3932 2013-06-25 16:08:10Z xlou $ -//#define NDEBUG -#include +// #define NDEBUG #include #include -#include using namespace std; #include @@ -33,13 +31,9 @@ using namespace boost; #include "idberrorinfo.h" using namespace logging; -#include "rowgroup.h" -using namespace rowgroup; - #include "idborderby.h" using namespace ordering; -#include "joblisttypes.h" #include "calpontsystemcatalog.h" using namespace execplan; @@ -177,7 +171,7 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) long double val = (long double)valIn; count_++; long double delta = val - mean_; - mean_ += delta/count_; + mean_ += delta / count_; scaledMomentum2_ += delta * (val - mean_); } @@ -208,13 +202,13 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) if (count_ == 0) { - setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*)NULL); + setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*)nullptr); } else if (count_ == 1) { if (fFunctionId == WF__STDDEV_SAMP || fFunctionId == WF__VAR_SAMP) { - setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*)NULL); + setValue(CalpontSystemCatalog::DOUBLE, b, e, c, (double*)nullptr); } else // fFunctionId == WF__STDDEV_POP || fFunctionId == WF__VAR_POP { diff --git a/utils/windowfunction/wf_stats.h b/utils/windowfunction/wf_stats.h index b5c98830d..843b0e98a 100644 --- a/utils/windowfunction/wf_stats.h +++ b/utils/windowfunction/wf_stats.h @@ -33,9 +33,9 @@ class WF_stats : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -47,4 +47,3 @@ class WF_stats : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_sum_avg.h b/utils/windowfunction/wf_sum_avg.h index c423e7617..130fbec7c 100644 --- a/utils/windowfunction/wf_sum_avg.h +++ b/utils/windowfunction/wf_sum_avg.h @@ -38,9 +38,9 @@ class WF_sum_avg : public WindowFunctionType } // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); @@ -59,4 +59,3 @@ class WF_sum_avg : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/wf_udaf.h b/utils/windowfunction/wf_udaf.h index 81cdee481..655184f82 100644 --- a/utils/windowfunction/wf_udaf.h +++ b/utils/windowfunction/wf_udaf.h @@ -55,18 +55,18 @@ class WF_udaf : public WindowFunctionType } WF_udaf(WF_udaf& rhs); // pure virtual in base - void operator()(int64_t b, int64_t e, int64_t c); - WindowFunctionType* clone() const; - void resetData(); - void parseParms(const std::vector&); - virtual bool dropValues(int64_t, int64_t); + void operator()(int64_t b, int64_t e, int64_t c) override; + WindowFunctionType* clone() const override; + void resetData() override; + void parseParms(const std::vector&) override; + bool dropValues(int64_t, int64_t) override; mcsv1sdk::mcsv1Context& getContext() { return fUDAFContext; } - bool getInterrupted() + bool getInterrupted() const { return bInterrupted; } @@ -76,7 +76,7 @@ class WF_udaf : public WindowFunctionType return &bInterrupted; } - bool getDistinct() + bool getDistinct() const { return fDistinct; } @@ -107,4 +107,3 @@ class WF_udaf : public WindowFunctionType }; } // namespace windowfunction - diff --git a/utils/windowfunction/windowframe.h b/utils/windowfunction/windowframe.h index 2dd3c23d7..b65eb54d2 100644 --- a/utils/windowfunction/windowframe.h +++ b/utils/windowfunction/windowframe.h @@ -50,7 +50,7 @@ class WindowFrame /** @brief WindowFrame destructor */ - virtual ~WindowFrame(){}; + virtual ~WindowFrame() = default; /** @brief clone */ @@ -118,4 +118,3 @@ class WindowFrame }; } // namespace windowfunction - diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index c29c08376..14b1ed39b 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -109,14 +109,14 @@ class WindowFunctionType { public: // @brief WindowFunctionType constructor - WindowFunctionType(int id = 0, const std::string& name = "") + explicit WindowFunctionType(int id = 0, const std::string& name = "") : fFunctionId(id), fFunctionName(name), fFrameUnit(0){}; // use default copy construct // WindowFunctionType(const WindowFunctionType&); // @brief WindowFunctionType destructor - virtual ~WindowFunctionType(){}; + virtual ~WindowFunctionType() = default; // @brief virtual operator(begin, end, current, data, row) virtual void operator()(int64_t, int64_t, int64_t) = 0; @@ -216,7 +216,7 @@ class WindowFunctionType // utility methods template - void getValue(uint64_t, T&, CDT* cdt = NULL); + void getValue(uint64_t, T&, CDT* cdt = nullptr); template void setValue(int, int64_t, int64_t, int64_t, T* = NULL); template @@ -303,4 +303,3 @@ class WindowFunctionType extern std::map colType2String; } // namespace windowfunction - diff --git a/versioning/BRM/brmtypes.h b/versioning/BRM/brmtypes.h index b5472d711..b82f41e5d 100644 --- a/versioning/BRM/brmtypes.h +++ b/versioning/BRM/brmtypes.h @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include "mcs_basic_types.h" #include "logicalpartition.h" @@ -248,10 +248,10 @@ struct TableLockInfo : public messageqcpp::Serializeable std::vector dbrootList; bool overlaps(const TableLockInfo&, const std::set& sPMList) const; - EXPORT void serialize(messageqcpp::ByteStream& bs) const; + EXPORT void serialize(messageqcpp::ByteStream& bs) const override; EXPORT void serialize(std::ostream&) const; EXPORT void deserialize(std::istream&); - EXPORT void deserialize(messageqcpp::ByteStream& bs); + EXPORT void deserialize(messageqcpp::ByteStream& bs) override; EXPORT void serialize(idbdatafile::IDBDataFile*) const; EXPORT void deserialize(idbdatafile::IDBDataFile*); EXPORT void serialize(char* buffer, uint32_t& offset); @@ -259,7 +259,7 @@ struct TableLockInfo : public messageqcpp::Serializeable bool operator<(const TableLockInfo&) const; private: - void serializeElement(char* buffer, const char* src, const uint32_t size, uint32_t& offset); + void serializeElement(char* buffer, const char* src, uint32_t size, uint32_t& offset); }; /// A Serializeable version of InlineLBIDRange @@ -274,15 +274,15 @@ class LBIDRange : public messageqcpp::Serializeable { } EXPORT LBIDRange(const LBIDRange& l); - EXPORT LBIDRange(const InlineLBIDRange& l); + EXPORT explicit LBIDRange(const InlineLBIDRange& l); EXPORT LBIDRange& operator=(const LBIDRange& l); EXPORT LBIDRange& operator=(const InlineLBIDRange& l); - EXPORT virtual ~LBIDRange(); + EXPORT ~LBIDRange() override; /** The Serializeable interface. Exports the instance to the bytestream */ - EXPORT virtual void serialize(messageqcpp::ByteStream& bs) const; + EXPORT void serialize(messageqcpp::ByteStream& bs) const override; /** The Serializeable interface. Initializes itself from the bytestrem. */ - EXPORT virtual void deserialize(messageqcpp::ByteStream& bs); + EXPORT void deserialize(messageqcpp::ByteStream& bs) override; }; /* To support bulkVSSLookup() */ @@ -312,7 +312,7 @@ struct BulkUpdateDBRootArg { return startLBID < b.startLBID; } - BulkUpdateDBRootArg(LBID_t l = 0, uint16_t d = 0) : startLBID(l), dbRoot(d) + explicit BulkUpdateDBRootArg(LBID_t l = 0, uint16_t d = 0) : startLBID(l), dbRoot(d) { } }; @@ -347,9 +347,9 @@ class VBRange : public messageqcpp::Serializeable EXPORT VBRange(); EXPORT VBRange(const VBRange& v); EXPORT VBRange& operator=(const VBRange& v); - EXPORT virtual ~VBRange(); - EXPORT virtual void serialize(messageqcpp::ByteStream& bs) const; - EXPORT virtual void deserialize(messageqcpp::ByteStream& bs); + EXPORT ~VBRange() override; + EXPORT void serialize(messageqcpp::ByteStream& bs) const override; + EXPORT void deserialize(messageqcpp::ByteStream& bs) override; }; // Structure used to return HWM information for each DbRoot in a PM @@ -371,7 +371,7 @@ struct EmDbRootHWMInfo { init(0); } - EmDbRootHWMInfo(uint16_t root) + explicit EmDbRootHWMInfo(uint16_t root) { init(root); } @@ -614,13 +614,13 @@ class QueryContext : public messageqcpp::Serializeable currentTxns.reset(new std::vector()); } - void serialize(messageqcpp::ByteStream& bs) const + void serialize(messageqcpp::ByteStream& bs) const override { bs << currentScn; serializeInlineVector(bs, *currentTxns); } - void deserialize(messageqcpp::ByteStream& bs) + void deserialize(messageqcpp::ByteStream& bs) override { bs >> currentScn; deserializeInlineVector(bs, *currentTxns); diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index b1e3e9890..84eb88b08 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -115,13 +115,7 @@ EMCasualPartition_struct::EMCasualPartition_struct() isValid = CP_INVALID; } -EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum, - const char status) - : sequenceNum(seqNum), isValid(status), loVal(lo), hiVal(hi) -{ -} - -EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum) +EMCasualPartition_struct::EMCasualPartition_struct(int64_t lo, int64_t hi, int32_t seqNum) { loVal = lo; hiVal = hi; @@ -129,8 +123,13 @@ EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64 isValid = CP_INVALID; } -EMCasualPartition_struct::EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, - const int32_t seqNum) +EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum, + const char status) + : sequenceNum(seqNum), isValid(status), loVal(lo), hiVal(hi) +{ +} + +EMCasualPartition_struct::EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, int32_t seqNum) { bigLoVal = bigLo; bigHiVal = bigHi; @@ -261,7 +260,7 @@ ExtentMapRBTreeImpl::ExtentMapRBTreeImpl(unsigned key, off_t size, bool readOnly boost::mutex FreeListImpl::fInstanceMutex; /*static*/ -FreeListImpl* FreeListImpl::fInstance = 0; +FreeListImpl* FreeListImpl::fInstance = nullptr; /*static*/ FreeListImpl* FreeListImpl::makeFreeListImpl(unsigned key, off_t size, bool readOnly) diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index 96e5f4b40..67c42d81b 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -162,10 +162,9 @@ struct EMCasualPartition_struct int64_t hiVal; }; EXPORT EMCasualPartition_struct(); - EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum, - const char status); + EXPORT EMCasualPartition_struct(int64_t lo, int64_t hi, int32_t seqNum); + EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, int32_t seqNum); + EXPORT EMCasualPartition_struct(int64_t lo, int64_t hi, int32_t seqNum, char status); EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em); }; @@ -276,7 +275,7 @@ class ExtentMapRBTreeImpl if (fInstance) { delete fInstance; - fInstance = NULL; + fInstance = nullptr; } } @@ -320,22 +319,22 @@ class ExtentMapRBTreeImpl class FreeListImpl { public: - ~FreeListImpl(){}; + ~FreeListImpl() = default; static FreeListImpl* makeFreeListImpl(unsigned key, off_t size, bool readOnly = false); - + static void refreshShmWithLock() { boost::mutex::scoped_lock lk(fInstanceMutex); return refreshShm(); } - + static void refreshShm() { if (fInstance) { delete fInstance; - fInstance = NULL; + fInstance = nullptr; } } @@ -387,7 +386,7 @@ class FreeListImpl class ExtentMapIndexImpl { public: - ~ExtentMapIndexImpl(){}; + ~ExtentMapIndexImpl() = default; static ExtentMapIndexImpl* makeExtentMapIndexImpl(unsigned key, off_t size, bool readOnly = false); static void refreshShm() @@ -505,7 +504,7 @@ class ExtentMap : public Undoable { public: EXPORT ExtentMap(); - EXPORT ~ExtentMap(); + EXPORT ~ExtentMap() override; /** @brief Loads the ExtentMap entries from a file * @@ -1010,9 +1009,9 @@ class ExtentMap : public Undoable EXPORT void setReadOnly(); - EXPORT virtual void undoChanges(); + EXPORT void undoChanges() override; - EXPORT virtual void confirmChanges(); + EXPORT void confirmChanges() override; EXPORT int markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); EXPORT int markInvalid(const std::vector& lbids, diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index 926ff6c93..22e98611e 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -66,7 +66,7 @@ class BulkLoad : public FileOp /** * @brief BulkLoad destructor */ - EXPORT ~BulkLoad(); + EXPORT ~BulkLoad() override; /** * @brief Load job information @@ -523,12 +523,12 @@ inline void BulkLoad::setUsername(const std::string& username) inline void BulkLoad::startTimer() { - gettimeofday(&fStartTime, 0); + gettimeofday(&fStartTime, nullptr); } inline void BulkLoad::stopTimer() { - gettimeofday(&fEndTime, 0); + gettimeofday(&fEndTime, nullptr); fTotalTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) - (fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0)); } diff --git a/writeengine/bulk/we_colbufcompressed.h b/writeengine/bulk/we_colbufcompressed.h index 13e81b06e..0c4570d5e 100644 --- a/writeengine/bulk/we_colbufcompressed.h +++ b/writeengine/bulk/we_colbufcompressed.h @@ -54,19 +54,19 @@ class ColumnBufferCompressed : public ColumnBuffer /** @brief default Destructor */ - virtual ~ColumnBufferCompressed(); + ~ColumnBufferCompressed() override; /** @brief Final flushing of data and headers prior to closing the file. * @param bTruncFile is file to be truncated * @return NO_ERROR or success */ - virtual int finishFile(bool bTruncFile); + int finishFile(bool bTruncFile) override; /** @brief Reset the ColBuf to-be-compressed buffer prior to importing the * next extent. * @param startFileOffset Byte offset where next extent chunk will start */ - virtual int resetToBeCompressedColBuf(long long& startFileOffset); + int resetToBeCompressedColBuf(long long& startFileOffset) override; /** @brief file mutator * @@ -74,7 +74,7 @@ class ColumnBufferCompressed : public ColumnBuffer * @param startHwm Starting HWM for cFile * @param hdrs Headers with ptr information. */ - virtual int setDbFile(IDBDataFile* const cFile, HWM startHwm, const char* hdrs); + int setDbFile(IDBDataFile* const cFile, HWM startHwm, const char* hdrs) override; /** @brief Write data to FILE * @@ -83,7 +83,7 @@ class ColumnBufferCompressed : public ColumnBuffer * @param fillUpWEmpties The flag to fill the buffer with empty magic * values up to the block boundary. */ - virtual int writeToFile(int startOffset, int writeSize, bool fillUpWEmpties = false); + int writeToFile(int startOffset, int writeSize, bool fillUpWEmpties = false) override; private: // Disable copy constructor and assignment operator by declaring and diff --git a/writeengine/bulk/we_colbufmgr.h b/writeengine/bulk/we_colbufmgr.h index a0adc9356..874389653 100644 --- a/writeengine/bulk/we_colbufmgr.h +++ b/writeengine/bulk/we_colbufmgr.h @@ -260,9 +260,9 @@ class ColumnBufferManagerDctnry : public ColumnBufferManager { public: ColumnBufferManagerDctnry(ColumnInfo* pColInfo, int colWidth, Log* logger, int compressionType); - virtual ~ColumnBufferManagerDctnry(); + ~ColumnBufferManagerDctnry() override; - virtual int rowsExtentCheck(int nRows, int& nRows2); + int rowsExtentCheck(int nRows, int& nRows2) override; using ColumnBufferManager::writeToFileExtentCheck; virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize); }; diff --git a/writeengine/bulk/we_colextinf.h b/writeengine/bulk/we_colextinf.h index bba0094ff..cf2de5700 100644 --- a/writeengine/bulk/we_colextinf.h +++ b/writeengine/bulk/we_colextinf.h @@ -136,12 +136,8 @@ struct uint64Hasher class ColExtInfBase { public: - ColExtInfBase() - { - } - virtual ~ColExtInfBase() - { - } + ColExtInfBase() = default; + virtual ~ColExtInfBase() = default; virtual void addFirstEntry(RID lastInputRow, BRM::LBID_t lbid, bool bIsNewExtent) { @@ -192,9 +188,7 @@ class ColExtInf : public ColExtInfBase ColExtInf(OID oid, Log* logger) : fColOid(oid), fLog(logger) { } - virtual ~ColExtInf() - { - } + ~ColExtInf() override = default; /** @brief Add an entry for first extent, for the specified Row and LBID. * @param lastInputRow Last input Row for old extent we are adding data to @@ -202,7 +196,7 @@ class ColExtInf : public ColExtInfBase * @param bIsNewExtent Treat as new or existing extent when CP min/max is * sent to BRM */ - virtual void addFirstEntry(RID lastInputRow, BRM::LBID_t lbid, bool bIsNewExtent); + void addFirstEntry(RID lastInputRow, BRM::LBID_t lbid, bool bIsNewExtent) override; /** @brief Add or update an entry for the specified Row and its min/max val. * If new extent, LBID will be added later when extent is allocated. @@ -213,31 +207,31 @@ class ColExtInf : public ColExtInfBase template void addOrUpdateEntryTemplate(RID lastInputRow, T minVal, T maxVal, ColDataType colDataType, int width); - virtual void addOrUpdateEntry(RID lastInputRow, int64_t minVal, int64_t maxVal, ColDataType colDataType, - int width) + void addOrUpdateEntry(RID lastInputRow, int64_t minVal, int64_t maxVal, ColDataType colDataType, + int width) override { addOrUpdateEntryTemplate(lastInputRow, minVal, maxVal, colDataType, width); } - virtual void addOrUpdateEntry(RID lastInputRow, int128_t minVal, int128_t maxVal, ColDataType colDataType, - int width) + void addOrUpdateEntry(RID lastInputRow, int128_t minVal, int128_t maxVal, ColDataType colDataType, + int width) override { addOrUpdateEntryTemplate(lastInputRow, minVal, maxVal, colDataType, width); } /** @brief Send updated Casual Partition (CP) info to BRM. */ - virtual void getCPInfoForBRM(JobColumn column, BRMReporter& brmReporter); + void getCPInfoForBRM(JobColumn column, BRMReporter& brmReporter) override; /** @brief Debug print function. */ - virtual void print(const JobColumn& column); + void print(const JobColumn& column) override; /** @brief Add extent's LBID to the oldest entry that is awaiting an LBID * @param startLbid Starting LBID for a pending extent. * @return NO_ERROR upon success; else error if extent entry not found */ - virtual int updateEntryLbid(BRM::LBID_t startLbid); + int updateEntryLbid(BRM::LBID_t startLbid) override; private: OID fColOid; // Column OID for the relevant extents diff --git a/writeengine/bulk/we_colopbulk.h b/writeengine/bulk/we_colopbulk.h index 618594e79..373a043fc 100644 --- a/writeengine/bulk/we_colopbulk.h +++ b/writeengine/bulk/we_colopbulk.h @@ -44,16 +44,15 @@ class ColumnOpBulk : public ColumnOp public: ColumnOpBulk(); ColumnOpBulk(Log* logger, int compressionType); - virtual ~ColumnOpBulk(); + ~ColumnOpBulk() override; - virtual bool abbreviatedExtent(IDBDataFile*, int) const; - virtual int blocksInFile(IDBDataFile*) const; - virtual IDBDataFile* openFile(const WriteEngine::Column& column, uint16_t dbRoot, uint32_t partition, - uint16_t segment, std::string& segFile, bool useTmpSuffix, - const char* mode = "r+b", int ioBuffSize = DEFAULT_BUFSIZ, - bool isReadOnly = false) const; - virtual int readBlock(IDBDataFile*, unsigned char*, const uint64_t); - virtual int saveBlock(IDBDataFile*, const unsigned char*, const uint64_t); + bool abbreviatedExtent(IDBDataFile*, int) const override; + int blocksInFile(IDBDataFile*) const override; + IDBDataFile* openFile(const WriteEngine::Column& column, uint16_t dbRoot, uint32_t partition, + uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", + int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const override; + int readBlock(IDBDataFile*, unsigned char*, const uint64_t) override; + int saveBlock(IDBDataFile*, const unsigned char*, const uint64_t) override; }; } // namespace WriteEngine diff --git a/writeengine/bulk/we_columnautoinc.h b/writeengine/bulk/we_columnautoinc.h index e53d1152e..8ddfea84a 100644 --- a/writeengine/bulk/we_columnautoinc.h +++ b/writeengine/bulk/we_columnautoinc.h @@ -97,9 +97,9 @@ class ColumnAutoIncJob : public ColumnAutoInc { public: explicit ColumnAutoIncJob(Log* logger); - virtual ~ColumnAutoIncJob(); + ~ColumnAutoIncJob() override; - virtual int reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue); + int reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue) override; }; //------------------------------------------------------------------------------ @@ -115,9 +115,9 @@ class ColumnAutoIncIncremental : public ColumnAutoInc { public: explicit ColumnAutoIncIncremental(Log* logger); - virtual ~ColumnAutoIncIncremental(); + ~ColumnAutoIncIncremental() override; - virtual int reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue); + int reserveNextRange(uint32_t autoIncCount, uint64_t& nextValue) override; }; } // namespace WriteEngine diff --git a/writeengine/bulk/we_columninfo.h b/writeengine/bulk/we_columninfo.h index db548e7d0..035b09245 100644 --- a/writeengine/bulk/we_columninfo.h +++ b/writeengine/bulk/we_columninfo.h @@ -184,7 +184,7 @@ class ColumnInfo : public WeUIDGID /** @brief Destructor */ - virtual ~ColumnInfo(); + ~ColumnInfo() override; /** @brief Returns last input Row num in current "logical" extent; used * to track min/max value per extent, as the data is parsed. 0-based @@ -205,7 +205,8 @@ class ColumnInfo : public WeUIDGID * returns the assigned tokens (tokenBuf) to be stored in the * corresponding column token file. */ - int updateDctnryStoreParquet(std::shared_ptr columnData, int tokenPos, const int totalRow, char* tokenBuf); + int updateDctnryStoreParquet(std::shared_ptr columnData, int tokenPos, const int totalRow, + char* tokenBuf); /** @brief Update dictionary method. * Parses and stores specified strings into the store file, and diff --git a/writeengine/bulk/we_columninfocompressed.h b/writeengine/bulk/we_columninfocompressed.h index 92890874a..4d06fecfe 100644 --- a/writeengine/bulk/we_columninfocompressed.h +++ b/writeengine/bulk/we_columninfocompressed.h @@ -50,14 +50,14 @@ class ColumnInfoCompressed : public ColumnInfo /** @brief Destructor */ - virtual ~ColumnInfoCompressed(); + ~ColumnInfoCompressed() override; /** @brief Close the current Column file. * @param bCompletedExtent are we completing an extent * @param bAbort indicates if job is aborting and file should be * closed without doing extra work: flushing buffer, etc. */ - virtual int closeColumnFile(bool bCompletingExtent, bool bAbort); + int closeColumnFile(bool bCompletingExtent, bool bAbort) override; /** @brief Truncate specified dictionary file. Only applies if compressed. * @param dctnryOid Dictionary store OID @@ -65,21 +65,21 @@ class ColumnInfoCompressed : public ColumnInfo * @param pNum Partition number of relevant dictionary store segment file. * @param sNum Segment number of relevant dictionary store segment file. */ - virtual int truncateDctnryStore(OID dctnryOid, uint16_t root, uint32_t pNum, uint16_t sNum) const; + int truncateDctnryStore(OID dctnryOid, uint16_t root, uint32_t pNum, uint16_t sNum) const override; private: - virtual int resetFileOffsetsNewExtent(const char* hdr); + int resetFileOffsetsNewExtent(const char* hdr) override; // Prepare initial compressed column seg file for importing of data. // oldHWM - Current HWM prior to initial block skipping. This is only // used for abbreviated extents, to detect when block skipping has // caused us to require a full expanded extent. // newHWM - Starting point for adding data after initial blockskipping - virtual int setupInitialColumnFile(HWM oldHWM, HWM newHWM); + int setupInitialColumnFile(HWM oldHWM, HWM newHWM) override; - virtual int saveDctnryStoreHWMChunk(bool& needBackup); - virtual int extendColumnOldExtent(uint16_t dbRootNext, uint32_t partitionNext, uint16_t segmentNext, - HWM hwmNext); + int saveDctnryStoreHWMChunk(bool& needBackup) override; + int extendColumnOldExtent(uint16_t dbRootNext, uint32_t partitionNext, uint16_t segmentNext, + HWM hwmNext) override; RBMetaWriter* fRBMetaWriter; FileOp fTruncateDctnryFileOp; // Used to truncate dctnry store file diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index 98a6048ea..33325ecfd 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -224,7 +224,7 @@ class TableInfo : public WeUIDGID /** @brief Default destructor */ - ~TableInfo(); + ~TableInfo() override; /** @brief Acquire the DB table lock for this table */ diff --git a/writeengine/bulk/we_tempxmlgendata.h b/writeengine/bulk/we_tempxmlgendata.h index 385d0b47d..8562f00fe 100644 --- a/writeengine/bulk/we_tempxmlgendata.h +++ b/writeengine/bulk/we_tempxmlgendata.h @@ -38,9 +38,9 @@ class TempXMLGenData : public XMLGenData public: TempXMLGenData(const std::string& jobId, const std::string& schema, const std::string& table); - virtual ~TempXMLGenData(); + ~TempXMLGenData() override; - virtual void print(std::ostream& os) const; + void print(std::ostream& os) const override; private: TempXMLGenData(const TempXMLGenData&); // disable default copy ctor diff --git a/writeengine/shared/we_brm.h b/writeengine/shared/we_brm.h index e091e38ae..59afea48f 100644 --- a/writeengine/shared/we_brm.h +++ b/writeengine/shared/we_brm.h @@ -381,7 +381,7 @@ class BRMWrapper : public WEObj * @brief Commit the transaction */ EXPORT int commit(const BRM::VER_t transID); - EXPORT uint8_t newCpimportJob(uint32_t &jobId); + EXPORT uint8_t newCpimportJob(uint32_t& jobId); EXPORT void finishCpimportJob(uint32_t jobId); /** @@ -489,10 +489,9 @@ inline BRMWrapper::BRMWrapper() inline BRMWrapper::~BRMWrapper() { - if (blockRsltnMgrPtr) - delete blockRsltnMgrPtr; + delete blockRsltnMgrPtr; - blockRsltnMgrPtr = 0; + blockRsltnMgrPtr = nullptr; } inline BRM::DBRM* BRMWrapper::getDbrmObject() diff --git a/writeengine/shared/we_bulkrollbackfilecompressed.h b/writeengine/shared/we_bulkrollbackfilecompressed.h index 6a91f65fc..bb133c2b7 100644 --- a/writeengine/shared/we_bulkrollbackfilecompressed.h +++ b/writeengine/shared/we_bulkrollbackfilecompressed.h @@ -53,7 +53,7 @@ class BulkRollbackFileCompressed : public BulkRollbackFile /** @brief BulkRollbackFile destructor */ - virtual ~BulkRollbackFileCompressed(); + ~BulkRollbackFileCompressed() override; /** @brief Do we reinit trailing blocks in the HWM extent for the specified * segment file @@ -63,7 +63,7 @@ class BulkRollbackFileCompressed : public BulkRollbackFile * @param partNum Partition number for the segment file in question * @param segNum Segment number for the segment file in question */ - virtual bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const; + bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const override; /** @brief Reinitialize the specified column segment file starting at * startOffsetBlk, and truncate trailing extents. @@ -78,10 +78,10 @@ class BulkRollbackFileCompressed : public BulkRollbackFile * @param colWidth Width in bytes of column. * @param restoreHwmChk Restore HWM chunk */ - virtual void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, - long long startOffsetBlk, int nBlocks, - execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth, - bool restoreHwmChk); + void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks, + execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth, + bool restoreHwmChk) override; /** @brief Reinitialize the specified dictionary store segment file starting * at startOffsetBlk, and truncate trailing extents. @@ -93,8 +93,8 @@ class BulkRollbackFileCompressed : public BulkRollbackFile * reinitialized * @param nBlocks Number of blocks to be reinitialized */ - virtual void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, - long long startOffsetBlk, int nBlocks); + void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks) override; /** @brief Truncate the specified segment file to a specified num of bytes * @param columnOID OID of the relevant segment file @@ -103,8 +103,8 @@ class BulkRollbackFileCompressed : public BulkRollbackFile * @param segNum Segment number of the relevant segment file * @param fileSizeBlocks Number of blocks to retain in the file */ - virtual void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, - long long filesSizeBlocks); + void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long filesSizeBlocks) override; private: // Disable unnecessary copy constructor and assignment operator diff --git a/writeengine/shared/we_bulkrollbackfilecompressedhdfs.h b/writeengine/shared/we_bulkrollbackfilecompressedhdfs.h index fa33855a4..4a270e399 100644 --- a/writeengine/shared/we_bulkrollbackfilecompressedhdfs.h +++ b/writeengine/shared/we_bulkrollbackfilecompressedhdfs.h @@ -51,7 +51,7 @@ class BulkRollbackFileCompressedHdfs : public BulkRollbackFile /** @brief BulkRollbackFile destructor */ - virtual ~BulkRollbackFileCompressedHdfs(); + ~BulkRollbackFileCompressedHdfs() override; /** @brief Do we reinit trailing blocks in the HWM extent for the specified * segment file @@ -61,7 +61,7 @@ class BulkRollbackFileCompressedHdfs : public BulkRollbackFile * @param partNum Partition number for the segment file in question * @param segNum Segment number for the segment file in question */ - virtual bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const; + bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const override; /** @brief Reinitialize the specified column segment file starting at * startOffsetBlk, and truncate trailing extents. @@ -76,10 +76,10 @@ class BulkRollbackFileCompressedHdfs : public BulkRollbackFile * @param colWidth Width in bytes of column. * @param restoreHwmChk Restore HWM chunk */ - virtual void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, - long long startOffsetBlk, int nBlocks, - execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth, - bool restoreHwmChk); + void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks, + execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth, + bool restoreHwmChk) override; /** @brief Reinitialize the specified dictionary store segment file starting * at startOffsetBlk, and truncate trailing extents. @@ -91,8 +91,8 @@ class BulkRollbackFileCompressedHdfs : public BulkRollbackFile * reinitialized * @param nBlocks Number of blocks to be reinitialized */ - virtual void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, - long long startOffsetBlk, int nBlocks); + void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long startOffsetBlk, int nBlocks) override; /** @brief Truncate the specified segment file to a specified num of bytes * @param columnOID OID of the relevant segment file @@ -101,8 +101,8 @@ class BulkRollbackFileCompressedHdfs : public BulkRollbackFile * @param segNum Segment number of the relevant segment file * @param fileSizeBlocks Number of blocks to retain in the file */ - virtual void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, - long long filesSizeBlocks); + void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum, + long long filesSizeBlocks) override; private: // Disable unnecessary copy constructor and assignment operator diff --git a/writeengine/shared/we_bulkrollbackmgr.h b/writeengine/shared/we_bulkrollbackmgr.h index b7c0bafba..a239abb63 100644 --- a/writeengine/shared/we_bulkrollbackmgr.h +++ b/writeengine/shared/we_bulkrollbackmgr.h @@ -68,7 +68,7 @@ class BulkRollbackMgr * Currently used for logging only. */ EXPORT BulkRollbackMgr(OID tableOID, uint64_t lockID, const std::string& tableName, - const std::string& applName, Log* logger = 0); + const std::string& applName, Log* logger = nullptr); /** * @brief BulkRollbackMgr destructor diff --git a/writeengine/shared/we_cache.cpp b/writeengine/shared/we_cache.cpp index 317ca8009..11825427a 100644 --- a/writeengine/shared/we_cache.cpp +++ b/writeengine/shared/we_cache.cpp @@ -27,11 +27,11 @@ using namespace std; namespace WriteEngine { -CacheControl* Cache::m_cacheParam = NULL; -FreeBufList* Cache::m_freeList = NULL; -CacheMap* Cache::m_lruList = NULL; -CacheMap* Cache::m_writeList = NULL; - bool Cache::m_useCache = false; +CacheControl* Cache::m_cacheParam = nullptr; +FreeBufList* Cache::m_freeList = nullptr; +CacheMap* Cache::m_lruList = nullptr; +CacheMap* Cache::m_writeList = nullptr; +bool Cache::m_useCache = false; /*********************************************************** * DESCRIPTION: * Clear all list and free memory @@ -47,7 +47,7 @@ void Cache::clear() size_t i; // free list - if (m_freeList != NULL) + if (m_freeList != nullptr) { for (i = 0; i < m_freeList->size(); i++) { @@ -57,7 +57,7 @@ void Cache::clear() } // LRU list - if (m_lruList != NULL) + if (m_lruList != nullptr) { for (it = m_lruList->begin(); it != m_lruList->end(); it++) { @@ -70,7 +70,7 @@ void Cache::clear() } // Write list - if (m_writeList != NULL) + if (m_writeList != nullptr) { for (it = m_writeList->begin(); it != m_writeList->end(); it++) { @@ -97,13 +97,13 @@ int Cache::flushCache() BlockBuffer* curBuf; // add lock here - if (m_lruList && m_lruList->size() > 0) + if (m_lruList && !m_lruList->empty()) { bHasReadBlock = true; - for (CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++) + for (auto& it : *m_lruList) { - curBuf = it->second; + curBuf = it.second; curBuf->clear(); m_freeList->push_back(curBuf); } @@ -112,19 +112,19 @@ int Cache::flushCache() } // must write to disk first - if (m_writeList && m_writeList->size() > 0) + if (m_writeList && !m_writeList->empty()) { if (!bHasReadBlock) - for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++) + for (auto& it : *m_writeList) { - curBuf = it->second; + curBuf = it.second; curBuf->clear(); m_freeList->push_back(curBuf); } else - for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++) + for (auto& it : *m_writeList) { - curBuf = it->second; + curBuf = it.second; (*curBuf).block.dirty = false; processCacheMap(m_lruList, curBuf, INSERT); } @@ -152,7 +152,7 @@ void Cache::freeMemory() size_t i; // free list - if (m_freeList != NULL) + if (m_freeList != nullptr) { for (i = 0; i < m_freeList->size(); i++) { @@ -163,11 +163,11 @@ void Cache::freeMemory() m_freeList->clear(); delete m_freeList; - m_freeList = NULL; + m_freeList = nullptr; } // LRU list - if (m_lruList != NULL) + if (m_lruList != nullptr) { for (it = m_lruList->begin(); it != m_lruList->end(); it++) { @@ -178,11 +178,11 @@ void Cache::freeMemory() m_lruList->clear(); delete m_lruList; - m_lruList = NULL; + m_lruList = nullptr; } // Write list - if (m_writeList != NULL) + if (m_writeList != nullptr) { for (it = m_writeList->begin(); it != m_writeList->end(); it++) { @@ -193,14 +193,14 @@ void Cache::freeMemory() m_writeList->clear(); delete m_writeList; - m_writeList = NULL; + m_writeList = nullptr; } // param - if (m_cacheParam != NULL) + if (m_cacheParam != nullptr) { delete m_cacheParam; - m_cacheParam = NULL; + m_cacheParam = nullptr; } } @@ -282,7 +282,7 @@ int Cache::insertLRUList(CommBlock& cb, const uint64_t lbid, const uint64_t fbo, BlockBuffer* buffer; vector::iterator it; - if (m_freeList->size() == 0) + if (m_freeList->empty()) return ERR_FREE_LIST_EMPTY; // make sure flush first if necessary @@ -397,10 +397,10 @@ void Cache::printCacheList() cout << "\nFree List has " << m_freeList->size() << " elements" << endl; cout << "LRU List has " << m_lruList->size() << " elements" << endl; - for (CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++) + for (const auto& it : *m_lruList) { - buffer = it->second; - cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType + buffer = it.second; + cout << "\t[" << i++ << "] key=" << it.first << " listType=" << buffer->listType << " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo << " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl; } @@ -408,10 +408,10 @@ void Cache::printCacheList() i = 0; cout << "Write List has " << m_writeList->size() << " elements" << endl; - for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++) + for (const auto& it : *m_writeList) { - buffer = it->second; - cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType + buffer = it.second; + cout << "\t[" << i++ << "] key=" << it.first << " listType=" << buffer->listType << " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo << " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl; } diff --git a/writeengine/shared/we_cache.h b/writeengine/shared/we_cache.h index 1367a9708..41bd711e2 100644 --- a/writeengine/shared/we_cache.h +++ b/writeengine/shared/we_cache.h @@ -23,11 +23,7 @@ #pragma once #include -#if __GNUC__ == 4 && __GNUC_MINOR__ < 2 -#include -#else -#include -#endif +#include #include #include @@ -65,13 +61,7 @@ struct eqCacheKey } }; -// typedef hash_map, eqSig> DCTNRYHASHMAP; -#if __GNUC__ == 4 && __GNUC_MINOR__ < 2 -typedef __gnu_cxx::hash_map, eqCacheKey> CacheMap; -#else -typedef std::tr1::unordered_map, eqCacheKey> CacheMap; -#endif -// typedef __gnu_cxx::hash_map CacheMap; +typedef std::unordered_map, eqCacheKey> CacheMap; typedef CacheMap::iterator CacheMapIt; // typedef CacheMap LRUBufList; /** @brief Least Recent Used Buffer list */ @@ -84,16 +74,12 @@ class Cache /** * @brief Constructor */ - Cache() - { - } + Cache() = default; /** * @brief Default Destructor */ - ~Cache() - { - } + ~Cache() = default; /** * @brief Check whether cache key exists @@ -215,7 +201,7 @@ class Cache static CacheMap* m_lruList; // LRU buffer list static CacheMap* m_writeList; // Write buffer list - EXPORT static bool m_useCache; // Use cache flag + EXPORT static bool m_useCache; // Use cache flag private: }; diff --git a/writeengine/shared/we_chunkmanager.h b/writeengine/shared/we_chunkmanager.h index 2cec57dff..fca2714fe 100644 --- a/writeengine/shared/we_chunkmanager.h +++ b/writeengine/shared/we_chunkmanager.h @@ -35,7 +35,7 @@ #define EXPORT -//#define IDB_COMP_DEBUG +// #define IDB_COMP_DEBUG #ifdef IDB_COMP_DEBUG #define WE_COMP_DBG(x) \ { \ @@ -136,7 +136,7 @@ class CompFileData , fColDataType(colDataType) , fColWidth(colWidth) , fDctnryCol(false) - , fFilePtr(NULL) + , fFilePtr(nullptr) , fCompressionType(1) , fReadOnly(readOnly) { diff --git a/writeengine/shared/we_config.h b/writeengine/shared/we_config.h index 3620619b5..946116d4a 100644 --- a/writeengine/shared/we_config.h +++ b/writeengine/shared/we_config.h @@ -30,7 +30,7 @@ #include "we_obj.h" -//#define SHARED_NOTHING_DEMO_2 +// #define SHARED_NOTHING_DEMO_2 #define EXPORT @@ -44,16 +44,12 @@ class Config /** * @brief Constructor */ - Config() - { - } + Config() = default; /** * @brief Default Destructor */ - ~Config() - { - } + ~Config() = default; /** * @brief Get DB root (for local PM) @@ -187,7 +183,7 @@ class Config static boost::mutex m_bulkRoot_lk; // mutex for m_bulkRoot sync #endif static int m_WaitPeriod; // secs to wait for transaction - static unsigned m_FilesPerColumnPartition; //# seg files per partition + static unsigned m_FilesPerColumnPartition; // # seg files per partition static unsigned m_ExtentsPerSegmentFile; // # extents per segment file static int m_BulkProcessPriority; // cpimport.bin proc priority static std::string m_BulkRollbackDir; // bulk rollback meta data dir diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 9ae0784b8..0d29e3d9f 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -201,7 +201,7 @@ struct Convertor::dmFilePathArgs_t const std::string Convertor::getTimeStr() { char buf[sizeof(DATE_TIME_FORMAT) + 10] = {0}; - time_t curTime = time(NULL); + time_t curTime = time(nullptr); struct tm pTime; localtime_r(&curTime, &pTime); string timeStr; @@ -250,7 +250,7 @@ const std::string Convertor::int2Str(int val) long long Convertor::convertDecimalString(const char* field, int fieldLength, int scale) { - long double dval = strtold(field, NULL); + long double dval = strtold(field, nullptr); long long ret = 0; // move scale digits to the left of the decimal point @@ -634,9 +634,9 @@ void Convertor::convertColType(ColStruct* curStruct) { CalpontSystemCatalog::ColDataType dataType // This will be updated later, = CalpontSystemCatalog::CHAR; // CHAR used only for initialization. - ColType* internalType = NULL; + ColType* internalType = nullptr; bool bTokenFlag = false; - int* width = NULL; + int* width = nullptr; dataType = curStruct->colDataType; internalType = &(curStruct->colType); diff --git a/writeengine/shared/we_dbfileop.h b/writeengine/shared/we_dbfileop.h index f0559622a..0a5bc0dba 100644 --- a/writeengine/shared/we_dbfileop.h +++ b/writeengine/shared/we_dbfileop.h @@ -46,7 +46,7 @@ class DbFileOp : public FileOp /** * @brief Default Destructor */ - EXPORT virtual ~DbFileOp(); + EXPORT ~DbFileOp() override; EXPORT virtual int flushCache(); diff --git a/writeengine/splitter/we_brmupdater.h b/writeengine/splitter/we_brmupdater.h index 09aeb7e38..68a7afe2b 100644 --- a/writeengine/splitter/we_brmupdater.h +++ b/writeengine/splitter/we_brmupdater.h @@ -36,12 +36,10 @@ class WESDHandler; // forward deceleration class WEBrmUpdater { public: - WEBrmUpdater(WESDHandler& Ref) : fRef(Ref), fpBrm(0) - { - } - virtual ~WEBrmUpdater() + WEBrmUpdater(WESDHandler& Ref) : fRef(Ref), fpBrm(nullptr) { } + virtual ~WEBrmUpdater() = default; public: bool updateCasualPartitionAndHighWaterMarkInBRM(); @@ -59,7 +57,7 @@ class WEBrmUpdater void releaseBrmConnection() { delete fpBrm; - fpBrm = 0; + fpBrm = nullptr; } public: diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index 310ec3170..b5bc48af9 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -34,9 +34,7 @@ class WECmdArgs { public: WECmdArgs(int argc, char** argv); - virtual ~WECmdArgs() - { - } + virtual ~WECmdArgs() = default; typedef std::vector VecInts; typedef std::vector VecArgs; diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index 3db0027d5..9eb84db59 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -21,7 +21,7 @@ #pragma once -#include +#include #include "we_dbfileop.h" #include "brmtypes.h" @@ -54,7 +54,7 @@ class ColumnOp : public DbFileOp /** * @brief Default Destructor */ - EXPORT virtual ~ColumnOp(); + EXPORT ~ColumnOp() override; EXPORT virtual int allocRowId(const TxnID& txnid, bool useStartingExtent, Column& column, uint64_t totalRow, RID* rowIdArray, HWM& hwm, bool& newExtent, uint64_t& rowsLeft, HWM& newHwm, @@ -145,7 +145,7 @@ class ColumnOp : public DbFileOp */ EXPORT int extendColumn(const Column& column, bool leaveFileOpen, HWM hwm, BRM::LBID_t startLbid, int allocSize, uint16_t dbRoot, uint32_t partition, uint16_t segment, - std::string& segFile, IDBDataFile*& pFile, bool& newFile, char* hdrs = NULL); + std::string& segFile, IDBDataFile*& pFile, bool& newFile, char* hdrs = nullptr); /** * @brief Add an extent to the OID specified in the column argument. @@ -165,7 +165,7 @@ class ColumnOp : public DbFileOp */ EXPORT int addExtent(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, BRM::LBID_t& startLbid, bool& newFile, int& allocSize, - char* hdrs = NULL); + char* hdrs = nullptr); /** * @brief Get columne data type @@ -216,13 +216,13 @@ class ColumnOp : public DbFileOp * @brief Write row(s) */ EXPORT virtual int writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, const void* valArray, - void* oldValArray = 0, bool bDelete = false); + void* oldValArray = nullptr, bool bDelete = false); /** * @brief Write row(s) for delete @Bug 1886,2870 */ EXPORT virtual int writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridList, - const void* valArray, void* oldValArray = 0, bool bDelete = false); + const void* valArray, void* oldValArray = nullptr, bool bDelete = false); /** * @brief MCOL-5021 Read-only version of the writeRows() function above. @@ -232,13 +232,13 @@ class ColumnOp : public DbFileOp database files. */ EXPORT virtual int writeRowsReadOnly(Column& curCol, uint64_t totalRow, const RIDList& ridList, - void* oldValArray = 0); + void* oldValArray = nullptr); /** * @brief Write row(s) for update @Bug 1886,2870 */ EXPORT virtual int writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& ridList, - const void* valArray, void* oldValArray = 0); + const void* valArray, void* oldValArray = nullptr); /** * @brief Test if the pFile is an abbreviated extent. diff --git a/writeengine/wrapper/we_colopcompress.h b/writeengine/wrapper/we_colopcompress.h index 15be5e614..5ebcbc909 100644 --- a/writeengine/wrapper/we_colopcompress.h +++ b/writeengine/wrapper/we_colopcompress.h @@ -21,7 +21,7 @@ #pragma once -#include +#include #include "we_colop.h" #include "we_chunkmanager.h" @@ -45,35 +45,35 @@ class ColumnOpCompress0 : public ColumnOp /** * @brief Default Destructor */ - EXPORT virtual ~ColumnOpCompress0(); + EXPORT ~ColumnOpCompress0() override; /** * @brief virtual method in ColumnOp */ IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const; + int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const override; /** * @brief virtual method in ColumnOp */ - bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const; + bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const override; /** * @brief virtual method in ColumnOp */ - int blocksInFile(IDBDataFile* pFile) const; + int blocksInFile(IDBDataFile* pFile) const override; protected: /** * @brief virtual method in ColumnOp */ - int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo); + int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) override; /** * @brief virtual method in ColumnOp */ - int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo); + int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) override; private: }; @@ -85,47 +85,47 @@ class ColumnOpCompress1 : public ColumnOp /** * @brief Constructor */ - EXPORT ColumnOpCompress1(uint32_t compressionType, Log* logger = 0); + EXPORT ColumnOpCompress1(uint32_t compressionType, Log* logger = nullptr); /** * @brief Default Destructor */ - EXPORT virtual ~ColumnOpCompress1(); + EXPORT ~ColumnOpCompress1() override; /** * @brief virtual method in FileOp */ - EXPORT int flushFile(int rc, std::map& columnOids); + EXPORT int flushFile(int rc, std::map& columnOids) override; /** * @brief virtual method in FileOp */ int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal, int width, - execplan::CalpontSystemCatalog::ColDataType colDataType); + execplan::CalpontSystemCatalog::ColDataType colDataType) override; /** * @brief virtual method in ColumnOp */ IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const; + int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const override; /** * @brief virtual method in ColumnOp */ - bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const; + bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const override; /** * @brief virtual method in ColumnOp */ - int blocksInFile(IDBDataFile* pFile) const; + int blocksInFile(IDBDataFile* pFile) const override; // void chunkManager(ChunkManager* cm); /** * @brief virtual method in FileOp */ - void setTransId(const TxnID& transId) + void setTransId(const TxnID& transId) override { ColumnOp::setTransId(transId); @@ -133,12 +133,12 @@ class ColumnOpCompress1 : public ColumnOp m_chunkManager->setTransId(transId); } - void setBulkFlag(bool isBulkLoad) + void setBulkFlag(bool isBulkLoad) override { m_chunkManager->setBulkFlag(isBulkLoad); }; - void setFixFlag(bool isFix) + void setFixFlag(bool isFix) override { m_chunkManager->setFixFlag(isFix); }; @@ -147,22 +147,22 @@ class ColumnOpCompress1 : public ColumnOp /** * @brief virtual method in FileOp */ - int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); + int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) override; /** * @brief virtual method in ColumnOp */ - void closeColumnFile(Column& column) const; + void closeColumnFile(Column& column) const override; /** * @brief virtual method in ColumnOp */ - int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo); + int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) override; /** * @brief virtual method in ColumnOp */ - int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo); + int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) override; /** * @brief Set the IsInsert flag in the ChunkManager. diff --git a/writeengine/wrapper/we_dctnrycompress.h b/writeengine/wrapper/we_dctnrycompress.h index 2831b5c95..367fd5482 100644 --- a/writeengine/wrapper/we_dctnrycompress.h +++ b/writeengine/wrapper/we_dctnrycompress.h @@ -21,7 +21,7 @@ #pragma once -#include +#include #include "../dictionary/we_dctnry.h" #include "we_chunkmanager.h" @@ -43,7 +43,7 @@ class DctnryCompress0 : public Dctnry /** * @brief Default Destructor */ - EXPORT virtual ~DctnryCompress0(); + EXPORT ~DctnryCompress0() override; }; /** Class DctnryCompress1 */ @@ -53,55 +53,56 @@ class DctnryCompress1 : public Dctnry /** * @brief Constructor */ - EXPORT DctnryCompress1(uint32_t compressionType, Log* logger = 0); + EXPORT DctnryCompress1(uint32_t compressionType, Log* logger = nullptr); /** * @brief Default Destructor */ - EXPORT virtual ~DctnryCompress1(); + EXPORT ~DctnryCompress1() override; /** * @brief virtual method in FileOp */ - EXPORT int flushFile(int rc, std::map& columnOids); + EXPORT int flushFile(int rc, std::map& columnOids) override; /** * @brief virtual method in DBFileOp */ EXPORT int readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid, - const bool isFbo = false); + const bool isFbo = false) override; /** * @brief virtual method in DBFileOp */ EXPORT int writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid, - const int numOfBlock = 1); + const int numOfBlock = 1) override; /** * @brief virtual method in DBFileOp */ EXPORT int writeDBFileNoVBCache(IDBDataFile* pFile, const unsigned char* writeBuf, const int fbo, - const int numOfBlock = 1); + const int numOfBlock = 1) override; /** * @brief virtual method in Dctnry */ - IDBDataFile* createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize, int64_t lbid); + IDBDataFile* createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize, + int64_t lbid) override; /** * @brief virtual method in Dctnry */ - IDBDataFile* openDctnryFile(bool useTmpSuffix); + IDBDataFile* openDctnryFile(bool useTmpSuffix) override; /** * @brief virtual method in Dctnry */ - void closeDctnryFile(bool doFlush, std::map& columnOids); + void closeDctnryFile(bool doFlush, std::map& columnOids) override; /** * @brief virtual method in Dctnry */ - int numOfBlocksInFile(); + int numOfBlocksInFile() override; /** * @brief For bulkload to use @@ -110,15 +111,15 @@ class DctnryCompress1 : public Dctnry { m_chunkManager->setMaxActiveChunkNum(maxActiveChunkNum); }; - void setBulkFlag(bool isBulkLoad) + void setBulkFlag(bool isBulkLoad) override { m_chunkManager->setBulkFlag(isBulkLoad); }; - void setFixFlag(bool isFix) + void setFixFlag(bool isFix) override { m_chunkManager->setFixFlag(isFix); }; - int checkFixLastDictChunk() + int checkFixLastDictChunk() override { return m_chunkManager->checkFixLastDictChunk(m_dctnryOID, m_dbRoot, m_partition, m_segment); }; @@ -127,7 +128,7 @@ class DctnryCompress1 : public Dctnry /** * @brief virtual method in FileOp */ - void setTransId(const TxnID& transId) + void setTransId(const TxnID& transId) override { Dctnry::setTransId(transId); @@ -148,7 +149,7 @@ class DctnryCompress1 : public Dctnry /** * @brief virtual method in FileOp */ - int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); + int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) override; /** * @brief convert lbid to fbo diff --git a/writeengine/wrapper/we_tablemetadata.h b/writeengine/wrapper/we_tablemetadata.h index 91fe7d2f8..fc92771b2 100644 --- a/writeengine/wrapper/we_tablemetadata.h +++ b/writeengine/wrapper/we_tablemetadata.h @@ -21,7 +21,7 @@ #pragma once -#include +#include #include "we_type.h" #include "brmtypes.h" #include @@ -82,7 +82,7 @@ class TableMetaData private: /** Constuctors */ explicit TableMetaData(); - explicit TableMetaData(const TableMetaData& rhs); + TableMetaData(const TableMetaData& rhs); ~TableMetaData(); static boost::mutex map_mutex; static TableMetaDataMap fTableMetaDataMap; @@ -93,4 +93,3 @@ class TableMetaData } // namespace WriteEngine #undef EXPORT - diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index ad472a302..07c67ca46 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -20,7 +20,7 @@ /** @file */ #pragma once -#include +#include #include // the header file for fd @@ -66,10 +66,8 @@ struct TxnLBIDRec std::vector m_LBIDs; std::vector m_ColDataTypes; - TxnLBIDRec(){}; - ~TxnLBIDRec() - { - } + TxnLBIDRec() = default; + ~TxnLBIDRec() = default; void AddLBID(BRM::LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType& colDataType) { if (m_LBIDSet.insert(lbid).second) @@ -92,7 +90,7 @@ struct ColSplitMaxMinInfo ColSplitMaxMinInfo(execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth) : fSplitMaxMinInfo{ExtCPInfo(colDataType, colWidth), ExtCPInfo(colDataType, colWidth)} { - fSplitMaxMinInfoPtrs[0] = fSplitMaxMinInfoPtrs[1] = NULL; // disable by default. + fSplitMaxMinInfoPtrs[0] = fSplitMaxMinInfoPtrs[1] = nullptr; // disable by default. } }; @@ -246,8 +244,7 @@ class WriteEngineWrapper : public WEObj */ EXPORT int deleteRow(const TxnID& txnid, const std::vector& colExtentsColType, std::vector& colExtentsStruct, std::vector& colOldValueList, - std::vector& ridLists, const int32_t tableOid, - bool hasAUXCol = false); + std::vector& ridLists, const int32_t tableOid, bool hasAUXCol = false); /** * @brief Delete a list of rows from a table @@ -702,7 +699,7 @@ class WriteEngineWrapper : public WEObj int writeColumnRec(const TxnID& txnid, const CSCTypesList& cscColTypes, const ColStructList& colStructList, ColValueList& colValueList, RID* rowIdArray, const ColStructList& newColStructList, ColValueList& newColValueList, const int32_t tableOid, bool useTmpSuffix, - bool versioning = true, ColSplitMaxMinInfoList* maxMins = NULL); + bool versioning = true, ColSplitMaxMinInfoList* maxMins = nullptr); int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, std::vector& colValueList, RID* rowIdArray, @@ -714,14 +711,14 @@ class WriteEngineWrapper : public WEObj const ColStructList& colStructList, const ColValueList& colValueList, std::vector& colOldValueList, const RIDList& ridList, const int32_t tableOid, bool convertStructFlag = true, - ColTupleList::size_type nRows = 0, std::vector* cpInfos = NULL, + ColTupleList::size_type nRows = 0, std::vector* cpInfos = nullptr, bool hasAUXCol = false); // For update column from column to use int writeColumnRecords(const TxnID& txnid, const CSCTypesList& cscColTypeList, std::vector& colStructList, ColValueList& colValueList, const RIDList& ridLists, const int32_t tableOid, bool versioning = true, - std::vector* cpInfos = NULL); + std::vector* cpInfos = nullptr); /** * @brief util method to convert rowid to a column file @@ -738,7 +735,7 @@ class WriteEngineWrapper : public WEObj // txn so that we don't waste time marking the same extent as invalid. This // list should be trimmed if it gets too big. int AddLBIDtoList(const TxnID txnid, const ColStruct& colStruct, const int fbo, - ExtCPInfo* cpInfo = NULL // provide CPInfo pointer if you want max/min updated. + ExtCPInfo* cpInfo = nullptr // provide CPInfo pointer if you want max/min updated. ); // Get CPInfo for given starting LBID and column description structure. diff --git a/writeengine/xml/we_xmlgendata.h b/writeengine/xml/we_xmlgendata.h index 5bf4873cc..633f1ed8a 100644 --- a/writeengine/xml/we_xmlgendata.h +++ b/writeengine/xml/we_xmlgendata.h @@ -48,13 +48,13 @@ class XMLGenData // Valid parms that can be stored and retrieved from XMLGenData EXPORT const static std::string DELIMITER; EXPORT const static std::string DESCRIPTION; - EXPORT const static std::string ENCLOSED_BY_CHAR; - EXPORT const static std::string ESCAPE_CHAR; - EXPORT const static std::string JOBID; + EXPORT const static std::string ENCLOSED_BY_CHAR; + EXPORT const static std::string ESCAPE_CHAR; + EXPORT const static std::string JOBID; EXPORT const static std::string MAXERROR; EXPORT const static std::string NAME; EXPORT const static std::string PATH; - EXPORT const static std::string RPT_DEBUG; + EXPORT const static std::string RPT_DEBUG; EXPORT const static std::string USER; EXPORT const static std::string NO_OF_READ_BUFFER; EXPORT const static std::string READ_BUFFER_CAPACITY; diff --git a/writeengine/xml/we_xmljob.h b/writeengine/xml/we_xmljob.h index d5943f8b0..088a2c65b 100644 --- a/writeengine/xml/we_xmljob.h +++ b/writeengine/xml/we_xmljob.h @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include "we_xmlop.h" @@ -54,7 +54,7 @@ class XMLJob : public XMLOp /** * @brief Default Destructor */ - EXPORT ~XMLJob(); + EXPORT ~XMLJob() override; /** * @brief Utility to generate a full path name for a Job XML file name @@ -115,7 +115,7 @@ class XMLJob : public XMLOp * @brief Process node * @param pParentNode Node to be parsed from XML */ - EXPORT bool processNode(xmlNode* pParentNode); + EXPORT bool processNode(xmlNode* pParentNode) override; /** * @brief Set timezone From b5bc649923bb00bdb8b72bf1708bd5293d31be16 Mon Sep 17 00:00:00 2001 From: Aleksei Antipovskii Date: Tue, 24 Sep 2024 14:47:44 +0200 Subject: [PATCH 43/65] add missing `override` --- dbcon/mysql/ha_mcs.h | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index 4050356e4..3af6717ea 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -45,7 +45,7 @@ class ha_mcs : public handler THR_LOCK_DATA lock; ///< MySQL lock COLUMNSTORE_SHARE* share; ///< Shared lock info ulonglong int_table_flags; - // We are using a vector here to mimick the stack functionality + // We are using a vector here to mimic the stack functionality // using push_back() and pop_back() // as apparently there is a linker error on the std::stack::pop() // call on Ubuntu18. @@ -58,7 +58,7 @@ class ha_mcs : public handler public: ha_mcs(handlerton* hton, TABLE_SHARE* table_arg); - virtual ~ha_mcs() = default; + ~ha_mcs() override = default; /** @brief The name that will be used for display purposes. @@ -174,7 +174,7 @@ class ha_mcs : public handler /** @brief We implement this in ha_example.cc; it's a required method. */ - int close(void) override; // required + int close() override; // required /** @brief We implement this in ha_example.cc. It's not an obligatory method; @@ -257,7 +257,7 @@ class ha_mcs : public handler int info(uint32_t) override; ///< required int extra(enum ha_extra_function operation) override; int external_lock(THD* thd, int lock_type) override; ///< required - int delete_all_rows(void) override; + int delete_all_rows() override; ha_rows records_in_range(uint32_t inx, const key_range* min_key, const key_range* max_key, page_range* res) override; int delete_table(const char* from) override; @@ -329,26 +329,26 @@ class ha_mcs_cache : public ha_mcs cache handler */ - int create(const char* name, TABLE* table_arg, HA_CREATE_INFO* ha_create_info); - int open(const char* name, int mode, uint open_flags); - int delete_table(const char* name); - int rename_table(const char* from, const char* to); - int delete_all_rows(void); - int close(void); + int create(const char* name, TABLE* table_arg, HA_CREATE_INFO* ha_create_info) override; + int open(const char* name, int mode, uint open_flags) override; + int delete_table(const char* name) override; + int rename_table(const char* from, const char* to) override; + int delete_all_rows() override; + int close() override; - uint lock_count(void) const; - THR_LOCK_DATA** store_lock(THD* thd, THR_LOCK_DATA** to, enum thr_lock_type lock_type); - int external_lock(THD* thd, int lock_type); - int repair(THD* thd, HA_CHECK_OPT* check_opt); - bool is_crashed() const; + uint lock_count() const override; + THR_LOCK_DATA** store_lock(THD* thd, THR_LOCK_DATA** to, enum thr_lock_type lock_type) override; + int external_lock(THD* thd, int lock_type) override; + int repair(THD* thd, HA_CHECK_OPT* check_opt) override; + bool is_crashed() const override; /* Write row uses cache_handler, for normal inserts, otherwise derived handler */ - int write_row(const uchar* buf); - void start_bulk_insert(ha_rows rows, uint flags); - int end_bulk_insert(); + int write_row(const uchar* buf) override; + void start_bulk_insert(ha_rows rows, uint flags) override; + int end_bulk_insert() override; /* Cache functions */ void free_locks(); From 0885ccc6e800e2a3fb9372b1a3cd4c2c83d5c373 Mon Sep 17 00:00:00 2001 From: Aleksei Antipovskii Date: Wed, 25 Sep 2024 00:42:20 +0200 Subject: [PATCH 44/65] more overrides --- dbcon/mysql/ha_mcs_datatype.h | 4 ++-- dbcon/mysql/ha_mcs_impl_if.h | 17 +++++++++-------- dbcon/mysql/ha_mcs_pushdown.h | 6 +++--- dbcon/mysql/ha_mcs_sysvars.h | 12 ++++++------ dbcon/mysql/ha_subquery.h | 4 ++-- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h index 3e588d86c..71e03c72d 100644 --- a/dbcon/mysql/ha_mcs_datatype.h +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -113,7 +113,7 @@ class StoreFieldMariaDB : public StoreField return m_field->store(static_cast(val), 1); } - int store_float(float dl) + int store_float(float dl) override { if (dl == std::numeric_limits::infinity()) { @@ -128,7 +128,7 @@ class StoreFieldMariaDB : public StoreField return m_field->store(dl); } - int store_double(double dl) + int store_double(double dl) override { if (dl == std::numeric_limits::infinity()) { diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index d1d29aaa3..9ef541cc7 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -203,7 +203,7 @@ struct gp_walk_info , condPush(false) , dropCond(false) , internalDecimalScale(4) - , thd(0) + , thd(nullptr) , subSelectType(uint64_t(-1)) , subQuery(nullptr) , clauseType(INIT) @@ -250,7 +250,8 @@ struct cal_table_info FROM_FILE }; - cal_table_info() : tpl_ctx(0), c(0), msTablePtr(0), conn_hndl(0), condInfo(0), moreRows(false) + cal_table_info() + : tpl_ctx(nullptr), c(0), msTablePtr(nullptr), conn_hndl(nullptr), condInfo(nullptr), moreRows(false) { } sm::sp_cpsm_tplh_t tpl_ctx; @@ -268,12 +269,12 @@ struct cal_table_info struct cal_group_info { cal_group_info() - : groupByFields(0) - , groupByTables(0) - , groupByWhere(0) - , groupByGroup(0) - , groupByOrder(0) - , groupByHaving(0) + : groupByFields(nullptr) + , groupByTables(nullptr) + , groupByWhere(nullptr) + , groupByGroup(nullptr) + , groupByOrder(nullptr) + , groupByHaving(nullptr) , groupByDistinct(false) { } diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index d9a73a374..79b2305f6 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -82,7 +82,7 @@ class ha_mcs_group_by_handler : public group_by_handler public: ha_mcs_group_by_handler(THD* thd_arg, Query* query); - ~ha_mcs_group_by_handler(); + ~ha_mcs_group_by_handler() override; int init_scan() override; int next_row() override; int end_scan() override; @@ -117,7 +117,7 @@ class ha_columnstore_derived_handler : public derived_handler public: ha_columnstore_derived_handler(THD* thd_arg, TABLE_LIST* tbl); - ~ha_columnstore_derived_handler(); + ~ha_columnstore_derived_handler() override; int init_scan() override; int next_row() override; int end_scan() override; @@ -155,7 +155,7 @@ class ha_columnstore_select_handler : public select_handler ha_columnstore_select_handler(THD* thd_arg, SELECT_LEX* sel_lex); ha_columnstore_select_handler(THD* thd_arg, SELECT_LEX_UNIT* sel_unit); ha_columnstore_select_handler(THD* thd_arg, SELECT_LEX* sel_lex, SELECT_LEX_UNIT* sel_unit); - ~ha_columnstore_select_handler(); + ~ha_columnstore_select_handler() override; int init_scan() override; int next_row() override; int end_scan() override; diff --git a/dbcon/mysql/ha_mcs_sysvars.h b/dbcon/mysql/ha_mcs_sysvars.h index e05272742..f31dfd69c 100644 --- a/dbcon/mysql/ha_mcs_sysvars.h +++ b/dbcon/mysql/ha_mcs_sysvars.h @@ -60,14 +60,14 @@ void set_original_query(THD* thd, char* query); mcs_compression_type_t get_compression_type(THD* thd); void set_compression_type(THD* thd, ulong value); -void* get_fe_conn_info_ptr(THD* thd = NULL); -void set_fe_conn_info_ptr(void* ptr, THD* thd = NULL); +void* get_fe_conn_info_ptr(THD* thd = nullptr); +void set_fe_conn_info_ptr(void* ptr, THD* thd = nullptr); -ulonglong get_original_optimizer_flags(THD* thd = NULL); -void set_original_optimizer_flags(ulonglong ptr, THD* thd = NULL); +ulonglong get_original_optimizer_flags(THD* thd = nullptr); +void set_original_optimizer_flags(ulonglong ptr, THD* thd = nullptr); -ulonglong get_original_option_bits(THD* thd = NULL); -void set_original_option_bits(ulonglong value, THD* thd = NULL); +ulonglong get_original_option_bits(THD* thd = nullptr); +void set_original_option_bits(ulonglong value, THD* thd = nullptr); mcs_select_handler_mode_t get_select_handler_mode(THD* thd); void set_select_handler_mode(THD* thd, ulong value); diff --git a/dbcon/mysql/ha_subquery.h b/dbcon/mysql/ha_subquery.h index 31e6c29d9..688c64771 100644 --- a/dbcon/mysql/ha_subquery.h +++ b/dbcon/mysql/ha_subquery.h @@ -96,7 +96,7 @@ struct SubQueryChainHolder class WhereSubQuery : public SubQuery { public: - WhereSubQuery(gp_walk_info& gwip) : SubQuery(gwip), fSub(NULL), fFunc(NULL) + WhereSubQuery(gp_walk_info& gwip) : SubQuery(gwip), fSub(nullptr), fFunc(nullptr) { } WhereSubQuery(gp_walk_info& gwip, const execplan::SRCP& column, Item_subselect* sub, Item_func* func) @@ -175,7 +175,7 @@ class InSub : public WhereSubQuery InSub(const InSub& rhs); ~InSub() override; execplan::ParseTree* transform() override; - void handleFunc(gp_walk_info* gwip, Item_func* func); + void handleFunc(gp_walk_info* gwip, Item_func* func) override; void handleNot() override; }; From cd1939ee07d5b5f9884509d27cb08f1e7dbfcf28 Mon Sep 17 00:00:00 2001 From: Allen Herrera Date: Fri, 21 Feb 2025 08:07:19 -0500 Subject: [PATCH 45/65] update backuo to v3.11, backup list and restore list + fixes --- cmapi/scripts/mcs_backup_manager.sh | 290 ++++++++++++++++++++-------- 1 file changed, 214 insertions(+), 76 deletions(-) diff --git a/cmapi/scripts/mcs_backup_manager.sh b/cmapi/scripts/mcs_backup_manager.sh index feeb17a0b..59355aff5 100644 --- a/cmapi/scripts/mcs_backup_manager.sh +++ b/cmapi/scripts/mcs_backup_manager.sh @@ -13,7 +13,7 @@ # ######################################################################## # Documentation: bash mcs_backup_manager.sh help -# Version: 3.9 +# Version: 3.11 # # Backup Example # LocalStorage: sudo ./mcs_backup_manager.sh backup @@ -26,26 +26,26 @@ # S3: sudo ./mcs_backup_manager.sh restore -bb s3://my-cs-backups -l # ######################################################################## -mcs_bk_manager_version="3.9" +mcs_bk_manager_version="3.11" start=$(date +%s) action=$1 print_action_help_text() { echo " - MariaDB Columnstore Backup Manager - - Actions: +MariaDB Columnstore Backup Manager - backup Full & Incremental columnstore backup with additional flags to augment the backup taken - restore Restore a backup taken with this script - dbrm_backup Quick hot backup of internal columnstore metadata only - only use under support recommendation - dbrm_restore Restore internal columnstore metadata from dbrm_backup - only use under support recommendation +Actions: - Documentation: - bash $0 help + backup Full & Incremental columnstore backup with additional flags to augment the backup taken + restore Restore a backup taken with this script + dbrm_backup Quick hot backup of internal columnstore metadata only - only use under support recommendation + dbrm_restore Restore internal columnstore metadata from dbrm_backup - only use under support recommendation - Example: - bash $0 backup help +Documentation: + bash $0 help + +Example: + bash $0 backup help " } @@ -170,6 +170,7 @@ load_default_backup_variables() { rsync_flags=" -av"; poll_interval=5 poll_max_wait=60; + list_backups=false # Compression Variables compress_format="" @@ -275,7 +276,7 @@ parse_backup_variables() { skip_locks=true shift # past argument ;; - -smdb | --skip-mariadb-backup) + -smdb | --skip-mariadb-backup) skip_mdb=true shift # past argument ;; @@ -321,6 +322,10 @@ parse_backup_variables() { shift # past argument shift # past value ;; + "list") + list_backups=true + shift # past argument + ;; -h|--help|-help|help) print_backup_help_text; exit 1; @@ -995,6 +1000,10 @@ poll_check_no_active_sql_writes() { no_writes=false while [ "$attempts" -lt "$max_poll_attempts" ]; do active_writes=$( mariadb -s -N -e "$query" ) + if [ "$?" -ne 0 ]; then + handle_early_exit_on_backup "\n$active_writes\n[X] Failed to poll for active writes\n" true + fi + if [ "$active_writes" -le 0 ]; then printf "Done\n" no_writes=true @@ -1479,6 +1488,18 @@ run_backup() { esac fi + # Save replication details + slave_status=$(mariadb -e "show slave status\G" 2>/dev/null) + if [[ $? -eq 0 ]]; then + if [[ -n "$slave_status" ]]; then + mariadb -e "show slave status\G" | grep -E "Master_Host|Master_User|Master_Port" > $backup_location$today/replication_details.txt + else + echo "No replication details to save" + fi + else + handle_early_exit_on_backup "[!] - Failed to execute the MariaDB command: mariadb -e 'show slave status\G' \n" + fi + if $incremental ; then # Log each incremental run now=$(date "+%m-%d-%Y %H:%M:%S"); echo "$pm updated on $now" >> $backup_location$today/incrementallyUpdated.txt @@ -1488,7 +1509,7 @@ run_backup() { extra_flags="" if [[ -n "$compress_format" ]]; then extra_flags+=" -c $compress_format"; fi; if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; - echo "./$0 restore -l $today -bl $backup_location -bd $backup_destination -s $storage --dbroots $DBROOT_COUNT -m $mode $extra_flags --quiet" > $backup_location$today/restore.job + echo "$0 restore -l $today -bl $backup_location -bd $backup_destination -s $storage --dbroots $DBROOT_COUNT -m $mode $extra_flags --quiet" > $backup_location$today/restore.job fi final_message+=" @ $backup_location$today" @@ -1734,7 +1755,7 @@ run_backup() { if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; if $skip_bucket_data; then extra_flags+=" --skip-bucket-data"; fi; if [ -n "$s3_url" ]; then extra_flags+=" -url $s3_url"; fi; - echo "./$0 restore -l $today -s $storage -bb $backup_bucket -dbs $DBROOT_COUNT -m $mode -nb $protocol://$bucket $extra_flags --quiet --continue" > restoreS3.job + echo "$0 restore -l $today -s $storage -bb $backup_bucket -dbs $DBROOT_COUNT -m $mode -nb $protocol://$bucket $extra_flags --quiet --continue" > restoreS3.job s3cp restoreS3.job $backup_bucket/$today/restoreS3.job rm -rf restoreS3.job fi @@ -1842,6 +1863,7 @@ load_default_restore_variables() { xtra_s3_args="" xtra_cmd_args="" rsync_flags=" -av"; + list_backups=false } parse_restore_variables() { @@ -1966,6 +1988,10 @@ parse_restore_variables() { no_verify_ssl=true shift # past argument ;; + "list") + list_backups=true + shift # past argument + ;; -h|--help|-help|help) print_restore_help_text; exit 1; @@ -2009,7 +2035,6 @@ print_restore_help_text() -sb | --skip-bucket-data Skip restoring columnstore data in the bucket - ideal if looking to only restore mariadb server -q | --quiet Silence verbose copy command outputs -c | --compress Hint that the backup is compressed in X format - Options: [ pigz ] - -P | --parallel Number of parallel decompression and mbstream threads to run -ha | --highavilability Hint for if shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) HA S3 ( /var/lib/columnstore/storagemanager/ ) @@ -2044,8 +2069,7 @@ print_restore_variables() if [[ -n "$compress_format" ]]; then echo "Compression: true" echo "Compression Format: $compress_format"; - echo "Decompression Threads:" "$PARALLEL_THREADS"; - else + else echo "Compression: false" fi if [ $storage == "LocalStorage" ]; then @@ -2135,7 +2159,7 @@ validation_prechecks_for_restore() { esac if eval $cmapi_installed_command ; then - if ! sudo mcs cmapi is-ready ; then + if ! sudo mcs cmapi is-ready &> /dev/null ; then printf " - Columnstore Management API Status .. Offline\n"; else handle_early_exit_on_restore "\n[X] Cmapi is ONLINE - please turn off \n\n"; @@ -2724,7 +2748,7 @@ load_default_dbrm_variables() { # Default variables backup_base_name="dbrm_backup" backup_interval_minutes=90 - retention_days=7 + retention_days=0 backup_location=/tmp/dbrm_backups STORAGEMANGER_CNF="/etc/columnstore/storagemanager.cnf" storage=$(grep -m 1 "^service = " $STORAGEMANGER_CNF | awk '{print $3}') @@ -2747,6 +2771,7 @@ load_default_dbrm_restore_variables() { backup_folder_to_restore="" skip_dbrm_backup=false skip_storage_manager=false + list_dbrm_backups=false dbrm_dir="/var/lib/columnstore/data1/systemFiles/dbrm" if [ "$storage" == "S3" ]; then @@ -2761,18 +2786,19 @@ print_dbrm_backup_help_text() { -m | --mode ['loop','once']; Determines if this script runs in a forever loop sleeping -i minutes or just once -i | --interval Number of minutes to sleep when --mode loop -r | --retention-days Retain dbrm backups created within the last X days, the rest are deleted - -p | --path path of where to save the dbrm backups on disk - -nb | --name-backup custom name to prefex dbrm backups with + -bl | --backup-location Path of where to save the dbrm backups on disk + -nb | --name-backup Define the prefix of the backup - default: dbrm_backup+date +%Y%m%d_%H%M%S -ssm | --skip-storage-manager skip backing up storagemanager directory - Default: ./$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups + Default: ./$0 dbrm_backup -m once --retention-days 0 --backup-location /tmp/dbrm_backups Examples: - ./$0 dbrm_backup --mode loop --interval 90 --retention-days 7 --path /mnt/dbrm_backups - ./$0 dbrm_backup --mode once --retention-days 7 --path /mnt/dbrm_backups -nb my-one-off-backup + ./$0 dbrm_backup --backup-location /mnt/columnstore/dbrm_backups + ./$0 dbrm_backup --retention-days 7 --backup-location /mnt/dbrm_backups --mode once -nb my-one-off-backup-before-upgrade + ./$0 dbrm_backup --retention-days 7 --backup-location /mnt/dbrm_backups --mode loop --interval 90 Cron Example: - */60 */3 * * * root bash /root/$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups >> /tmp/dbrm_backups/cs_backup.log 2>&1 + */60 */3 * * * root bash /root/$0 dbrm_backup -m once --retention-days 7 --backup-location /tmp/dbrm_backups >> /tmp/dbrm_backups/cs_backup.log 2>&1 "; } @@ -2780,17 +2806,17 @@ print_dbrm_restore_help_text() { echo " Columnstore DBRM Restore - -p | --path path of where to save the dbrm backups on disk - -d | --directory date or directory chose to restore from - -ns | --no-start do not attempt columnstore startup post dbrm_restore - -sdbk| --skip-dbrm-backup skip backing up dbrms brefore restoring - -ssm | --skip-storage-manager skip restoring storagemanager directory + -bl | --backup-location Path of where the dbrm backups exist on disk + -l | --load Name of the directory to restore from -bl + -ns | --no-start Do not attempt columnstore startup post dbrm_restore + -sdbk| --skip-dbrm-backup Skip backing up dbrms brefore restoring + -ssm | --skip-storage-manager Skip restoring storagemanager directory - Default: ./$0 dbrm_restore --path /tmp/dbrm_backups + Default: ./$0 dbrm_restore --backup-location /tmp/dbrm_backups Examples: - ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 - ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 --no-start + ./$0 dbrm_restore --backup-location /tmp/dbrm_backups --load dbrm_backup_20240318_172842 + ./$0 dbrm_restore --backup-location /tmp/dbrm_backups --load dbrm_backup_20240318_172842 --no-start "; } @@ -2813,7 +2839,7 @@ parse_dbrms_variables() { shift # past argument shift # past value ;; - -p|--path) + -bl|--backup-location) backup_location="$2" shift # past argument shift # past value @@ -2836,14 +2862,14 @@ parse_dbrms_variables() { quiet=true shift # past argument ;; - -h|--help|-help|help) - print_dbrm_backup_help_text; - exit 1; - ;; "list") list_dbrm_backups=true shift # past argument ;; + -h|--help|-help|help) + print_dbrm_backup_help_text; + exit 1; + ;; *) # unknown option printf "\nunknown flag: $1\n" print_dbrm_backup_help_text @@ -2861,12 +2887,12 @@ parse_dbrm_restore_variables() { dbrm_restore) shift # past argument ;; - -p|--path) + -bl|--backup-location) backup_location="$2" shift # past argument shift # past value ;; - -d|--directory) + -l|--load) backup_folder_to_restore="$2" shift # past argument shift # past value @@ -2883,6 +2909,10 @@ parse_dbrm_restore_variables() { skip_storage_manager=true shift # past argument ;; + "list") + list_dbrm_backups=true + shift # past argument + ;; -h|--help|-help|help) print_dbrm_restore_help_text; exit 1; @@ -2956,14 +2986,14 @@ validation_prechecks_before_listing_restore_options() { # confirm backup directory exists if [ ! -d $backup_location ]; then - printf "[!!] Backups Directory does NOT exist --path $backup_location \n" + printf "[!!] Backups Directory does NOT exist --backup-location $backup_location \n" printf "ls -la $backup_location\n\n" exit 1; fi # Check if backup directory is empty if [ -z "$(find "$backup_location" -mindepth 1 | head )" ]; then - printf "[!!] Backups Directory is empty --path $backup_location \n" + printf "[!!] Backups Directory is empty --backup-location $backup_location \n" printf "ls -la $backup_location\n\n" exit 1 fi @@ -2990,7 +3020,7 @@ validation_prechecks_for_dbrm_restore() { fi if [ ! -d "${backup_location}/${backup_folder_to_restore_dbrms}" ]; then - printf "[!] \$backup_folder_to_restore: Path of backup to restore does Not exist\n" + printf "[!] --load: Path of backup to restore does NOT exist\n" printf "Path: ${backup_location}/${backup_folder_to_restore_dbrms}\n\n" exit 2; else @@ -3098,14 +3128,7 @@ process_dbrm_backup() { load_default_dbrm_variables parse_dbrms_variables "$@"; - if $list_dbrm_backups; then - validation_prechecks_before_listing_restore_options - printf "\nExisting DBRM Backups\n"; - list_restore_options_from_backups "$@" - echo "--------------------------------------------------------------------------" - printf "Restore with ./$0 dbrm_restore --path $backup_location --directory \n\n" - exit 0; - fi; + handle_list_dbrm_backups if ! $quiet ; then @@ -3249,13 +3272,13 @@ shutdown_columnstore_mariadb_cmapi() { # Input # $1 - directory to search # Output -# subdir_dbrms -# latest_em_file -# em_file_size -# em_file_created -# em_file_full_path -# storagemanager_dir_exists -get_latest_em_from_directory() { +# subdir_dbrms +# latest_em_file +# em_file_size +# em_file_created +# em_file_full_path +# storagemanager_dir_exists +get_latest_em_from_dbrm_directory() { subdir_dbrms="" latest_em_file="" @@ -3307,13 +3330,82 @@ get_latest_em_from_directory() { list_restore_options_from_backups() { + if [ $storage == "S3" ]; then + echo " Not implemented" ; exit 1; + else + + if [ -n "$retention_days" ] && [ $retention_days -ne 0 ]; then + echo "--------------------------------------------------------------------------" + echo "Retention Policy: $retention_days days" + fi; + echo "--------------------------------------------------------------------------" + printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s %-10s\n" "Options" "Last-Updated" "Extent Map" "EM-Size" "Journal-Size" "VBBM-Size" "VSS-Size" "Days Old" + + # Iterate over subdirectories + for subdir in "${backup_location}"/*; do + + if [ -f "${subdir}/cs-localfiles.tar.pigz" ]; then + em_file_created=$( date -r "$subdir" +"%b %d %H:%M" ) + em_file_size="N/A" + em_file_name="N/A" + journal_file="N/A" + vbbm_file="N/A" + vss_file="N/A" + + file_age=$(($(date +%s) - $(date -r "$subdir" +%s))) + file_age_days=$((file_age / 86400)) + will_delete="$(find "$subdir" -mindepth 1 -maxdepth 1 -type d -name "*" -mtime +$retention_days -exec echo {} \;)" + if [ -n "$will_delete" ]; then + file_age_days="$file_age_days (Will Delete)" + fi + + printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s %-10s\n" "$(basename "$subdir")" "$em_file_created" "$em_file_name" "$em_file_size" "$journal_file" "$vbbm_file" "$vss_file" "$file_age_days" + continue + fi + + get_latest_em_from_dbrm_directory "$subdir/data1/systemFiles/dbrm/" + + + if [ -f "${subdir_dbrms}/BRM_saves_journal" ]; then + em_file_name=$(basename "$em_file_full_path") + version_prefix=${em_file_name::-3} + journal_file=$(ls -la "${subdir_dbrms}/BRM_saves_journal" 2>/dev/null | awk 'NR==1 {print $5}' ) + vbbm_file=$(ls -la "${subdir_dbrms}/${version_prefix}_vbbm" 2>/dev/null | awk 'NR==1 {print $5}' ) + vss_file=$(ls -la "${subdir_dbrms}/${version_prefix}_vss" 2>/dev/null | awk 'NR==1 {print $5}' ) + + file_age=$(($(date +%s) - $(date -r "$subdir/data1/systemFiles" +%s))) + file_age_days=$((file_age / 86400)) + + # Check if the backup will be deleted given the retention policy defined + if [ -n "$retention_days" ] && [ $retention_days -ne 0 ]; then + will_delete="$(find "$subdir/data1/systemFiles" -mindepth 1 -maxdepth 1 -type d -name "*" -mtime +$retention_days -exec echo {} \;)" + + if [ -n "$will_delete" ]; then + file_age_days="$file_age_days (Will Delete)" + fi + fi; + + printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s %-10s\n" "$(basename "$subdir")" "$em_file_created" "$em_file_name" "$em_file_size" "$journal_file" "$vbbm_file" "$vss_file" "$file_age_days" + fi + done + fi + + +} + +list_dbrm_restore_options_from_backups() { + + if [ -n "$retention_days" ] && [ $retention_days -ne 0 ]; then + echo "--------------------------------------------------------------------------" + echo "Retention Policy: $retention_days days" + fi; echo "--------------------------------------------------------------------------" printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s\n" "Options" "Last-Updated" "Extent Map" "EM-Size" "Journal-Size" "VBBM-Size" "VSS-Size" # Iterate over subdirectories for subdir in "${backup_location}"/*; do - get_latest_em_from_directory "$subdir" + get_latest_em_from_dbrm_directory "$subdir" if [ -f "${subdir_dbrms}/BRM_saves_journal" ]; then em_file_name=$(basename "$em_file_full_path") @@ -3324,6 +3416,7 @@ list_restore_options_from_backups() { if [ $storagemanager_dir_exists == false ]; then vss_file+=" (No Storagemanager Dir)" fi; + printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s\n" "$(basename "$subdir")" "$em_file_created" "$em_file_name" "$em_file_size" "$journal_file" "$vbbm_file" "$vss_file" fi done @@ -3331,6 +3424,33 @@ list_restore_options_from_backups() { } +# $1 message +dynamic_list_dbrm_backups() { + + validation_prechecks_before_listing_restore_options + printf "$1"; + list_dbrm_restore_options_from_backups "$@" + echo "--------------------------------------------------------------------------" + printf "Restore with ./$0 dbrm_restore --backup-location $backup_location --load \n\n" +} + +handle_list_dbrm_backups () { + + if $list_dbrm_backups; then + dynamic_list_dbrm_backups "\nExisting DBRM Backups\n" + exit 0 + fi; + +} + +handle_empty_dbrm_restore_folder() { + + if [ -z "$backup_folder_to_restore" ]; then + dynamic_list_dbrm_backups "[!] Pick Option [Required]\n" + exit 1 + fi; +} + process_dbrm_restore() { load_default_dbrm_restore_variables @@ -3346,26 +3466,15 @@ process_dbrm_restore() { printf "Backup to Restore: $backup_folder_to_restore \n\n" validation_prechecks_before_listing_restore_options - - # Display restore options - if [ -z "$backup_folder_to_restore" ]; then - printf "[!] Pick Option\n" - list_restore_options_from_backups "$@" - printf "\nExample: \n" - printf " --directory dbrm_backup_20240103_183536 \n\n" - printf "Define which backup to restore via flag --directory \n" - echo "Rerun: $0 $@ --directory xxxxxxx" - echo "" - exit 1; - fi; - + handle_list_dbrm_backups + handle_empty_dbrm_restore_folder validation_prechecks_for_dbrm_restore shutdown_columnstore_mariadb_cmapi # Take an automated backup if [[ $skip_dbrm_backup == false ]]; then printf " - Saving a DBRM backup before restoring ... \n" - if ! process_dbrm_backup -p $backup_location -r 9999 -nb dbrms_before_restore_backup --quiet ; then + if ! process_dbrm_backup -bl $backup_location -r 0 -nb dbrms_before_restore_backup --quiet ; then echo "[!!] Failed to take a DBRM backup before restoring" echo "exiting ..." exit 1; @@ -3373,7 +3482,7 @@ process_dbrm_restore() { fi; # Detect newest date _em from the set, if smaller than the current one throw a warning - get_latest_em_from_directory "${backup_location}/${backup_folder_to_restore}" + get_latest_em_from_dbrm_directory "${backup_location}/${backup_folder_to_restore}" if [ ! -f $em_file_full_path ]; then echo "[!] Failed to parse _em file: $em_file_full_path doesnt exist" exit 1; @@ -3661,7 +3770,33 @@ manually_run_loadbrm_and_savebrm() { clearShm printf "Done\n" sleep 2 +} +# $1 - message +dynamic_list_backups() { + + validation_prechecks_before_listing_restore_options + printf "$1"; + list_restore_options_from_backups "$@" + echo "--------------------------------------------------------------------------" + printf "Restore with ./$0 restore --backup-location $backup_location --load \n\n" +} + + +handle_list_backups () { + + if $list_backups ; then + dynamic_list_backups "\nExisting Backups\n" + exit 0 + fi; +} + +handle_empty_restore_folder() { + + if [ -z "$load_date" ]; then + dynamic_list_backups "\n[!] Pick Option [Required]\n" + exit 1 + fi; } @@ -3669,6 +3804,7 @@ process_backup() { load_default_backup_variables; parse_backup_variables "$@"; + handle_list_backups "$@" print_backup_variables; check_for_dependancies "backup"; validation_prechecks_for_backup; @@ -3682,6 +3818,8 @@ process_restore() { load_default_restore_variables; parse_restore_variables "$@"; + handle_list_backups "$@" + handle_empty_restore_folder "$@" print_restore_variables; check_for_dependancies "restore"; validation_prechecks_for_restore; From 1a5f48d5ce533c5aa9c591d2bb1af461f2d0cf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Van=C4=9Bk?= Date: Tue, 25 Feb 2025 12:48:37 +0100 Subject: [PATCH 46/65] Allow packagers to use system Thrift library The commit introduces a new CMake configuration option, WITH_THRIFT, which accepts "auto", "system", or "bundled". In "auto" mode (the default), the build system attempts to use the system-installed Thrift library and falls back to the bundled version if the system library is not available. Setting WITH_THRIFT to "system" enforces the use of the system Thrift, causing the configuration to fail if it isn't found, while "bundled" forces the use of the bundled version. The change mainly useful for downstream maintainers as it gives them flexibility over dependency management. Downstream-issue: https://bugs.gentoo.org/949680 --- cmake/thrift.cmake | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmake/thrift.cmake b/cmake/thrift.cmake index 519606e47..1130b72e1 100644 --- a/cmake/thrift.cmake +++ b/cmake/thrift.cmake @@ -1,3 +1,19 @@ +set(WITH_THRIFT "auto" CACHE STRING + "Which Thrift to use (possible values are 'bundled', 'system', or 'auto')") + +if(WITH_THRIFT STREQUAL "system" OR WITH_THRIFT STREQUAL "auto") + FIND_PACKAGE(Thrift) + + if (Thrift_FOUND) + add_custom_target(external_thrift) + set(THRIFT_INCLUDE_DIR "${THRIFT_INCLUDE_DIR}") + set(THRIFT_LIBRARY "${THRIFT_LIBRARIES}") + return() + elseif(WITH_THRIFT STREQUAL "system") + message(FATAL_ERROR "System Thrift requested but not found!") + endif() +endif() + include(ExternalProject) set(INSTALL_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/thrift) From 4b7016e67b5d7cedc5dbad07802f77ba15a2118e Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Tue, 25 Feb 2025 08:09:49 +0000 Subject: [PATCH 47/65] fix(MCOL-5842): Fix JSON_OBJECT's handling of empty strings JSON_OBJECT() (and probably some other JSON functions) now properly handle empty strings in their arguments - JSON_OBJECT used to return NULL, now it returns empty string. --- .../MCOL-5842-group-concat-json-object.result | 12 ++++++++++++ .../bugfixes/MCOL-5842-group-concat-json-object.test | 12 ++++++++++++ utils/funcexp/jsonhelpers.cpp | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 mysql-test/columnstore/bugfixes/MCOL-5842-group-concat-json-object.result create mode 100644 mysql-test/columnstore/bugfixes/MCOL-5842-group-concat-json-object.test diff --git a/mysql-test/columnstore/bugfixes/MCOL-5842-group-concat-json-object.result b/mysql-test/columnstore/bugfixes/MCOL-5842-group-concat-json-object.result new file mode 100644 index 000000000..8d8f9b3aa --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-5842-group-concat-json-object.result @@ -0,0 +1,12 @@ +DROP DATABASE IF EXISTS MCOL5842; +CREATE DATABASE MCOL5842; +USE MCOL5842; +CREATE TABLE tcs(t TEXT) ENGINE=Columnstore; +INSERT INTO tcs(t) VALUES (''); +SELECT JSON_OBJECT('t', t, 'a', 'b') FROM tcs; +JSON_OBJECT('t', t, 'a', 'b') +{"t": "", "a": "b"} +SELECT GROUP_CONCAT(JSON_OBJECT('t', t, 'a', 'b')) FROM tcs; +GROUP_CONCAT(JSON_OBJECT('t', t, 'a', 'b')) +{"t": "", "a": "b"} +DROP DATABASE MCOL5842; diff --git a/mysql-test/columnstore/bugfixes/MCOL-5842-group-concat-json-object.test b/mysql-test/columnstore/bugfixes/MCOL-5842-group-concat-json-object.test new file mode 100644 index 000000000..6e203ac27 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-5842-group-concat-json-object.test @@ -0,0 +1,12 @@ +--disable_warnings +DROP DATABASE IF EXISTS MCOL5842; +--enable_warnings +CREATE DATABASE MCOL5842; +USE MCOL5842; + +CREATE TABLE tcs(t TEXT) ENGINE=Columnstore; +INSERT INTO tcs(t) VALUES (''); +SELECT JSON_OBJECT('t', t, 'a', 'b') FROM tcs; +SELECT GROUP_CONCAT(JSON_OBJECT('t', t, 'a', 'b')) FROM tcs; + +DROP DATABASE MCOL5842; diff --git a/utils/funcexp/jsonhelpers.cpp b/utils/funcexp/jsonhelpers.cpp index 042a3f33a..a1a0ca4a6 100644 --- a/utils/funcexp/jsonhelpers.cpp +++ b/utils/funcexp/jsonhelpers.cpp @@ -32,7 +32,7 @@ bool appendEscapedJS(string& ret, const CHARSET_INFO* retCS, const utils::NullSt int strLen = jsLen * 12 * jsCS->mbmaxlen / jsCS->mbminlen; char* buf = (char*)alloca(strLen); if ((strLen = json_escape(retCS, (const uchar*)rawJS, (const uchar*)rawJS + jsLen, jsCS, (uchar*)buf, - (uchar*)buf + strLen)) > 0) + (uchar*)buf + strLen)) >= 0) { buf[strLen] = '\0'; ret.append(buf, strLen); From e99db9c21298ad56fc07333a408006aec2da1e43 Mon Sep 17 00:00:00 2001 From: Sergey Zefirov <72864488+mariadb-SergeyZefirov@users.noreply.github.com> Date: Wed, 5 Mar 2025 10:36:05 +0300 Subject: [PATCH 48/65] fix(plugin): MCOL-4942 No-table-SELECT now can return empty set (#3415) The query like "SELECT 1 WHERE 1=0" was returning a row despite unsatisfiable condition in WHERE. Now it returns an empty set. --- dbcon/joblist/jlf_tuplejoblist.cpp | 54 +++++++++---------- dbcon/joblist/tupleconstantstep.cpp | 11 +++- dbcon/joblist/tupleconstantstep.h | 1 + ...MCOL-4942-constant-select-empty-set.result | 22 ++++++++ .../MCOL-4942-constant-select-empty-set.test | 40 ++++++++++++++ utils/loggingcpp/exceptclasses.h | 18 +++++++ 6 files changed, 117 insertions(+), 29 deletions(-) create mode 100644 mysql-test/columnstore/bugfixes/MCOL-4942-constant-select-empty-set.result create mode 100644 mysql-test/columnstore/bugfixes/MCOL-4942-constant-select-empty-set.test diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 0413a8bae..866deed85 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -4520,33 +4520,6 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte cout << endl; } - // @bug 2771, handle no table select query - if (jobInfo.tableList.empty()) - { - makeNoTableJobStep(querySteps, projectSteps, deliverySteps, jobInfo); - return; - } - - // Create a step vector for each table in the from clause. - TableInfoMap tableInfoMap; - - for (uint64_t i = 0; i < jobInfo.tableList.size(); i++) - { - uint32_t tableUid = jobInfo.tableList[i]; - tableInfoMap[tableUid] = TableInfo(); - tableInfoMap[tableUid].fTableOid = jobInfo.keyInfo->tupleKeyVec[tableUid].fId; - tableInfoMap[tableUid].fName = jobInfo.keyInfo->keyName[tableUid]; - tableInfoMap[tableUid].fAlias = jobInfo.keyInfo->tupleKeyVec[tableUid].fTable; - tableInfoMap[tableUid].fView = jobInfo.keyInfo->tupleKeyVec[tableUid].fView; - tableInfoMap[tableUid].fSchema = jobInfo.keyInfo->tupleKeyVec[tableUid].fSchema; - tableInfoMap[tableUid].fSubId = jobInfo.keyInfo->tupleKeyVec[tableUid].fSubId; - tableInfoMap[tableUid].fColsInColMap = jobInfo.columnMap[tableUid]; - } - - // Set of the columns being projected. - for (auto i = jobInfo.pjColList.begin(); i != jobInfo.pjColList.end(); i++) - jobInfo.returnColSet.insert(i->key); - // Strip constantbooleanquerySteps for (uint64_t i = 0; i < querySteps.size();) { @@ -4582,6 +4555,33 @@ void associateTupleJobSteps(JobStepVector& querySteps, JobStepVector& projectSte } } + // @bug 2771, handle no table select query + if (jobInfo.tableList.empty()) + { + makeNoTableJobStep(querySteps, projectSteps, deliverySteps, jobInfo); + return; + } + + // Create a step vector for each table in the from clause. + TableInfoMap tableInfoMap; + + for (uint64_t i = 0; i < jobInfo.tableList.size(); i++) + { + uint32_t tableUid = jobInfo.tableList[i]; + tableInfoMap[tableUid] = TableInfo(); + tableInfoMap[tableUid].fTableOid = jobInfo.keyInfo->tupleKeyVec[tableUid].fId; + tableInfoMap[tableUid].fName = jobInfo.keyInfo->keyName[tableUid]; + tableInfoMap[tableUid].fAlias = jobInfo.keyInfo->tupleKeyVec[tableUid].fTable; + tableInfoMap[tableUid].fView = jobInfo.keyInfo->tupleKeyVec[tableUid].fView; + tableInfoMap[tableUid].fSchema = jobInfo.keyInfo->tupleKeyVec[tableUid].fSchema; + tableInfoMap[tableUid].fSubId = jobInfo.keyInfo->tupleKeyVec[tableUid].fSubId; + tableInfoMap[tableUid].fColsInColMap = jobInfo.columnMap[tableUid]; + } + + // Set of the columns being projected. + for (auto i = jobInfo.pjColList.begin(); i != jobInfo.pjColList.end(); i++) + jobInfo.returnColSet.insert(i->key); + // double check if the function join canditates are still there. JobStepVector steps = querySteps; diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index ce540ed0e..d55deb5dc 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -284,7 +284,9 @@ void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo) void TupleConstantStep::run() { if (fInputJobStepAssociation.outSize() == 0) + { throw logic_error("No input data list for constant step."); + } fInputDL = fInputJobStepAssociation.outAt(0)->rowGroupDL(); @@ -585,7 +587,9 @@ void TupleConstantStep::formatMiniStats() } // class TupleConstantOnlyStep -TupleConstantOnlyStep::TupleConstantOnlyStep(const JobInfo& jobInfo) : TupleConstantStep(jobInfo) +TupleConstantOnlyStep::TupleConstantOnlyStep(const JobInfo& jobInfo) + : TupleConstantStep(jobInfo) + , fEmptySet(jobInfo.constantFalse) { // fExtendedInfo = "TCOS: "; } @@ -667,7 +671,10 @@ void TupleConstantOnlyStep::run() fillInConstants(); - fOutputDL->insert(rgDataOut); + if (!fEmptySet) + { + fOutputDL->insert(rgDataOut); + } } catch (...) { diff --git a/dbcon/joblist/tupleconstantstep.h b/dbcon/joblist/tupleconstantstep.h index 3f92470ab..44867057e 100644 --- a/dbcon/joblist/tupleconstantstep.h +++ b/dbcon/joblist/tupleconstantstep.h @@ -132,6 +132,7 @@ class TupleConstantOnlyStep : public TupleConstantStep uint32_t nextBand(messageqcpp::ByteStream& bs) override; protected: + bool fEmptySet; using TupleConstantStep::fillInConstants; void fillInConstants() override; }; diff --git a/mysql-test/columnstore/bugfixes/MCOL-4942-constant-select-empty-set.result b/mysql-test/columnstore/bugfixes/MCOL-4942-constant-select-empty-set.result new file mode 100644 index 000000000..9b2af1c22 --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-4942-constant-select-empty-set.result @@ -0,0 +1,22 @@ +DROP DATABASE IF EXISTS MCOL4942; +CREATE DATABASE MCOL4942; +USE MCOL4942; +CREATE TABLE t1col (id INT) ENGINE=Columnstore; +SELECT * FROM +( +SELECT ID +FROM +( +SELECT 1 ID +FROM +t1col +) V +UNION ALL +SELECT ID +FROM +( +SELECT NULL ID WHERE 1111=2222 +) V +) U; +ID +DROP DATABASE MCOL4942; diff --git a/mysql-test/columnstore/bugfixes/MCOL-4942-constant-select-empty-set.test b/mysql-test/columnstore/bugfixes/MCOL-4942-constant-select-empty-set.test new file mode 100644 index 000000000..8e1c62aed --- /dev/null +++ b/mysql-test/columnstore/bugfixes/MCOL-4942-constant-select-empty-set.test @@ -0,0 +1,40 @@ +--disable_warnings +DROP DATABASE IF EXISTS MCOL4942; +--enable_warnings +CREATE DATABASE MCOL4942; +USE MCOL4942; +CREATE TABLE t1col (id INT) ENGINE=Columnstore; + +SELECT * FROM + +( + + SELECT ID + + FROM + + ( + + SELECT 1 ID + + FROM + + t1col + + ) V + + UNION ALL + + SELECT ID + + FROM + + ( + + SELECT NULL ID WHERE 1111=2222 + + ) V + +) U; + +DROP DATABASE MCOL4942; diff --git a/utils/loggingcpp/exceptclasses.h b/utils/loggingcpp/exceptclasses.h index 456436066..511c52aa7 100644 --- a/utils/loggingcpp/exceptclasses.h +++ b/utils/loggingcpp/exceptclasses.h @@ -283,6 +283,24 @@ class ProtocolError : public std::logic_error } \ } while (0) +#define idblog(x) \ + do \ + { \ + { \ + std::ostringstream os; \ + \ + os << __FILE__ << "@" << __LINE__ << ": \'" << x << "\'"; \ + std::cerr << os.str() << std::endl; \ + logging::MessageLog logger((logging::LoggingID())); \ + logging::Message message; \ + logging::Message::Args args; \ + \ + args.add(os.str()); \ + message.format(args); \ + logger.logErrorMessage(message); \ + } \ + } while (0) + #define idbassert_s(x, s) \ do \ { \ From 8e508849285299f2c8f83b7b854b765d80b7bbcf Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Tue, 4 Feb 2025 16:06:39 +0300 Subject: [PATCH 49/65] MCOL-5526: CLI rename "mcs CLUSTER" commands to just mcs. [add] now all 'cluster' subcommand commands working without using 'cluster' subcommand itself [fix] 'cluster' command hidding its help message [add] explicit names for commands [add] help-all command to list all nested help messages in man page view [add] rich colored tabled help [add] table panels for cluster commands [upd] typer dependency version and some extra indirect dependencies [add] README.md for mcs cli tool with all command reference [add] mcs.1 man page [add] README_DEV.md for mcs cli tool [fix] cmapi/CMakeLists.txt to install mcs.1 man page --- cmapi/CMakeLists.txt | 5 +- cmapi/mcs_cluster_tool/README.md | 576 +++++++++++++++++++ cmapi/mcs_cluster_tool/README_DEV.md | 19 + cmapi/mcs_cluster_tool/__main__.py | 19 +- cmapi/mcs_cluster_tool/cluster_app.py | 12 +- cmapi/mcs_cluster_tool/mcs.1 | 776 ++++++++++++++++++++++++++ cmapi/requirements.txt | 12 +- 7 files changed, 1406 insertions(+), 13 deletions(-) create mode 100644 cmapi/mcs_cluster_tool/README.md create mode 100644 cmapi/mcs_cluster_tool/README_DEV.md create mode 100644 cmapi/mcs_cluster_tool/mcs.1 diff --git a/cmapi/CMakeLists.txt b/cmapi/CMakeLists.txt index af72e94c1..059bae3a8 100644 --- a/cmapi/CMakeLists.txt +++ b/cmapi/CMakeLists.txt @@ -48,6 +48,7 @@ SET(CMAPI_DIR "${SHARE_DIR}/cmapi") SET(SYSTEMD_UNIT_DIR "/usr/lib/systemd/system") SET(SYSTEMD_ENGINE_UNIT_NAME "mariadb-columnstore") SET(CMAPI_CONF_FILEPATH "${ETC_DIR}/cmapi_server.conf") +SET(MAN_DIR "/usr/share/man/man1") STRING(TOLOWER ${CPACK_PACKAGE_NAME} SYSTEMD_UNIT_NAME) @@ -65,7 +66,8 @@ INSTALL(DIRECTORY python deps mcs_node_control failover cmapi_server engine_file DESTINATION ${CMAPI_DIR} USE_SOURCE_PERMISSIONS PATTERN "test" EXCLUDE - PATTERN "cmapi_server.conf" EXCLUDE) + PATTERN "cmapi_server.conf" EXCLUDE + PATTERN "README" EXCLUDE) INSTALL(FILES LICENSE.GPL2 VERSION DESTINATION ${CMAPI_DIR}) INSTALL(FILES check_ready.sh @@ -87,6 +89,7 @@ INSTALL(FILES mcs_gsutil INSTALL(FILES scripts/mcs_backup_manager.sh scripts/cs_package_manager.sh PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ DESTINATION ${BIN_DIR}) +INSTALL(FILES mcs_cluster_tool/mcs.1 DESTINATION ${MAN_DIR}) OPTION(RPM "Build an RPM" OFF) IF(RPM) diff --git a/cmapi/mcs_cluster_tool/README.md b/cmapi/mcs_cluster_tool/README.md new file mode 100644 index 000000000..3fd286d51 --- /dev/null +++ b/cmapi/mcs_cluster_tool/README.md @@ -0,0 +1,576 @@ +# `mcs` + +The MCS Command Line Interface is a unified tool to manage your MCS services + +**Usage**: + +```console +$ mcs [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--help`: Show this message and exit. + +**Commands**: + +* `backup`: Backup Columnstore and/or MariDB data. +* `dbrm_backup`: Columnstore DBRM Backup. +* `restore`: Restore Columnstore (and/or MariaDB) data. +* `dbrm_restore`: Restore Columnstore DBRM data. +* `help-all`: Show help for all commands in man page style. +* `status`: Get status information. +* `stop`: Stop the Columnstore cluster. +* `start`: Start the Columnstore cluster. +* `restart`: Restart the Columnstore cluster. +* `node`: Cluster nodes management. +* `set`: Set cluster parameters. +* `cluster`: MariaDB Columnstore cluster management... +* `cmapi`: CMAPI itself related commands. + +## `mcs backup` + +Backup Columnstore and/or MariDB data. + +**Usage**: + +```console +$ mcs backup [OPTIONS] +``` + +**Options**: + +* `-bl, --backup-location TEXT`: What directory to store the backups on this machine or the target machine. +Consider write permissions of the scp user and the user running this script. +Mariadb-backup will use this location as a tmp dir for S3 and remote backups temporarily. +Example: /mnt/backups/ [default: /tmp/backups/] +* `-bd, --backup-destination TEXT`: Are the backups going to be stored on the same machine this script is running on or another server - if Remote you need to setup scp=Options: "Local" or "Remote" [default: Local] +* `-scp TEXT`: Used only if --backup-destination="Remote". +The user/credentials that will be used to scp the backup files +Example: "centos@10.14.51.62" +* `-bb, --backup-bucket TEXT`: Only used if --storage=S3 +Name of the bucket to store the columnstore backups. +Example: "s3://my-cs-backups" +* `-url, --endpoint-url TEXT`: Used by on premise S3 vendors. +Example: "http://127.0.0.1:8000" +* `-nv-ssl, --no-verify-ssl / -v-ssl, --verify-ssl`: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v-ssl] +* `-s, --storage TEXT`: What storage topogoly is being used by Columnstore - found in /etc/columnstore/storagemanager.cnf. +Options: "LocalStorage" or "S3" [default: LocalStorage] +* `-i, --incremental TEXT`: Adds columnstore deltas to an existing full backup. Backup folder to apply increment could be a value or "auto_most_recent" - the incremental backup applies to last full backup. +* `-ha, --highavilability / -no-ha, --no-highavilability`: Hint wether shared storage is attached @ below on all nodes to see all data +HA LocalStorage ( /var/lib/columnstore/dataX/ ) +HA S3 ( /var/lib/columnstore/storagemanager/ ) [default: no-ha] +* `-f, --config-file TEXT`: Path to backup configuration file to load variables from. +* `-sbrm, --skip-save-brm / -no-sbrm, --no-skip-save-brm`: Skip saving brm prior to running a backup - ideal for dirty backups. [default: no-sbrm] +* `-spoll, --skip-polls / -no-spoll, --no-skip-polls`: Skip sql checks confirming no write/cpimports running. [default: no-spoll] +* `-slock, --skip-locks / -no-slock, --no-skip-locks`: Skip issuing write locks - ideal for dirty backups. [default: no-slock] +* `-smdb, --skip-mariadb-backup / -no-smdb, --no-skip-mariadb-backup`: Skip running a mariadb-backup for innodb data - ideal for incremental dirty backups. [default: no-smdb] +* `-sb, --skip-bucket-data / -no-sb, --no-skip-bucket-data`: Skip taking a copy of the columnstore data in the bucket. [default: no-sb] +* `-pi, --poll-interval INTEGER`: Number of seconds between poll checks for active writes & cpimports. [default: 5] +* `-pmw, --poll-max-wait INTEGER`: Max number of minutes for polling checks for writes to wait before exiting as a failed backup attempt. [default: 60] +* `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] +* `-c, --compress TEXT`: Compress backup in X format - Options: [ pigz ]. +* `-P, --parallel INTEGER`: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if "-c/--compress" argument not set. [default: 4] +* `-nb, --name-backup TEXT`: Define the name of the backup - default: $(date +%m-%d-%Y) [default: 03-06-2025] +* `-r, --retention-days INTEGER`: Retain backups created within the last X days, default 0 == keep all backups. [default: 0] +* `--help`: Show this message and exit. + +## `mcs dbrm_backup` + +Columnstore DBRM Backup. + +**Usage**: + +```console +$ mcs dbrm_backup [OPTIONS] +``` + +**Options**: + +* `-m, --mode TEXT`: "loop" or "once" ; Determines if this script runs in a forever loop sleeping -i minutes or just once. [default: once] +* `-i, --interval INTEGER`: Number of minutes to sleep when --mode=loop. [default: 90] +* `-r, --retention-days INTEGER`: Retain dbrm backups created within the last X days, the rest are deleted [default: 7] +* `-p, --path TEXT`: Path of where to save the dbrm backups on disk. [default: /tmp/dbrm_backups] +* `-nb, --name-backup TEXT`: Custom name to prefex dbrm backups with. [default: dbrm_backup] +* `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] +* `-ssm, --skip-storage-manager / -no-ssm, --no-skip-storage-manager`: Skip backing up storagemanager directory. [default: no-ssm] +* `--help`: Show this message and exit. + +## `mcs restore` + +Restore Columnstore (and/or MariaDB) data. + +**Usage**: + +```console +$ mcs restore [OPTIONS] +``` + +**Options**: + +* `-l, --load TEXT`: What date folder to load from the backup_location. +* `-bl, --backup-location TEXT`: Where the backup to load is found. +Example: /mnt/backups/ [default: /tmp/backups/] +* `-bd, --backup_destination TEXT`: Is this backup on the same or remote server compared to where this script is running. +Options: "Local" or "Remote" [default: Local] +* `-scp, --secure-copy-protocol TEXT`: Used only if --backup-destination=RemoteThe user/credentials that will be used to scp the backup files.Example: "centos@10.14.51.62" +* `-bb, --backup-bucket TEXT`: Only used if --storage=S3 +Name of the bucket to store the columnstore backups. +Example: "s3://my-cs-backups" +* `-url, --endpoint-url TEXT`: Used by on premise S3 vendors. +Example: "http://127.0.0.1:8000" +* `-s, --storage TEXT`: What storage topogoly is being used by Columnstore - found in /etc/columnstore/storagemanager.cnf. +Options: "LocalStorage" or "S3" [default: LocalStorage] +* `-dbs, --dbroots INTEGER`: Number of database roots in the backup. [default: 1] +* `-pm, --nodeid TEXT`: Forces the handling of the restore as this node as opposed to whats detected on disk. +* `-nb, --new-bucket TEXT`: Defines the new bucket to copy the s3 data to from the backup bucket. Use -nb if the new restored cluster should use a different bucket than the backup bucket itself. +* `-nr, --new-region TEXT`: Defines the region of the new bucket to copy the s3 data to from the backup bucket. +* `-nk, --new-key TEXT`: Defines the aws key to connect to the new_bucket. +* `-ns, --new-secret TEXT`: Defines the aws secret of the aws key to connect to the new_bucket. +* `-ha, --highavilability / -no-ha, --no-highavilability`: Flag for high available systems (meaning shared storage exists supporting the topology so that each node sees all data) [default: no-ha] +* `-cont, --continue / -no-cont, --no-continue`: This acknowledges data in your --new_bucket is ok to delete when restoring S3. When set to true skips the enforcement that new_bucket should be empty prior to starting a restore. [default: no-cont] +* `-f, --config-file TEXT`: Path to backup configuration file to load variables from. +* `-smdb, --skip-mariadb-backup / -no-smdb, --no-skip-mariadb-backup`: Skip restoring mariadb server via mariadb-backup - ideal for only restoring columnstore. [default: no-smdb] +* `-sb, --skip-bucket-data / -no-sb, --no-skip-bucket-data`: Skip restoring columnstore data in the bucket - ideal if looking to only restore mariadb server. [default: no-sb] +* `-c, --compress TEXT`: Hint that the backup is compressed in X format. Options: [ pigz ]. +* `-P, --parallel INTEGER`: Determines number of decompression and mdbstream threads. Ignored if "-c/--compress" argument not set. [default: 4] +* `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] +* `-nv-ssl, --no-verify-ssl / -v-ssl, --verify-ssl`: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v-ssl] +* `--help`: Show this message and exit. + +## `mcs dbrm_restore` + +Restore Columnstore DBRM data. + +**Usage**: + +```console +$ mcs dbrm_restore [OPTIONS] +``` + +**Options**: + +* `-p, --path TEXT`: Path of where dbrm backups stored on disk. [default: /tmp/dbrm_backups] +* `-d, --directory TEXT`: Date or directory chose to restore from. +* `-ns, --no-start`: Do not attempt columnstore startup post dbrm_restore. +* `-sdbk, --skip-dbrm-backup / -no-sdbk, --no-skip-dbrm-backup`: Skip backing up dbrms before restoring. [default: sdbk] +* `-ssm, --skip-storage-manager / -no-ssm, --no-skip-storage-manager`: Skip backing up storagemanager directory. [default: ssm] +* `--help`: Show this message and exit. + +## `mcs help-all` + +Show help for all commands in man page style. + +**Usage**: + +```console +$ mcs help-all [OPTIONS] +``` + +## `mcs status` + +Get status information. + +**Usage**: + +```console +$ mcs status [OPTIONS] +``` + +**Options**: + +* `--help`: Show this message and exit. + +## `mcs stop` + +Stop the Columnstore cluster. + +**Usage**: + +```console +$ mcs stop [OPTIONS] +``` + +**Options**: + +* `-i, --interactive / -no-i, --no-interactive`: Use this option on active cluster as interactive stop waits for current writes to complete in DMLProc before shutting down. Ensuring consistency, preventing data loss of active writes. [default: no-interactive] +* `-t, --timeout INTEGER`: Time in seconds to wait for DMLproc to gracefully stop.Warning: Low wait timeout values could result in data loss if the cluster is very active.In interactive mode means delay time between promts. [default: 15] +* `--help`: Show this message and exit. + +## `mcs start` + +Start the Columnstore cluster. + +**Usage**: + +```console +$ mcs start [OPTIONS] +``` + +**Options**: + +* `--help`: Show this message and exit. + +## `mcs restart` + +Restart the Columnstore cluster. + +**Usage**: + +```console +$ mcs restart [OPTIONS] +``` + +**Options**: + +* `--help`: Show this message and exit. + +## `mcs node` + +Cluster nodes management. + +**Usage**: + +```console +$ mcs node [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--help`: Show this message and exit. + +**Commands**: + +* `add`: Add nodes to the Columnstore cluster. +* `remove`: Remove nodes from the Columnstore cluster. + +### `mcs node add` + +Add nodes to the Columnstore cluster. + +**Usage**: + +```console +$ mcs node add [OPTIONS] +``` + +**Options**: + +* `--node TEXT`: node IP, name or FQDN. Can be used multiple times to add several nodes at a time. [required] +* `--help`: Show this message and exit. + +### `mcs node remove` + +Remove nodes from the Columnstore cluster. + +**Usage**: + +```console +$ mcs node remove [OPTIONS] +``` + +**Options**: + +* `--node TEXT`: node IP, name or FQDN. Can be used multiple times to remove several nodes at a time. [required] +* `--help`: Show this message and exit. + +## `mcs set` + +Set cluster parameters. + +**Usage**: + +```console +$ mcs set [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--help`: Show this message and exit. + +**Commands**: + +* `mode`: Set Columnstore cluster mode. +* `api-key`: Set API key for communication with cluster... +* `log-level`: Set logging level on all cluster nodes for... + +### `mcs set mode` + +Set Columnstore cluster mode. + +**Usage**: + +```console +$ mcs set mode [OPTIONS] +``` + +**Options**: + +* `--mode TEXT`: cluster mode to set. "readonly" or "readwrite" are the only acceptable values. [required] +* `--help`: Show this message and exit. + +### `mcs set api-key` + +Set API key for communication with cluster nodes via API. + +WARNING: this command will affect API key value on all cluster nodes. + +**Usage**: + +```console +$ mcs set api-key [OPTIONS] +``` + +**Options**: + +* `--key TEXT`: API key to set. [required] +* `--help`: Show this message and exit. + +### `mcs set log-level` + +Set logging level on all cluster nodes for develop purposes. + +WARNING: this could dramatically affect the number of log lines. + +**Usage**: + +```console +$ mcs set log-level [OPTIONS] +``` + +**Options**: + +* `--level TEXT`: Logging level to set. [required] +* `--help`: Show this message and exit. + +## `mcs cluster` + +MariaDB Columnstore cluster management command line tool. + +**Usage**: + +```console +$ mcs cluster [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--help`: Show this message and exit. + +**Commands**: + +* `status`: Get status information. +* `stop`: Stop the Columnstore cluster. +* `start`: Start the Columnstore cluster. +* `restart`: Restart the Columnstore cluster. +* `node`: Cluster nodes management. +* `set`: Set cluster parameters. + +### `mcs cluster status` + +Get status information. + +**Usage**: + +```console +$ mcs cluster status [OPTIONS] +``` + +**Options**: + +* `--help`: Show this message and exit. + +### `mcs cluster stop` + +Stop the Columnstore cluster. + +**Usage**: + +```console +$ mcs cluster stop [OPTIONS] +``` + +**Options**: + +* `-i, --interactive / -no-i, --no-interactive`: Use this option on active cluster as interactive stop waits for current writes to complete in DMLProc before shutting down. Ensuring consistency, preventing data loss of active writes. [default: no-interactive] +* `-t, --timeout INTEGER`: Time in seconds to wait for DMLproc to gracefully stop.Warning: Low wait timeout values could result in data loss if the cluster is very active.In interactive mode means delay time between promts. [default: 15] +* `--help`: Show this message and exit. + +### `mcs cluster start` + +Start the Columnstore cluster. + +**Usage**: + +```console +$ mcs cluster start [OPTIONS] +``` + +**Options**: + +* `--help`: Show this message and exit. + +### `mcs cluster restart` + +Restart the Columnstore cluster. + +**Usage**: + +```console +$ mcs cluster restart [OPTIONS] +``` + +**Options**: + +* `--help`: Show this message and exit. + +### `mcs cluster node` + +Cluster nodes management. + +**Usage**: + +```console +$ mcs cluster node [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--help`: Show this message and exit. + +**Commands**: + +* `add`: Add nodes to the Columnstore cluster. +* `remove`: Remove nodes from the Columnstore cluster. + +#### `mcs cluster node add` + +Add nodes to the Columnstore cluster. + +**Usage**: + +```console +$ mcs cluster node add [OPTIONS] +``` + +**Options**: + +* `--node TEXT`: node IP, name or FQDN. Can be used multiple times to add several nodes at a time. [required] +* `--help`: Show this message and exit. + +#### `mcs cluster node remove` + +Remove nodes from the Columnstore cluster. + +**Usage**: + +```console +$ mcs cluster node remove [OPTIONS] +``` + +**Options**: + +* `--node TEXT`: node IP, name or FQDN. Can be used multiple times to remove several nodes at a time. [required] +* `--help`: Show this message and exit. + +### `mcs cluster set` + +Set cluster parameters. + +**Usage**: + +```console +$ mcs cluster set [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--help`: Show this message and exit. + +**Commands**: + +* `mode`: Set Columnstore cluster mode. +* `api-key`: Set API key for communication with cluster... +* `log-level`: Set logging level on all cluster nodes for... + +#### `mcs cluster set mode` + +Set Columnstore cluster mode. + +**Usage**: + +```console +$ mcs cluster set mode [OPTIONS] +``` + +**Options**: + +* `--mode TEXT`: cluster mode to set. "readonly" or "readwrite" are the only acceptable values. [required] +* `--help`: Show this message and exit. + +#### `mcs cluster set api-key` + +Set API key for communication with cluster nodes via API. + +WARNING: this command will affect API key value on all cluster nodes. + +**Usage**: + +```console +$ mcs cluster set api-key [OPTIONS] +``` + +**Options**: + +* `--key TEXT`: API key to set. [required] +* `--help`: Show this message and exit. + +#### `mcs cluster set log-level` + +Set logging level on all cluster nodes for develop purposes. + +WARNING: this could dramatically affect the number of log lines. + +**Usage**: + +```console +$ mcs cluster set log-level [OPTIONS] +``` + +**Options**: + +* `--level TEXT`: Logging level to set. [required] +* `--help`: Show this message and exit. + +## `mcs cmapi` + +CMAPI itself related commands. + +**Usage**: + +```console +$ mcs cmapi [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--help`: Show this message and exit. + +**Commands**: + +* `is-ready`: Check CMAPI is ready to handle requests. + +### `mcs cmapi is-ready` + +Check CMAPI is ready to handle requests. + +**Usage**: + +```console +$ mcs cmapi is-ready [OPTIONS] +``` + +**Options**: + +* `--node TEXT`: Which node to check the CMAPI is ready to handle requests. [default: 127.0.0.1] +* `--help`: Show this message and exit. diff --git a/cmapi/mcs_cluster_tool/README_DEV.md b/cmapi/mcs_cluster_tool/README_DEV.md new file mode 100644 index 000000000..fbf0ebebf --- /dev/null +++ b/cmapi/mcs_cluster_tool/README_DEV.md @@ -0,0 +1,19 @@ +# Generating documentation for MCS cli tool +- install cmapi requirements + ```bash + pip install -r requirements.txt + ``` +- generate markdown + ```bash + typer mcs_cluster_tool/__main__.py utils docs --name mcs --output README.md + ``` +- install `md2man` (for now it's the only one tool that make convertation without any issues) + ```bash + sudo yum install -y ruby ruby-devel + gem install md2man + ``` +- convert to perfect `.roff` file (`man` page) + ```bash + md2man README.md > mcs.1 + ``` +- enjoy =) \ No newline at end of file diff --git a/cmapi/mcs_cluster_tool/__main__.py b/cmapi/mcs_cluster_tool/__main__.py index c2e1a8b38..35bcaa82d 100644 --- a/cmapi/mcs_cluster_tool/__main__.py +++ b/cmapi/mcs_cluster_tool/__main__.py @@ -1,4 +1,5 @@ import logging +import subprocess import sys import typer @@ -17,15 +18,27 @@ app = typer.Typer( 'The MCS Command Line Interface is a unified tool to manage your ' 'MCS services' ), + rich_markup_mode='rich', ) -app.add_typer(cluster_app.app, name='cluster') +app.add_typer(cluster_app.app) +# TODO: keep this only for potential backward compatibility +app.add_typer(cluster_app.app, name='cluster', hidden=True) app.add_typer(cmapi_app.app, name='cmapi') -app.command()(backup_commands.backup) +app.command('backup')(backup_commands.backup) app.command('dbrm_backup')(backup_commands.dbrm_backup) -app.command()(restore_commands.restore) +app.command('restore')(restore_commands.restore) app.command('dbrm_restore')(restore_commands.dbrm_restore) +@app.command( + name='help-all', help='Show help for all commands in man page style.', + add_help_option=False +) +def help_all(): + # Open the man page in interactive mode + subprocess.run(['man', 'mcs']) + + if __name__ == '__main__': add_logging_level('TRACE', 5) #TODO: remove when stadalone mode added. dict_config(MCS_CLI_LOG_CONF_PATH) diff --git a/cmapi/mcs_cluster_tool/cluster_app.py b/cmapi/mcs_cluster_tool/cluster_app.py index 07934f31e..67602abe9 100644 --- a/cmapi/mcs_cluster_tool/cluster_app.py +++ b/cmapi/mcs_cluster_tool/cluster_app.py @@ -35,14 +35,14 @@ set_app = typer.Typer(help='Set cluster parameters.') app.add_typer(set_app, name='set') -@app.command() +@app.command(rich_help_panel='cluster and single node commands') @handle_output def status(): """Get status information.""" return ClusterHandler.status(logger=logger) -@app.command() +@app.command(rich_help_panel='cluster and single node commands') @handle_output @TransactionManager( timeout=timedelta(days=1).total_seconds(), handle_signals=True @@ -161,14 +161,14 @@ def stop( return {'timestamp': start_time} -@app.command() +@app.command(rich_help_panel='cluster and single node commands') @handle_output def start(): """Start the Columnstore cluster.""" return ClusterHandler.start(logger=logger) -@app.command() +@app.command(rich_help_panel='cluster and single node commands') @handle_output def restart(): """Restart the Columnstore cluster.""" @@ -180,7 +180,7 @@ def restart(): return result -@node_app.command() +@node_app.command(rich_help_panel='cluster node commands') @handle_output def add( nodes: Optional[List[str]] = typer.Option( @@ -199,7 +199,7 @@ def add( return result -@node_app.command() +@node_app.command(rich_help_panel='cluster node commands') @handle_output def remove(nodes: Optional[List[str]] = typer.Option( ..., diff --git a/cmapi/mcs_cluster_tool/mcs.1 b/cmapi/mcs_cluster_tool/mcs.1 new file mode 100644 index 000000000..6b1e32ed2 --- /dev/null +++ b/cmapi/mcs_cluster_tool/mcs.1 @@ -0,0 +1,776 @@ +.TH \fB\fCmcs\fR +.PP +The MCS Command Line Interface is a unified tool to manage your MCS services +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs [OPTIONS] COMMAND [ARGS]... +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.PP +\fBCommands\fP: +.RS +.IP \(bu 2 +\fB\fCbackup\fR: Backup Columnstore and/or MariDB data. +.IP \(bu 2 +\fB\fCdbrm_backup\fR: Columnstore DBRM Backup. +.IP \(bu 2 +\fB\fCrestore\fR: Restore Columnstore (and/or MariaDB) data. +.IP \(bu 2 +\fB\fCdbrm_restore\fR: Restore Columnstore DBRM data. +.IP \(bu 2 +\fB\fChelp\-all\fR: Show help for all commands in man page style. +.IP \(bu 2 +\fB\fCstatus\fR: Get status information. +.IP \(bu 2 +\fB\fCstop\fR: Stop the Columnstore cluster. +.IP \(bu 2 +\fB\fCstart\fR: Start the Columnstore cluster. +.IP \(bu 2 +\fB\fCrestart\fR: Restart the Columnstore cluster. +.IP \(bu 2 +\fB\fCnode\fR: Cluster nodes management. +.IP \(bu 2 +\fB\fCset\fR: Set cluster parameters. +.IP \(bu 2 +\fB\fCcluster\fR: MariaDB Columnstore cluster management... +.IP \(bu 2 +\fB\fCcmapi\fR: CMAPI itself related commands. +.RE +.SH \fB\fCmcs backup\fR +.PP +Backup Columnstore and/or MariDB data. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs backup [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-bl, \-\-backup\-location TEXT\fR: What directory to store the backups on this machine or the target machine. +Consider write permissions of the scp user and the user running this script. +Mariadb\-backup will use this location as a tmp dir for S3 and remote backups temporarily. +Example: /mnt/backups/ [default: /tmp/backups/] +.IP \(bu 2 +\fB\fC\-bd, \-\-backup\-destination TEXT\fR: Are the backups going to be stored on the same machine this script is running on or another server \- if Remote you need to setup scp=Options: \[dq]Local\[dq] or \[dq]Remote\[dq] [default: Local] +.IP \(bu 2 +\fB\fC\-scp TEXT\fR: Used only if \-\-backup\-destination=\[dq]Remote\[dq]\&. +The user/credentials that will be used to scp the backup files +Example: \[dq]\[la]centos@10.14.51.62\[ra]\[dq] +.IP \(bu 2 +\fB\fC\-bb, \-\-backup\-bucket TEXT\fR: Only used if \-\-storage=S3 +Name of the bucket to store the columnstore backups. +Example: \[dq]s3://my\-cs\-backups\[dq] +.IP \(bu 2 +\fB\fC\-url, \-\-endpoint\-url TEXT\fR: Used by on premise S3 vendors. +Example: \[dq]\[la]http://127.0.0.1:8000\[ra]\[dq] +.IP \(bu 2 +\fB\fC\-nv\-ssl, \-\-no\-verify\-ssl / \-v\-ssl, \-\-verify\-ssl\fR: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v\-ssl] +.IP \(bu 2 +\fB\fC\-s, \-\-storage TEXT\fR: What storage topogoly is being used by Columnstore \- found in /etc/columnstore/storagemanager.cnf. +Options: \[dq]LocalStorage\[dq] or \[dq]S3\[dq] [default: LocalStorage] +.IP \(bu 2 +\fB\fC\-i, \-\-incremental TEXT\fR: Adds columnstore deltas to an existing full backup. Backup folder to apply increment could be a value or \[dq]auto\fImost\fPrecent\[dq] \- the incremental backup applies to last full backup. +.IP \(bu 2 +\fB\fC\-ha, \-\-highavilability / \-no\-ha, \-\-no\-highavilability\fR: Hint wether shared storage is attached @ below on all nodes to see all data +HA LocalStorage ( /var/lib/columnstore/dataX/ ) +HA S3 ( /var/lib/columnstore/storagemanager/ ) [default: no\-ha] +.IP \(bu 2 +\fB\fC\-f, \-\-config\-file TEXT\fR: Path to backup configuration file to load variables from. +.IP \(bu 2 +\fB\fC\-sbrm, \-\-skip\-save\-brm / \-no\-sbrm, \-\-no\-skip\-save\-brm\fR: Skip saving brm prior to running a backup \- ideal for dirty backups. [default: no\-sbrm] +.IP \(bu 2 +\fB\fC\-spoll, \-\-skip\-polls / \-no\-spoll, \-\-no\-skip\-polls\fR: Skip sql checks confirming no write/cpimports running. [default: no\-spoll] +.IP \(bu 2 +\fB\fC\-slock, \-\-skip\-locks / \-no\-slock, \-\-no\-skip\-locks\fR: Skip issuing write locks \- ideal for dirty backups. [default: no\-slock] +.IP \(bu 2 +\fB\fC\-smdb, \-\-skip\-mariadb\-backup / \-no\-smdb, \-\-no\-skip\-mariadb\-backup\fR: Skip running a mariadb\-backup for innodb data \- ideal for incremental dirty backups. [default: no\-smdb] +.IP \(bu 2 +\fB\fC\-sb, \-\-skip\-bucket\-data / \-no\-sb, \-\-no\-skip\-bucket\-data\fR: Skip taking a copy of the columnstore data in the bucket. [default: no\-sb] +.IP \(bu 2 +\fB\fC\-pi, \-\-poll\-interval INTEGER\fR: Number of seconds between poll checks for active writes & cpimports. [default: 5] +.IP \(bu 2 +\fB\fC\-pmw, \-\-poll\-max\-wait INTEGER\fR: Max number of minutes for polling checks for writes to wait before exiting as a failed backup attempt. [default: 60] +.IP \(bu 2 +\fB\fC\-q, \-\-quiet / \-no\-q, \-\-no\-quiet\fR: Silence verbose copy command outputs. [default: no\-q] +.IP \(bu 2 +\fB\fC\-c, \-\-compress TEXT\fR: Compress backup in X format \- Options: [ pigz ]. +.IP \(bu 2 +\fB\fC\-P, \-\-parallel INTEGER\fR: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if \[dq]\-c/\-\-compress\[dq] argument not set. [default: 4] +.IP \(bu 2 +\fB\fC\-nb, \-\-name\-backup TEXT\fR: Define the name of the backup \- default: $(date +%m\-%d\-%Y) [default: 03\-06\-2025] +.IP \(bu 2 +\fB\fC\-r, \-\-retention\-days INTEGER\fR: Retain backups created within the last X days, default 0 == keep all backups. [default: 0] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs dbrm_backup\fR +.PP +Columnstore DBRM Backup. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs dbrm_backup [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-m, \-\-mode TEXT\fR: \[dq]loop\[dq] or \[dq]once\[dq] ; Determines if this script runs in a forever loop sleeping \-i minutes or just once. [default: once] +.IP \(bu 2 +\fB\fC\-i, \-\-interval INTEGER\fR: Number of minutes to sleep when \-\-mode=loop. [default: 90] +.IP \(bu 2 +\fB\fC\-r, \-\-retention\-days INTEGER\fR: Retain dbrm backups created within the last X days, the rest are deleted [default: 7] +.IP \(bu 2 +\fB\fC\-p, \-\-path TEXT\fR: Path of where to save the dbrm backups on disk. [default: /tmp/dbrm_backups] +.IP \(bu 2 +\fB\fC\-nb, \-\-name\-backup TEXT\fR: Custom name to prefex dbrm backups with. [default: dbrm_backup] +.IP \(bu 2 +\fB\fC\-q, \-\-quiet / \-no\-q, \-\-no\-quiet\fR: Silence verbose copy command outputs. [default: no\-q] +.IP \(bu 2 +\fB\fC\-ssm, \-\-skip\-storage\-manager / \-no\-ssm, \-\-no\-skip\-storage\-manager\fR: Skip backing up storagemanager directory. [default: no\-ssm] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs restore\fR +.PP +Restore Columnstore (and/or MariaDB) data. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs restore [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-l, \-\-load TEXT\fR: What date folder to load from the backup_location. +.IP \(bu 2 +\fB\fC\-bl, \-\-backup\-location TEXT\fR: Where the backup to load is found. +Example: /mnt/backups/ [default: /tmp/backups/] +.IP \(bu 2 +\fB\fC\-bd, \-\-backup_destination TEXT\fR: Is this backup on the same or remote server compared to where this script is running. +Options: \[dq]Local\[dq] or \[dq]Remote\[dq] [default: Local] +.IP \(bu 2 +\fB\fC\-scp, \-\-secure\-copy\-protocol TEXT\fR: Used only if \-\-backup\-destination=RemoteThe user/credentials that will be used to scp the backup files.Example: \[dq]\[la]centos@10.14.51.62\[ra]\[dq] +.IP \(bu 2 +\fB\fC\-bb, \-\-backup\-bucket TEXT\fR: Only used if \-\-storage=S3 +Name of the bucket to store the columnstore backups. +Example: \[dq]s3://my\-cs\-backups\[dq] +.IP \(bu 2 +\fB\fC\-url, \-\-endpoint\-url TEXT\fR: Used by on premise S3 vendors. +Example: \[dq]\[la]http://127.0.0.1:8000\[ra]\[dq] +.IP \(bu 2 +\fB\fC\-s, \-\-storage TEXT\fR: What storage topogoly is being used by Columnstore \- found in /etc/columnstore/storagemanager.cnf. +Options: \[dq]LocalStorage\[dq] or \[dq]S3\[dq] [default: LocalStorage] +.IP \(bu 2 +\fB\fC\-dbs, \-\-dbroots INTEGER\fR: Number of database roots in the backup. [default: 1] +.IP \(bu 2 +\fB\fC\-pm, \-\-nodeid TEXT\fR: Forces the handling of the restore as this node as opposed to whats detected on disk. +.IP \(bu 2 +\fB\fC\-nb, \-\-new\-bucket TEXT\fR: Defines the new bucket to copy the s3 data to from the backup bucket. Use \-nb if the new restored cluster should use a different bucket than the backup bucket itself. +.IP \(bu 2 +\fB\fC\-nr, \-\-new\-region TEXT\fR: Defines the region of the new bucket to copy the s3 data to from the backup bucket. +.IP \(bu 2 +\fB\fC\-nk, \-\-new\-key TEXT\fR: Defines the aws key to connect to the new_bucket. +.IP \(bu 2 +\fB\fC\-ns, \-\-new\-secret TEXT\fR: Defines the aws secret of the aws key to connect to the new_bucket. +.IP \(bu 2 +\fB\fC\-ha, \-\-highavilability / \-no\-ha, \-\-no\-highavilability\fR: Flag for high available systems (meaning shared storage exists supporting the topology so that each node sees all data) [default: no\-ha] +.IP \(bu 2 +\fB\fC\-cont, \-\-continue / \-no\-cont, \-\-no\-continue\fR: This acknowledges data in your \-\-new\fIbucket is ok to delete when restoring S3. When set to true skips the enforcement that new\fPbucket should be empty prior to starting a restore. [default: no\-cont] +.IP \(bu 2 +\fB\fC\-f, \-\-config\-file TEXT\fR: Path to backup configuration file to load variables from. +.IP \(bu 2 +\fB\fC\-smdb, \-\-skip\-mariadb\-backup / \-no\-smdb, \-\-no\-skip\-mariadb\-backup\fR: Skip restoring mariadb server via mariadb\-backup \- ideal for only restoring columnstore. [default: no\-smdb] +.IP \(bu 2 +\fB\fC\-sb, \-\-skip\-bucket\-data / \-no\-sb, \-\-no\-skip\-bucket\-data\fR: Skip restoring columnstore data in the bucket \- ideal if looking to only restore mariadb server. [default: no\-sb] +.IP \(bu 2 +\fB\fC\-c, \-\-compress TEXT\fR: Hint that the backup is compressed in X format. Options: [ pigz ]. +.IP \(bu 2 +\fB\fC\-P, \-\-parallel INTEGER\fR: Determines number of decompression and mdbstream threads. Ignored if \[dq]\-c/\-\-compress\[dq] argument not set. [default: 4] +.IP \(bu 2 +\fB\fC\-q, \-\-quiet / \-no\-q, \-\-no\-quiet\fR: Silence verbose copy command outputs. [default: no\-q] +.IP \(bu 2 +\fB\fC\-nv\-ssl, \-\-no\-verify\-ssl / \-v\-ssl, \-\-verify\-ssl\fR: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v\-ssl] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs dbrm_restore\fR +.PP +Restore Columnstore DBRM data. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs dbrm_restore [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-p, \-\-path TEXT\fR: Path of where dbrm backups stored on disk. [default: /tmp/dbrm_backups] +.IP \(bu 2 +\fB\fC\-d, \-\-directory TEXT\fR: Date or directory chose to restore from. +.IP \(bu 2 +\fB\fC\-ns, \-\-no\-start\fR: Do not attempt columnstore startup post dbrm_restore. +.IP \(bu 2 +\fB\fC\-sdbk, \-\-skip\-dbrm\-backup / \-no\-sdbk, \-\-no\-skip\-dbrm\-backup\fR: Skip backing up dbrms before restoring. [default: sdbk] +.IP \(bu 2 +\fB\fC\-ssm, \-\-skip\-storage\-manager / \-no\-ssm, \-\-no\-skip\-storage\-manager\fR: Skip backing up storagemanager directory. [default: ssm] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs help\-all\fR +.PP +Show help for all commands in man page style. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs help\-all [OPTIONS] +.fi +.RE +.SH \fB\fCmcs status\fR +.PP +Get status information. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs status [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs stop\fR +.PP +Stop the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs stop [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-i, \-\-interactive / \-no\-i, \-\-no\-interactive\fR: Use this option on active cluster as interactive stop waits for current writes to complete in DMLProc before shutting down. Ensuring consistency, preventing data loss of active writes. [default: no\-interactive] +.IP \(bu 2 +\fB\fC\-t, \-\-timeout INTEGER\fR: Time in seconds to wait for DMLproc to gracefully stop.Warning: Low wait timeout values could result in data loss if the cluster is very active.In interactive mode means delay time between promts. [default: 15] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs start\fR +.PP +Start the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs start [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs restart\fR +.PP +Restart the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs restart [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs node\fR +.PP +Cluster nodes management. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs node [OPTIONS] COMMAND [ARGS]... +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.PP +\fBCommands\fP: +.RS +.IP \(bu 2 +\fB\fCadd\fR: Add nodes to the Columnstore cluster. +.IP \(bu 2 +\fB\fCremove\fR: Remove nodes from the Columnstore cluster. +.RE +.SS \fB\fCmcs node add\fR +.PP +Add nodes to the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs node add [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-node TEXT\fR: node IP, name or FQDN. Can be used multiple times to add several nodes at a time. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs node remove\fR +.PP +Remove nodes from the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs node remove [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-node TEXT\fR: node IP, name or FQDN. Can be used multiple times to remove several nodes at a time. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs set\fR +.PP +Set cluster parameters. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs set [OPTIONS] COMMAND [ARGS]... +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.PP +\fBCommands\fP: +.RS +.IP \(bu 2 +\fB\fCmode\fR: Set Columnstore cluster mode. +.IP \(bu 2 +\fB\fCapi\-key\fR: Set API key for communication with cluster... +.IP \(bu 2 +\fB\fClog\-level\fR: Set logging level on all cluster nodes for... +.RE +.SS \fB\fCmcs set mode\fR +.PP +Set Columnstore cluster mode. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs set mode [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-mode TEXT\fR: cluster mode to set. \[dq]readonly\[dq] or \[dq]readwrite\[dq] are the only acceptable values. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs set api\-key\fR +.PP +Set API key for communication with cluster nodes via API. +.PP +WARNING: this command will affect API key value on all cluster nodes. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs set api\-key [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-key TEXT\fR: API key to set. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs set log\-level\fR +.PP +Set logging level on all cluster nodes for develop purposes. +.PP +WARNING: this could dramatically affect the number of log lines. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs set log\-level [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-level TEXT\fR: Logging level to set. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs cluster\fR +.PP +MariaDB Columnstore cluster management command line tool. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster [OPTIONS] COMMAND [ARGS]... +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.PP +\fBCommands\fP: +.RS +.IP \(bu 2 +\fB\fCstatus\fR: Get status information. +.IP \(bu 2 +\fB\fCstop\fR: Stop the Columnstore cluster. +.IP \(bu 2 +\fB\fCstart\fR: Start the Columnstore cluster. +.IP \(bu 2 +\fB\fCrestart\fR: Restart the Columnstore cluster. +.IP \(bu 2 +\fB\fCnode\fR: Cluster nodes management. +.IP \(bu 2 +\fB\fCset\fR: Set cluster parameters. +.RE +.SS \fB\fCmcs cluster status\fR +.PP +Get status information. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster status [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs cluster stop\fR +.PP +Stop the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster stop [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-i, \-\-interactive / \-no\-i, \-\-no\-interactive\fR: Use this option on active cluster as interactive stop waits for current writes to complete in DMLProc before shutting down. Ensuring consistency, preventing data loss of active writes. [default: no\-interactive] +.IP \(bu 2 +\fB\fC\-t, \-\-timeout INTEGER\fR: Time in seconds to wait for DMLproc to gracefully stop.Warning: Low wait timeout values could result in data loss if the cluster is very active.In interactive mode means delay time between promts. [default: 15] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs cluster start\fR +.PP +Start the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster start [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs cluster restart\fR +.PP +Restart the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster restart [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs cluster node\fR +.PP +Cluster nodes management. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster node [OPTIONS] COMMAND [ARGS]... +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.PP +\fBCommands\fP: +.RS +.IP \(bu 2 +\fB\fCadd\fR: Add nodes to the Columnstore cluster. +.IP \(bu 2 +\fB\fCremove\fR: Remove nodes from the Columnstore cluster. +.RE +.SS \fB\fCmcs cluster node add\fR +.PP +Add nodes to the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster node add [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-node TEXT\fR: node IP, name or FQDN. Can be used multiple times to add several nodes at a time. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs cluster node remove\fR +.PP +Remove nodes from the Columnstore cluster. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster node remove [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-node TEXT\fR: node IP, name or FQDN. Can be used multiple times to remove several nodes at a time. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs cluster set\fR +.PP +Set cluster parameters. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster set [OPTIONS] COMMAND [ARGS]... +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.PP +\fBCommands\fP: +.RS +.IP \(bu 2 +\fB\fCmode\fR: Set Columnstore cluster mode. +.IP \(bu 2 +\fB\fCapi\-key\fR: Set API key for communication with cluster... +.IP \(bu 2 +\fB\fClog\-level\fR: Set logging level on all cluster nodes for... +.RE +.SS \fB\fCmcs cluster set mode\fR +.PP +Set Columnstore cluster mode. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster set mode [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-mode TEXT\fR: cluster mode to set. \[dq]readonly\[dq] or \[dq]readwrite\[dq] are the only acceptable values. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs cluster set api\-key\fR +.PP +Set API key for communication with cluster nodes via API. +.PP +WARNING: this command will affect API key value on all cluster nodes. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster set api\-key [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-key TEXT\fR: API key to set. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SS \fB\fCmcs cluster set log\-level\fR +.PP +Set logging level on all cluster nodes for develop purposes. +.PP +WARNING: this could dramatically affect the number of log lines. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cluster set log\-level [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-level TEXT\fR: Logging level to set. [required] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs cmapi\fR +.PP +CMAPI itself related commands. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cmapi [OPTIONS] COMMAND [ARGS]... +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.PP +\fBCommands\fP: +.RS +.IP \(bu 2 +\fB\fCis\-ready\fR: Check CMAPI is ready to handle requests. +.RE +.SS \fB\fCmcs cmapi is\-ready\fR +.PP +Check CMAPI is ready to handle requests. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cmapi is\-ready [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-node TEXT\fR: Which node to check the CMAPI is ready to handle requests. [default: 127.0.0.1] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE diff --git a/cmapi/requirements.txt b/cmapi/requirements.txt index c5367e045..217a819a4 100644 --- a/cmapi/requirements.txt +++ b/cmapi/requirements.txt @@ -7,7 +7,8 @@ lxml==4.7.1 psutil==5.9.1 pyotp==2.6.0 requests==2.27.1 -typer==0.9.0 +typer==0.15.1 + # indirect dependencies aiohttp==3.8.1 @@ -24,7 +25,7 @@ certifi==2021.10.8 cffi==1.15.0 charset-normalizer==2.0.12 cheroot==8.6.0 -click==8.1.7 +click==8.1.8 colorama==0.4.4 crcmod==1.7 docutils==0.16 @@ -44,6 +45,8 @@ jaraco.context==4.1.1 jaraco.functools==3.5.0 jaraco.text==3.7.0 jmespath==1.0.1 +markdown-it-py==3.0.0 +mdurl==0.1.2 monotonic==1.6 more-itertools==8.12.0 multidict==6.0.2 @@ -53,20 +56,23 @@ portend==3.1.0 pyasn1-modules==0.2.8 pyasn1==0.4.8 pycparser==2.21 +Pygments==2.19.1 pyOpenSSL==22.0.0 pyparsing==3.0.9 python-dateutil==2.8.2 pytz==2021.3 pyu2f==0.1.5 PyYAML==5.4.1 +rich==13.9.4 repoze.lru==0.7 retry-decorator==1.1.1 Routes==2.5.1 rsa==4.7.2 s3transfer==0.6.0 +shellingham==1.5.4 six==1.16.0 tempora==5.0.1 -typing-extensions==4.8.0 +typing_extensions==4.12.2 urllib3==1.26.8 yarl==1.8.1 zc.lockfile==2.0 From ef592038cbe2eeae7ceadcfa6e4be655163d3b25 Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Fri, 28 Feb 2025 14:05:16 +0300 Subject: [PATCH 50/65] fix(MCOL-5396): Fix possible infinite loop in plugin--PrimProc communication If you manage to shut down PrimProc just before plugin is trying to send Calpont Select Execution Plan to PrimProc, you now get a nice error message about PrimProc being down instead of endless logs of failed reconnection attempts. --- dbcon/mysql/ha_mcs_impl.cpp | 54 ++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 955602b62..04c1a3c17 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -19,6 +19,8 @@ #define PREFER_MY_CONFIG_H #include #include +#include +#include #include #include #include @@ -130,6 +132,7 @@ using namespace funcexp; #include "ha_mcs_logging.h" #include "ha_subquery.h" + namespace cal_impl_if { extern bool nonConstFunc(Item_func* ifp); @@ -2352,8 +2355,18 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) ByteStream msg; ByteStream emsgBs; + int ntries = 10; + + // XXX: MCOL-5396: unable to reach this code. while (true) { + string emsg; + if (ntries < 0) + { + emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; + } try { ByteStream::quadbyte qb = 4; @@ -2371,7 +2384,6 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) emsgBs.restart(); msg = hndl->exeMgr->read(); emsgBs = hndl->exeMgr->read(); - string emsg; if (msg.length() == 0 || emsgBs.length() == 0) { @@ -2437,6 +2449,10 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) ti.conn_hndl = hndl; + using namespace chrono_literals; + std::this_thread::sleep_for(100ms); + ntries --; + try { hndl->connect(); @@ -4239,8 +4255,18 @@ int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table) ByteStream msg; ByteStream emsgBs; + int ntries = 10; + + // XXX: MCOL-5396: unable to reach this code. while (true) { + string emsg; + if (ntries < 0) + { + emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; + } try { ByteStream::quadbyte qb = 4; @@ -4258,7 +4284,6 @@ int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table) emsgBs.restart(); msg = hndl->exeMgr->read(); emsgBs = hndl->exeMgr->read(); - string emsg; if (msg.length() == 0 || emsgBs.length() == 0) { @@ -4325,6 +4350,11 @@ int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table) ci->cal_conn_hndl = hndl; ci->cal_conn_hndl_st.pop(); ci->cal_conn_hndl_st.push(ci->cal_conn_hndl); + + using namespace std::chrono_literals; + std::this_thread::sleep_for(100ms); + ntries --; + try { hndl->connect(); @@ -4943,8 +4973,23 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool ByteStream msg; ByteStream emsgBs; + int ntries = 10; + + // The issue is MCOL-5396. + // The delay below is used to trigger old infinite loop condition and + // prove that mitigation works. While it looks like unused code, it is + // important enough to have it just in case. + // using namespace std::chrono_literals; + // std::this_thread::sleep_for(10000ms); // shut PrimProc down now. while (true) { + string emsg; + if (ntries < 0) + { + emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; + } try { ByteStream::quadbyte qb = 4; @@ -4962,7 +5007,6 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool emsgBs.restart(); msg = hndl->exeMgr->read(); emsgBs = hndl->exeMgr->read(); - string emsg; if (msg.length() == 0 || emsgBs.length() == 0) { @@ -5030,6 +5074,10 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool ci->cal_conn_hndl = hndl; + using namespace std::chrono_literals; + std::this_thread::sleep_for(100ms); + ntries --; + try { hndl->connect(); From 36a412962da907ae56f3dd1d8021ccd6d26f7920 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 30 Aug 2024 18:04:02 +0300 Subject: [PATCH 51/65] Recognize lsb_release 'RedHatEnterprise' as rhel --- cmapi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmapi/CMakeLists.txt b/cmapi/CMakeLists.txt index 059bae3a8..937175cb6 100644 --- a/cmapi/CMakeLists.txt +++ b/cmapi/CMakeLists.txt @@ -141,7 +141,7 @@ IF(RPM) STRING(REGEX MATCH "^." OS_VERSION_MAJOR "${LSB_RELEASE_VERSION_SHORT}") MESSAGE(STATUS ${OS_VERSION_MAJOR}) - IF (LSB_RELEASE_ID_SHORT MATCHES "centos|rocky|rhel|alma") + IF (LSB_RELEASE_ID_SHORT MATCHES "centos|rocky|rhel|alma|RedHatEnterprise") SET(OS_NAME_SHORT "el") IF (OS_VERSION_MAJOR MATCHES "9") SET(CPACK_RPM_PACKAGE_REQUIRES "libxcrypt-compat") From dafe35ef498b9d9ac4accad25f6561345bf9a4da Mon Sep 17 00:00:00 2001 From: Alan Mologorsky <89034356+mariadb-AlanMologorsky@users.noreply.github.com> Date: Wed, 12 Mar 2025 14:21:32 +0300 Subject: [PATCH 52/65] feat(cmapi): MCOL-5133: Stage1 to stand alone cli tool. (#3378) [add] cluster api client class [fix] cli cluster_app using cluster api client [add] ClusterAction Enum [add] toggle_cluster_state function to reduce code duplication --- cmapi/cmapi_server/constants.py | 7 + cmapi/cmapi_server/controllers/api_clients.py | 126 +++++++++++++++ cmapi/cmapi_server/controllers/endpoints.py | 6 +- cmapi/cmapi_server/handlers/cluster.py | 150 ++++++++---------- cmapi/cmapi_server/helpers.py | 1 + cmapi/cmapi_server/managers/transaction.py | 8 +- cmapi/mcs_cluster_tool/cluster_app.py | 37 +++-- 7 files changed, 226 insertions(+), 109 deletions(-) create mode 100644 cmapi/cmapi_server/controllers/api_clients.py diff --git a/cmapi/cmapi_server/constants.py b/cmapi/cmapi_server/constants.py index a1e4142b9..9fba94305 100644 --- a/cmapi/cmapi_server/constants.py +++ b/cmapi/cmapi_server/constants.py @@ -82,3 +82,10 @@ MCS_INSTALL_BIN = '/usr/bin' IFLAG = os.path.join(MCS_ETC_PATH, 'container-initialized') LIBJEMALLOC_DEFAULT_PATH = os.path.join(MCS_DATA_PATH, 'libjemalloc.so.2') MCS_LOG_PATH = '/var/log/mariadb/columnstore' + + +# client constants +CMAPI_PORT = 8640 #TODO: use it in all places +CURRENT_NODE_CMAPI_URL = f'https://localhost:{CMAPI_PORT}' +REQUEST_TIMEOUT: float = 30.0 +TRANSACTION_TIMEOUT: float = 300.0 # 5 minutes \ No newline at end of file diff --git a/cmapi/cmapi_server/controllers/api_clients.py b/cmapi/cmapi_server/controllers/api_clients.py new file mode 100644 index 000000000..424d9d99c --- /dev/null +++ b/cmapi/cmapi_server/controllers/api_clients.py @@ -0,0 +1,126 @@ +import requests +from typing import Any, Dict, Optional, Union +from cmapi_server.controllers.dispatcher import _version +from cmapi_server.constants import CURRENT_NODE_CMAPI_URL + + +class ClusterControllerClient: + def __init__( + self, base_url: str = CURRENT_NODE_CMAPI_URL, + request_timeout: Optional[float] = None + ): + """Initialize the ClusterControllerClient with the base URL. + + :param base_url: The base URL for the API endpoints, + defaults to CURRENT_NODE_CMAPI_URL + :type base_url: str, optional + :param request_timeout: request timeout, defaults to None + :type request_timeout: Optional[float], optional + """ + self.base_url = base_url + self.request_timeout = request_timeout + + def start_cluster( + self, data: Optional[Dict[str, Any]] = None + ) -> Union[Dict[str, Any], Dict[str, str]]: + """Start the cluster. + + :return: The response from the API. + """ + return self._request('PUT', 'start', data) + + def shutdown_cluster( + self, data: Optional[Dict[str, Any]] = None + ) -> Union[Dict[str, Any], Dict[str, str]]: + """Shutdown the cluster. + + :return: The response from the API. + """ + return self._request('PUT', 'shutdown', data) + + def set_mode(self, mode: str) -> Union[Dict[str, Any], Dict[str, str]]: + """Set the cluster mode. + + :param mode: The mode to set. + :return: The response from the API. + """ + return self._request('PUT', 'mode-set', {'mode': mode}) + + def add_node( + self, node_info: Dict[str, Any] + ) -> Union[Dict[str, Any], Dict[str, str]]: + """Add a node to the cluster. + + :param node_info: Information about the node to add. + :return: The response from the API. + """ + return self._request('PUT', 'node', node_info) + + def remove_node( + self, node_id: str + ) -> Union[Dict[str, Any], Dict[str, str]]: + """Remove a node from the cluster. + + :param node_id: The ID of the node to remove. + :return: The response from the API. + """ + return self._request('DELETE', 'node', {'node_id': node_id}) + + def get_status(self) -> Union[Dict[str, Any], Dict[str, str]]: + """Get the status of the cluster. + + :return: The response from the API. + """ + return self._request('GET', 'status') + + def set_api_key( + self, api_key: str + ) -> Union[Dict[str, Any], Dict[str, str]]: + """Set the API key for the cluster. + + :param api_key: The API key to set. + :return: The response from the API. + """ + return self._request('put', 'apikey-set', {'api_key': api_key}) + + def set_log_level( + self, log_level: str + ) -> Union[Dict[str, Any], Dict[str, str]]: + """Set the log level for the cluster. + + :param log_level: The log level to set. + :return: The response from the API. + """ + return self._request('put', 'log-level', {'log_level': log_level}) + + def load_s3data( + self, s3data_info: Dict[str, Any] + ) -> Union[Dict[str, Any], Dict[str, str]]: + """Load S3 data into the cluster. + + :param s3data_info: Information about the S3 data to load. + :return: The response from the API. + """ + return self._request('put', 'load_s3data', s3data_info) + + def _request( + self, method: str, endpoint: str, + data: Optional[Dict[str, Any]] = None + ) -> Union[Dict[str, Any], Dict[str, str]]: + """Make a request to the API. + + :param method: The HTTP method to use. + :param endpoint: The API endpoint to call. + :param data: The data to send with the request. + :return: The response from the API. + """ + url = f'{self.base_url}/cmapi/{_version}/cluster/{endpoint}' + try: + response = requests.request( + method, url, json=data, timeout=self.request_timeout + ) + response.raise_for_status() + return response.json() + # TODO: different handler for timeout exception? + except requests.exceptions.RequestException as e: + return {'error': str(e)} diff --git a/cmapi/cmapi_server/controllers/endpoints.py b/cmapi/cmapi_server/controllers/endpoints.py index bf3b009c1..6f48d8f3d 100644 --- a/cmapi/cmapi_server/controllers/endpoints.py +++ b/cmapi/cmapi_server/controllers/endpoints.py @@ -797,9 +797,10 @@ class ClusterController: request = cherrypy.request request_body = request.json config = request_body.get('config', DEFAULT_MCS_CONF_PATH) + in_transaction = request_body.get('in_transaction', False) try: - response = ClusterHandler.start(config) + response = ClusterHandler.start(config, in_transaction) except CMAPIBasicError as err: raise_422_error(module_logger, func_name, err.message) @@ -817,9 +818,10 @@ class ClusterController: request = cherrypy.request request_body = request.json config = request_body.get('config', DEFAULT_MCS_CONF_PATH) + in_transaction = request_body.get('in_transaction', False) try: - response = ClusterHandler.shutdown(config) + response = ClusterHandler.shutdown(config, in_transaction) except CMAPIBasicError as err: raise_422_error(module_logger, func_name, err.message) diff --git a/cmapi/cmapi_server/handlers/cluster.py b/cmapi/cmapi_server/handlers/cluster.py index 628c3da6b..1e8192a36 100644 --- a/cmapi/cmapi_server/handlers/cluster.py +++ b/cmapi/cmapi_server/handlers/cluster.py @@ -1,6 +1,7 @@ """Module contains Cluster business logic functions.""" import logging from datetime import datetime +from enum import Enum from typing import Optional import requests @@ -22,26 +23,61 @@ from mcs_node_control.models.misc import get_dbrm_master from mcs_node_control.models.node_config import NodeConfig +class ClusterAction(Enum): + START = 'start' + STOP = 'stop' + + +def toggle_cluster_state(action: ClusterAction, config: str) -> dict: + """ + Toggle the state of the cluster (start or stop). + + :param action: The cluster action to perform. + (ClusterAction.START or ClusterAction.STOP). + :type action: ClusterAction + :param config: The path to the MariaDB Columnstore configuration file. + :type config: str + """ + if action == ClusterAction.START: + maintainance_flag = True + elif action == ClusterAction.STOP: + maintainance_flag = True + else: + raise ValueError( + 'Invalid action. Use ClusterAction.START or ClusterAction.STOP.' + ) + + switch_node_maintenance(maintainance_flag) + update_revision_and_manager() + + # TODO: move this from multiple places to one + try: + broadcast_successful = broadcast_new_config(config) + except Exception as err: + raise CMAPIBasicError( + 'Error while distributing config file.' + ) from err + + if not broadcast_successful: + raise CMAPIBasicError('Config distribution isn\'t successful.') + + class ClusterHandler(): """Class for handling MCS Cluster operations.""" @staticmethod - def status( - config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server') - ) -> dict: + def status(config: str = DEFAULT_MCS_CONF_PATH) -> dict: """Method to get MCS Cluster status information :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional :raises CMAPIBasicError: if catch some exception while getting status from each node separately :return: status result :rtype: dict """ + logger: logging.Logger = logging.getLogger('cmapi_server') logger.debug('Cluster status command called. Getting status.') response = {'timestamp': str(datetime.now())} @@ -73,78 +109,36 @@ class ClusterHandler(): @staticmethod def start( - config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server') + config: str = DEFAULT_MCS_CONF_PATH, in_transaction: bool = False ) -> dict: """Method to start MCS Cluster. :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional - :raises CMAPIBasicError: on exception while starting transaction - :raises CMAPIBasicError: if transaction start isn't successful + :param in_transaction: is function called in existing transaction or no + If we started transaction in cli tool than we + don't need to handle it here again + :type in_transaction: bool :raises CMAPIBasicError: if no nodes in the cluster - :raises CMAPIBasicError: on exception while distributing new config - :raises CMAPIBasicError: on unsuccessful distibuting config file - :raises CMAPIBasicError: on exception while committing transaction :return: start timestamp :rtype: dict """ - logger.debug('Cluster start command called. Starting the cluster.') - start_time = str(datetime.now()) - transaction_id = get_id() + logger: logging.Logger = logging.getLogger('cmapi_server') + logger.info('Cluster start command called. Starting the cluster.') + operation_start_time = str(datetime.now()) + if not in_transaction: + with TransactionManager(): + toggle_cluster_state(ClusterAction.START, config) + else: + toggle_cluster_state(ClusterAction.START, config) - try: - suceeded, transaction_id, successes = start_transaction( - cs_config_filename=config, txn_id=transaction_id - ) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while starting the transaction.' - ) from err - if not suceeded: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError('Starting transaction isn\'t successful.') - - if suceeded and len(successes) == 0: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError('There are no nodes in the cluster.') - - switch_node_maintenance(False) - update_revision_and_manager() - - # TODO: move this from multiple places to one, eg to helpers - try: - broadcast_successful = broadcast_new_config(config) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while distributing config file.' - ) from err - - if not broadcast_successful: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError('Config distribution isn\'t successful.') - - try: - commit_transaction(transaction_id, cs_config_filename=config) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while committing transaction.' - ) from err - - logger.debug('Successfully finished cluster start.') - return {'timestamp': start_time} + logger.info('Successfully finished cluster start.') + return {'timestamp': operation_start_time} @staticmethod def shutdown( - config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server'), - in_transaction: bool = False, + config: str = DEFAULT_MCS_CONF_PATH, in_transaction: bool = False, timeout: int = 15 ) -> dict: """Method to stop the MCS Cluster. @@ -152,8 +146,6 @@ class ClusterHandler(): :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional :param in_transaction: is function called in existing transaction or no :type in_transaction: bool :param timeout: timeout in seconds to gracefully stop DMLProc @@ -163,35 +155,19 @@ class ClusterHandler(): :return: start timestamp :rtype: dict """ + logger: logging.Logger = logging.getLogger('cmapi_server') logger.debug( 'Cluster shutdown command called. Shutting down the cluster.' ) - - def process_shutdown(): - """Raw node shutdown processing.""" - switch_node_maintenance(True) - update_revision_and_manager() - - # TODO: move this from multiple places to one, eg to helpers - try: - broadcast_successful = broadcast_new_config(config) - except Exception as err: - raise CMAPIBasicError( - 'Error while distributing config file.' - ) from err - - if not broadcast_successful: - raise CMAPIBasicError('Config distribution isn\'t successful.') - - start_time = str(datetime.now()) + operation_start_time = str(datetime.now()) if not in_transaction: with TransactionManager(): - process_shutdown() + toggle_cluster_state(ClusterAction.STOP, config) else: - process_shutdown() + toggle_cluster_state(ClusterAction.STOP, config) logger.debug('Successfully finished shutting down the cluster.') - return {'timestamp': start_time} + return {'timestamp': operation_start_time} @staticmethod def add_node( diff --git a/cmapi/cmapi_server/helpers.py b/cmapi/cmapi_server/helpers.py index ea2b61726..9348393ee 100644 --- a/cmapi/cmapi_server/helpers.py +++ b/cmapi/cmapi_server/helpers.py @@ -314,6 +314,7 @@ def broadcast_new_config( :rtype: bool """ + # TODO: move this from multiple places to one, eg to helpers cfg_parser = get_config_parser(cmapi_config_filename) key = get_current_key(cfg_parser) version = get_version() diff --git a/cmapi/cmapi_server/managers/transaction.py b/cmapi/cmapi_server/managers/transaction.py index 10db998df..cffb71386 100644 --- a/cmapi/cmapi_server/managers/transaction.py +++ b/cmapi/cmapi_server/managers/transaction.py @@ -6,7 +6,7 @@ from signal import ( ) from typing import Optional, Type -from cmapi_server.constants import DEFAULT_MCS_CONF_PATH +from cmapi_server.constants import DEFAULT_MCS_CONF_PATH, TRANSACTION_TIMEOUT from cmapi_server.exceptions import CMAPIBasicError from cmapi_server.helpers import ( get_id, commit_transaction, rollback_transaction, start_transaction @@ -17,7 +17,7 @@ class TransactionManager(ContextDecorator): """Context manager and decorator to put any code inside CMAPI transaction. :param timeout: time in sec after transaction will be autocommitted, - defaults to 300.0 + defaults to 300.0 (TRANSACTION_TIMEOUT) :param timeout: _description_, defaults to 300 :type timeout: float, optional @@ -28,8 +28,8 @@ class TransactionManager(ContextDecorator): """ def __init__( - self, timeout: float = 300, txn_id: Optional[int] = None, - handle_signals: bool = False + self, timeout: float = TRANSACTION_TIMEOUT, + txn_id: Optional[int] = None, handle_signals: bool = False ): self.timeout = timeout self.txn_id = txn_id or get_id() diff --git a/cmapi/mcs_cluster_tool/cluster_app.py b/cmapi/mcs_cluster_tool/cluster_app.py index 67602abe9..f911057f2 100644 --- a/cmapi/mcs_cluster_tool/cluster_app.py +++ b/cmapi/mcs_cluster_tool/cluster_app.py @@ -13,16 +13,16 @@ import typer from typing_extensions import Annotated from cmapi_server.constants import ( - CMAPI_CONF_PATH, DEFAULT_MCS_CONF_PATH, SECRET_KEY + CMAPI_CONF_PATH, DEFAULT_MCS_CONF_PATH, REQUEST_TIMEOUT ) from cmapi_server.exceptions import CMAPIBasicError -from cmapi_server.handlers.cluster import ClusterHandler from cmapi_server.helpers import ( get_config_parser, get_current_key, get_version, build_url ) from cmapi_server.managers.transaction import TransactionManager from mcs_cluster_tool.decorators import handle_output from mcs_node_control.models.node_config import NodeConfig +from cmapi.cmapi_server.controllers.api_clients import ClusterControllerClient logger = logging.getLogger('mcs_cli') @@ -33,13 +33,15 @@ node_app = typer.Typer(help='Cluster nodes management.') app.add_typer(node_app, name='node') set_app = typer.Typer(help='Set cluster parameters.') app.add_typer(set_app, name='set') +client = ClusterControllerClient() @app.command(rich_help_panel='cluster and single node commands') @handle_output def status(): """Get status information.""" - return ClusterHandler.status(logger=logger) + client.request_timeout = REQUEST_TIMEOUT + return client.get_status() @app.command(rich_help_panel='cluster and single node commands') @@ -157,25 +159,29 @@ def stop( # TODO: investigate more on how changing the hardcoded timeout # could affect put_config (helpers.py broadcast_config) operation timeout = 0 - _ = ClusterHandler.shutdown(logger=logger, in_transaction=True) + + resp = client.shutdown_cluster({'in_transaction': True}) return {'timestamp': start_time} @app.command(rich_help_panel='cluster and single node commands') @handle_output +@TransactionManager( + timeout=timedelta(days=1).total_seconds(), handle_signals=True +) def start(): """Start the Columnstore cluster.""" - return ClusterHandler.start(logger=logger) + return client.start_cluster({'in_transaction': True}) @app.command(rich_help_panel='cluster and single node commands') @handle_output def restart(): """Restart the Columnstore cluster.""" - stop_result = ClusterHandler.shutdown(logger=logger) + stop_result = client.shutdown_cluster() if 'error' in stop_result: return stop_result - result = ClusterHandler.start(logger=logger) + result = client.start_cluster() result['stop_timestamp'] = stop_result['timestamp'] return result @@ -195,7 +201,7 @@ def add( """Add nodes to the Columnstore cluster.""" result = [] for node in nodes: - result.append(ClusterHandler.add_node(node, logger=logger)) + result.append(client.add_node({'node': node})) return result @@ -213,7 +219,7 @@ def remove(nodes: Optional[List[str]] = typer.Option( """Remove nodes from the Columnstore cluster.""" result = [] for node in nodes: - result.append(ClusterHandler.remove_node(node, logger=logger)) + result.append(client.remove_node(node)) return result @@ -233,7 +239,8 @@ def mode(cluster_mode: str = typer.Option( raise typer.BadParameter( '"readonly" or "readwrite" are the only acceptable modes now.' ) - return ClusterHandler.set_mode(cluster_mode, logger=logger) + client.request_timeout = REQUEST_TIMEOUT + return client.set_mode(cluster_mode) @set_app.command() @@ -245,10 +252,8 @@ def api_key(key: str = typer.Option(..., help='API key to set.')): """ if not key: raise typer.BadParameter('Empty API key not allowed.') - - totp = pyotp.TOTP(SECRET_KEY) - - return ClusterHandler.set_api_key(key, totp.now(), logger=logger) + client.request_timeout = REQUEST_TIMEOUT + return client.set_api_key(key) @set_app.command() @@ -260,5 +265,5 @@ def log_level(level: str = typer.Option(..., help='Logging level to set.')): """ if not level: raise typer.BadParameter('Empty log level not allowed.') - - return ClusterHandler.set_log_level(level, logger=logger) + client.request_timeout = REQUEST_TIMEOUT + return client.set_log_level(level) From af10cb15262d9fa0970f78cddd16ffdf3eba65cb Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Fri, 24 Jan 2025 15:54:59 +0300 Subject: [PATCH 53/65] feat(cmapi): MCOL-5133: Stage2 stand alone cli tool. [fix] client class methods [fix] Transaction management at endpoint handling level [fix] add_node and set_mode methods to use TransactionManager --- cmapi/cmapi_server/controllers/api_clients.py | 20 ++-- cmapi/cmapi_server/controllers/endpoints.py | 32 +++++-- cmapi/cmapi_server/handlers/cluster.py | 93 ++----------------- cmapi/mcs_cluster_tool/cluster_app.py | 13 ++- 4 files changed, 58 insertions(+), 100 deletions(-) diff --git a/cmapi/cmapi_server/controllers/api_clients.py b/cmapi/cmapi_server/controllers/api_clients.py index 424d9d99c..c03d0b263 100644 --- a/cmapi/cmapi_server/controllers/api_clients.py +++ b/cmapi/cmapi_server/controllers/api_clients.py @@ -21,43 +21,45 @@ class ClusterControllerClient: self.request_timeout = request_timeout def start_cluster( - self, data: Optional[Dict[str, Any]] = None + self, extra: Dict[str, Any] = dict() ) -> Union[Dict[str, Any], Dict[str, str]]: """Start the cluster. :return: The response from the API. """ - return self._request('PUT', 'start', data) + return self._request('PUT', 'start', extra) def shutdown_cluster( - self, data: Optional[Dict[str, Any]] = None + self, extra: Dict[str, Any] = dict() ) -> Union[Dict[str, Any], Dict[str, str]]: """Shutdown the cluster. :return: The response from the API. """ - return self._request('PUT', 'shutdown', data) + return self._request('PUT', 'shutdown', extra) - def set_mode(self, mode: str) -> Union[Dict[str, Any], Dict[str, str]]: + def set_mode( + self, mode: str, extra: Dict[str, Any] = dict() + ) -> Union[Dict[str, Any], Dict[str, str]]: """Set the cluster mode. :param mode: The mode to set. :return: The response from the API. """ - return self._request('PUT', 'mode-set', {'mode': mode}) + return self._request('PUT', 'mode-set', {'mode': mode, **extra}) def add_node( - self, node_info: Dict[str, Any] + self, node_info: Dict[str, Any], extra: Dict[str, Any] = dict() ) -> Union[Dict[str, Any], Dict[str, str]]: """Add a node to the cluster. :param node_info: Information about the node to add. :return: The response from the API. """ - return self._request('PUT', 'node', node_info) + return self._request('PUT', 'node', {**node_info, **extra}) def remove_node( - self, node_id: str + self, node_id: str, extra: Dict[str, Any] = dict() ) -> Union[Dict[str, Any], Dict[str, str]]: """Remove a node from the cluster. diff --git a/cmapi/cmapi_server/controllers/endpoints.py b/cmapi/cmapi_server/controllers/endpoints.py index 6f48d8f3d..53adcfc9c 100644 --- a/cmapi/cmapi_server/controllers/endpoints.py +++ b/cmapi/cmapi_server/controllers/endpoints.py @@ -25,8 +25,9 @@ from cmapi_server.helpers import ( system_ready, save_cmapi_conf_file, dequote, in_maintenance_state, ) from cmapi_server.logging_management import change_loggers_level -from cmapi_server.managers.process import MCSProcessManager from cmapi_server.managers.application import AppManager +from cmapi_server.managers.process import MCSProcessManager +from cmapi_server.managers.transaction import TransactionManager from cmapi_server.node_manipulation import is_master, switch_node_maintenance from mcs_node_control.models.dbrm import set_cluster_mode from mcs_node_control.models.node_config import NodeConfig @@ -800,7 +801,11 @@ class ClusterController: in_transaction = request_body.get('in_transaction', False) try: - response = ClusterHandler.start(config, in_transaction) + if not in_transaction: + with TransactionManager(): + response = ClusterHandler.start(config) + else: + response = ClusterHandler.start(config) except CMAPIBasicError as err: raise_422_error(module_logger, func_name, err.message) @@ -821,7 +826,11 @@ class ClusterController: in_transaction = request_body.get('in_transaction', False) try: - response = ClusterHandler.shutdown(config, in_transaction) + if not in_transaction: + with TransactionManager(): + response = ClusterHandler.shutdown(config) + else: + response = ClusterHandler.shutdown(config) except CMAPIBasicError as err: raise_422_error(module_logger, func_name, err.message) @@ -840,9 +849,14 @@ class ClusterController: request_body = request.json mode = request_body.get('mode', 'readonly') config = request_body.get('config', DEFAULT_MCS_CONF_PATH) + in_transaction = request_body.get('in_transaction', False) try: - response = ClusterHandler.set_mode(mode, config=config) + if not in_transaction: + with TransactionManager(): + response = ClusterHandler.set_mode(mode, config=config) + else: + response = ClusterHandler.set_mode(mode, config=config) except CMAPIBasicError as err: raise_422_error(module_logger, func_name, err.message) @@ -861,12 +875,17 @@ class ClusterController: request_body = request.json node = request_body.get('node', None) config = request_body.get('config', DEFAULT_MCS_CONF_PATH) + in_transaction = request_body.get('in_transaction', False) if node is None: raise_422_error(module_logger, func_name, 'missing node argument') try: - response = ClusterHandler.add_node(node, config) + if not in_transaction: + with TransactionManager(): + response = ClusterHandler.add_node(node, config) + else: + response = ClusterHandler.add_node(node, config) except CMAPIBasicError as err: raise_422_error(module_logger, func_name, err.message) @@ -884,7 +903,8 @@ class ClusterController: request_body = request.json node = request_body.get('node', None) config = request_body.get('config', DEFAULT_MCS_CONF_PATH) - response = {'timestamp': str(datetime.now())} + #TODO: for next release + in_transaction = request_body.get('in_transaction', False) #TODO: add arguments verification decorator if node is None: diff --git a/cmapi/cmapi_server/handlers/cluster.py b/cmapi/cmapi_server/handlers/cluster.py index 1e8192a36..88c381d07 100644 --- a/cmapi/cmapi_server/handlers/cluster.py +++ b/cmapi/cmapi_server/handlers/cluster.py @@ -108,18 +108,12 @@ class ClusterHandler(): return response @staticmethod - def start( - config: str = DEFAULT_MCS_CONF_PATH, in_transaction: bool = False - ) -> dict: + def start(config: str = DEFAULT_MCS_CONF_PATH) -> dict: """Method to start MCS Cluster. :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param in_transaction: is function called in existing transaction or no - If we started transaction in cli tool than we - don't need to handle it here again - :type in_transaction: bool :raises CMAPIBasicError: if no nodes in the cluster :return: start timestamp :rtype: dict @@ -127,27 +121,19 @@ class ClusterHandler(): logger: logging.Logger = logging.getLogger('cmapi_server') logger.info('Cluster start command called. Starting the cluster.') operation_start_time = str(datetime.now()) - if not in_transaction: - with TransactionManager(): - toggle_cluster_state(ClusterAction.START, config) - else: - toggle_cluster_state(ClusterAction.START, config) - + toggle_cluster_state(ClusterAction.START, config) logger.info('Successfully finished cluster start.') return {'timestamp': operation_start_time} @staticmethod def shutdown( - config: str = DEFAULT_MCS_CONF_PATH, in_transaction: bool = False, - timeout: int = 15 + config: str = DEFAULT_MCS_CONF_PATH, timeout: int = 15 ) -> dict: """Method to stop the MCS Cluster. :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param in_transaction: is function called in existing transaction or no - :type in_transaction: bool :param timeout: timeout in seconds to gracefully stop DMLProc TODO: for next releases :type timeout: int @@ -160,20 +146,12 @@ class ClusterHandler(): 'Cluster shutdown command called. Shutting down the cluster.' ) operation_start_time = str(datetime.now()) - if not in_transaction: - with TransactionManager(): - toggle_cluster_state(ClusterAction.STOP, config) - else: - toggle_cluster_state(ClusterAction.STOP, config) - + toggle_cluster_state(ClusterAction.STOP, config) logger.debug('Successfully finished shutting down the cluster.') return {'timestamp': operation_start_time} @staticmethod - def add_node( - node: str, config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server') - ) -> dict: + def add_node(node: str, config: str = DEFAULT_MCS_CONF_PATH) -> dict: """Method to add node to MCS CLuster. :param node: node IP or name or FQDN @@ -181,8 +159,6 @@ class ClusterHandler(): :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional :raises CMAPIBasicError: on exception while starting transaction :raises CMAPIBasicError: if transaction start isn't successful :raises CMAPIBasicError: on exception while adding node @@ -192,24 +168,10 @@ class ClusterHandler(): :return: result of adding node :rtype: dict """ + logger: logging.Logger = logging.getLogger('cmapi_server') logger.debug(f'Cluster add node command called. Adding node {node}.') response = {'timestamp': str(datetime.now())} - transaction_id = get_id() - - try: - suceeded, transaction_id, successes = start_transaction( - cs_config_filename=config, extra_nodes=[node], - txn_id=transaction_id - ) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while starting the transaction.' - ) from err - if not suceeded: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError('Starting transaction isn\'t successful.') try: add_node( @@ -222,7 +184,6 @@ class ClusterHandler(): output_config_filename=config ) except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) raise CMAPIBasicError('Error while adding node.') from err response['node_id'] = node @@ -233,31 +194,18 @@ class ClusterHandler(): try: broadcast_successful = broadcast_new_config(config) except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) raise CMAPIBasicError( 'Error while distributing config file.' ) from err if not broadcast_successful: - rollback_transaction(transaction_id, cs_config_filename=config) raise CMAPIBasicError('Config distribution isn\'t successful.') - try: - commit_transaction(transaction_id, cs_config_filename=config) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while committing transaction.' - ) from err - logger.debug(f'Successfully finished adding node {node}.') return response @staticmethod - def remove_node( - node: str, config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server') - ) -> dict: + def remove_node(node: str, config: str = DEFAULT_MCS_CONF_PATH) -> dict: """Method to remove node from MCS CLuster. :param node: node IP or name or FQDN @@ -265,8 +213,6 @@ class ClusterHandler(): :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional :raises CMAPIBasicError: on exception while starting transaction :raises CMAPIBasicError: if transaction start isn't successful :raises CMAPIBasicError: on exception while removing node @@ -276,6 +222,9 @@ class ClusterHandler(): :return: result of node removing :rtype: dict """ + #TODO: This method will be moved to transaction manager in next release + # Due to specific use of txn_nodes inside. + logger: logging.Logger = logging.getLogger('cmapi_server') logger.debug( f'Cluster remove node command called. Removing node {node}.' ) @@ -387,19 +336,6 @@ class ClusterHandler(): payload = {'cluster_mode': mode} url = f'https://{master}:8640/cmapi/{get_version()}/node/config' - try: - suceeded, transaction_id, successes = start_transaction( - cs_config_filename=config, txn_id=transaction_id - ) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while starting the transaction.' - ) from err - if not suceeded: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError('Starting transaction isn\'t successful.') - nc = NodeConfig() root = nc.get_current_config_root(config_filename=config) payload['manager'] = root.find('./ClusterManager').text @@ -412,19 +348,10 @@ class ClusterHandler(): r.raise_for_status() response['cluster-mode'] = mode except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) raise CMAPIBasicError( f'Error while setting cluster mode to {mode}' ) from err - try: - commit_transaction(transaction_id, cs_config_filename=config) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while committing transaction.' - ) from err - logger.debug(f'Successfully set cluster mode to {mode}.') return response diff --git a/cmapi/mcs_cluster_tool/cluster_app.py b/cmapi/mcs_cluster_tool/cluster_app.py index f911057f2..0b23ff021 100644 --- a/cmapi/mcs_cluster_tool/cluster_app.py +++ b/cmapi/mcs_cluster_tool/cluster_app.py @@ -176,18 +176,24 @@ def start(): @app.command(rich_help_panel='cluster and single node commands') @handle_output +@TransactionManager( + timeout=timedelta(days=1).total_seconds(), handle_signals=True +) def restart(): """Restart the Columnstore cluster.""" - stop_result = client.shutdown_cluster() + stop_result = client.shutdown_cluster({'in_transaction': True}) if 'error' in stop_result: return stop_result - result = client.start_cluster() + result = client.start_cluster({'in_transaction': True}) result['stop_timestamp'] = stop_result['timestamp'] return result @node_app.command(rich_help_panel='cluster node commands') @handle_output +@TransactionManager( + timeout=timedelta(days=1).total_seconds(), handle_signals=True +) def add( nodes: Optional[List[str]] = typer.Option( ..., @@ -225,6 +231,9 @@ def remove(nodes: Optional[List[str]] = typer.Option( @set_app.command() @handle_output +@TransactionManager( + timeout=timedelta(days=1).total_seconds(), handle_signals=True +) def mode(cluster_mode: str = typer.Option( ..., '--mode', From d132c1174a2f08154304700c132d694405724cc2 Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Fri, 7 Mar 2025 15:56:56 +0300 Subject: [PATCH 54/65] MCOL-5618: Sync with PR3410. [add] list option but in argument notation "list" for backup, restore, dbrm_restore [fix] help messages for --config-file [fix] -p/--path > -bl/--backup-location option for dbrm_backup and dbrm_restore [fix] -d/--directory > '-l', '--load' option for dbrm_restore --- cmapi/mcs_cluster_tool/backup_commands.py | 32 +++++++++++++------ cmapi/mcs_cluster_tool/restore_commands.py | 37 ++++++++++++++++------ 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/cmapi/mcs_cluster_tool/backup_commands.py b/cmapi/mcs_cluster_tool/backup_commands.py index c0679ded5..f9496dd94 100644 --- a/cmapi/mcs_cluster_tool/backup_commands.py +++ b/cmapi/mcs_cluster_tool/backup_commands.py @@ -102,7 +102,7 @@ def backup( 'Adds columnstore deltas to an existing full backup. ' 'Backup folder to apply increment could be a value or ' '"auto_most_recent" - the incremental backup applies to ' - 'last full backup.' + 'last full backup.' ), show_default=False ) @@ -123,7 +123,10 @@ def backup( str, typer.Option( '-f', '--config-file', - help='Path to backup configuration file to load variables from.', + help=( + 'Path to backup configuration file to load variables from - ' + 'relative or full path accepted.' + ), show_default=False ) ] = '', @@ -245,6 +248,13 @@ def backup( ) ) ] = 0, + list: Annotated[ + bool, + typer.Option( + 'list', + help='List backups.' + ) + ] = False ): """Backup Columnstore and/or MariDB data.""" @@ -303,10 +313,10 @@ def dbrm_backup( ) ) ] = 7, - p: Annotated[ + bl: Annotated[ str, typer.Option( - '-p', '--path', + '-bl', '--backup-location', help='Path of where to save the dbrm backups on disk.' ) ] = '/tmp/dbrm_backups', @@ -314,7 +324,10 @@ def dbrm_backup( str, typer.Option( '-nb', '--name-backup', - help='Custom name to prefex dbrm backups with.' + help=( + 'Define the prefix of the backup - ' + 'default: dbrm_backup+date +%Y%m%d_%H%M%S' + ) ) ] = 'dbrm_backup', q: Annotated[ @@ -334,14 +347,15 @@ def dbrm_backup( ): """Columnstore DBRM Backup.""" - # Default: ./$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups + # Default: ./$0 dbrm_backup -m once --retention-days 0 --backup-location /tmp/dbrm_backups # Examples: - # ./$0 dbrm_backup --mode loop --interval 90 --retention-days 7 --path /mnt/dbrm_backups - # ./$0 dbrm_backup --mode once --retention-days 7 --path /mnt/dbrm_backups -nb my-one-off-backup + # ./$0 dbrm_backup --backup-location /mnt/columnstore/dbrm_backups + # ./$0 dbrm_backup --retention-days 7 --backup-location /mnt/dbrm_backups --mode once -nb my-one-off-backup-before-upgrade + # ./$0 dbrm_backup --retention-days 7 --backup-location /mnt/dbrm_backups --mode loop --interval 90brm_backup --mode once --retention-days 7 --path /mnt/dbrm_backups -nb my-one-off-backup # Cron Example: - # */60 */3 * * * root bash /root/$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups >> /tmp/dbrm_backups/cs_backup.log 2>&1 + # */60 */3 * * * root bash /root/$0 dbrm_backup -m once --retention-days 7 --backup-location /tmp/dbrm_backups >> /tmp/dbrm_backups/cs_backup.log 2>&1 arguments = [] for arg_name, value in locals().items(): sh_arg = cook_sh_arg(arg_name, value) diff --git a/cmapi/mcs_cluster_tool/restore_commands.py b/cmapi/mcs_cluster_tool/restore_commands.py index e4881f917..ba0ad3533 100644 --- a/cmapi/mcs_cluster_tool/restore_commands.py +++ b/cmapi/mcs_cluster_tool/restore_commands.py @@ -170,7 +170,10 @@ def restore( str, typer.Option( '-f', '--config-file', - help='Path to backup configuration file to load variables from.', + help=( + 'Path to backup configuration file to load variables from - ' + 'relative or full path accepted.' + ), show_default=False ) ] = '', @@ -242,6 +245,13 @@ def restore( help='Skips verifying ssl certs, useful for onpremise s3 storage.' ) ] = False, + list: Annotated[ + bool, + typer.Option( + 'list', + help='List backups.' + ) + ] = False ): """Restore Columnstore (and/or MariaDB) data.""" @@ -266,18 +276,18 @@ def restore( @handle_output def dbrm_restore( - p: Annotated[ + bl: Annotated[ str, typer.Option( - '-p', '--path', - help='Path of where dbrm backups stored on disk.' + '-bl', '--backup-location', + help='Path of where dbrm backups exist on disk.' ) ] = '/tmp/dbrm_backups', - d: Annotated[ + l: Annotated[ str, typer.Option( - '-d', '--directory', - help='Date or directory chose to restore from.' + '-l', '--load', + help='Name of the directory to restore from -bl' ) ] = '', ns: Annotated[ @@ -305,14 +315,21 @@ def dbrm_restore( help='Skip backing up storagemanager directory.' ) ] = True, + list: Annotated[ + bool, + typer.Option( + 'list', + help='List backups.' + ) + ] = False ): """Restore Columnstore DBRM data.""" - # Default: ./$0 dbrm_restore --path /tmp/dbrm_backups + # Default: ./$0 dbrm_restore --backup-location /tmp/dbrm_backups # Examples: - # ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 - # ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 --no-start + # ./$0 dbrm_restore --backup-location /tmp/dbrm_backups --load dbrm_backup_20240318_172842 + # ./$0 dbrm_restore --backup-location /tmp/dbrm_backups --load dbrm_backup_20240318_172842 --no-startdbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 --no-start arguments = [] for arg_name, value in locals().items(): sh_arg = cook_sh_arg(arg_name, value) From 001367e68ae92d766affcf8f4b26f47bca9d34ea Mon Sep 17 00:00:00 2001 From: Allen Herrera <82840027+mariadb-AllenHerrera@users.noreply.github.com> Date: Fri, 14 Mar 2025 07:36:01 -0400 Subject: [PATCH 55/65] feat(backup): added timestamps to localstorage backups major sections, new flag backup --apply-retention-only , more verbose logging in poll_check_no_active_sql_writes(), fixes to multinode race conditions to apply apply_backup_retention_policy() (#3422) --- cmapi/scripts/mcs_backup_manager.sh | 471 ++++++++++++++++++++++------ 1 file changed, 381 insertions(+), 90 deletions(-) diff --git a/cmapi/scripts/mcs_backup_manager.sh b/cmapi/scripts/mcs_backup_manager.sh index 59355aff5..af1a1f2c0 100644 --- a/cmapi/scripts/mcs_backup_manager.sh +++ b/cmapi/scripts/mcs_backup_manager.sh @@ -13,7 +13,7 @@ # ######################################################################## # Documentation: bash mcs_backup_manager.sh help -# Version: 3.11 +# Version: 3.12 # # Backup Example # LocalStorage: sudo ./mcs_backup_manager.sh backup @@ -26,7 +26,7 @@ # S3: sudo ./mcs_backup_manager.sh restore -bb s3://my-cs-backups -l # ######################################################################## -mcs_bk_manager_version="3.11" +mcs_bk_manager_version="3.12" start=$(date +%s) action=$1 @@ -171,6 +171,8 @@ load_default_backup_variables() { poll_interval=5 poll_max_wait=60; list_backups=false + retention_file_lock_name=".delete-lock" + apply_retention_only=false # Compression Variables compress_format="" @@ -322,6 +324,10 @@ parse_backup_variables() { shift # past argument shift # past value ;; + -aro| --apply-retention-only) + apply_retention_only=true + shift # past argument + ;; "list") list_backups=true shift # past argument @@ -371,14 +377,15 @@ print_backup_help_text() { -c | --compress Compress backup in X format - Options: [ pigz ] -nb | --name-backup Define the name of the backup - default: date +%m-%d-%Y -r | --retention-days Retain backups created within the last X days, the rest are deleted, default 0 = keep all backups + -aro | --apply-retention-only Only apply retention policy to existing backups, does not run a backup -ha | --highavilability Hint wether shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) HA S3 ( /var/lib/columnstore/storagemanager/ ) Local Storage Examples: - ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage - ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage -P 8 - ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage --incremental auto_most_recent + ./$0 backup -bl /tmp/backups/ + ./$0 backup -bl /tmp/backups/ -P 8 + ./$0 backup -bl /tmp/backups/ --incremental auto_most_recent ./$0 backup -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -s LocalStorage S3 Examples: @@ -666,7 +673,7 @@ validation_prechecks_for_backup() { printf " - Columnstore is OFFLINE \n"; export columnstore_online=false; else - printf " - Columnstore is ONLINE - safer if offline \n"; + printf " - Columnstore is ONLINE - safer if offline (but not required to be offline) \n"; export columnstore_online=true; fi fi; @@ -796,7 +803,26 @@ auto_select_most_recent_backup_for_incremental() { printf " - Searching for most recent backup ...." if [ $storage == "LocalStorage" ]; then - most_recent_backup=$(ls -td "${backup_location}"* 2>/dev/null | head -n 1) + + # Example: find /mnt/backups/ -mindepth 1 -maxdepth 1 -type d ! -exec test -f "{}/.auto-delete-via-retention" \; -printf "%T@ %p\n" | sort -nr | awk '{print $2}' | head -n 1 + if [ -f "${backup_location}${retention_file_lock_name}-$pm" ] ; then + handle_early_exit_on_backup "\n[!!!] ${backup_location}${retention_file_lock_name}-$pm exists. are multiple backups running or was there an error? Manually resolve before retrying\n" true + fi + + # Primary node is in the middle of applying retention policy, distorting the most recent backup + if [ -f "${backup_location}${retention_file_lock_name}-pm1" ] ; then + most_recent_backup="$(head -n1 "${backup_location}${retention_file_lock_name}-pm1" | awk '{print $2}')" + else + most_recent_backup=$(find "$backup_location" -mindepth 1 -maxdepth 1 -type d -printf "%T@ %p\n" | sort -nr | awk '{print $2}' | head -n 1) + fi + + # Debug messaging to understand the most recent backup modified times + if ! $quiet; then + echo "" + find "$backup_location" -mindepth 1 -maxdepth 1 -type d -printf "%T@ %p\n" | sort -nr + fi + + # If no backup found, exit if [[ -z "$most_recent_backup" ]]; then handle_early_exit_on_backup "\n[!!!] No backup found to increment in '$backup_location', please run a full backup or define a folder that exists --incremental \n" true else @@ -849,55 +875,124 @@ auto_select_most_recent_backup_for_incremental() { apply_backup_retention_policy() { - if [ $retention_days -eq 0 ]; then + if [ "$retention_days" -eq 0 ]; then printf " - Skipping Backup Rentention Policy\n" return 0; fi - printf " - Applying Backup Rentention Policy...." - if [ $storage == "LocalStorage" ]; then - # example: find /tmp/backups/ -mindepth 1 -maxdepth 1 -type d -name "*" -amin +0 - find "$backup_location" -mindepth 1 -maxdepth 1 -type d -name "*" -mtime +$retention_days -exec rm -r {} \; + # PM1 is in charge of deleting the backup + # Rest of the nodes should wait for the delete to be over if it finds the lock file + applying_retention_start=$(date +%s) + printf "\nApplying Backup Rentention Policy\n" + if [ "$pm" == "pm1" ] || [ -n "${PM_FORCE_DELETE-}" ] ; then + + retention_file_lock_name="${retention_file_lock_name}-$pm" - elif [ $storage == "S3" ]; then + if [ $storage == "LocalStorage" ]; then + + # Create a lock file to prevent other PMs from deleting backups + touch "${backup_location}${retention_file_lock_name}" + echo "most_recent_backup: $most_recent_backup" >> "${backup_location}${retention_file_lock_name}" + printf " - Created: ${backup_location}${retention_file_lock_name}\n" - current_date=$(date +%s) - backups=$(s3ls $backup_bucket) - - while IFS= read -r line; do + # example1: find /mnt/backups/ -mindepth 1 -maxdepth 1 -type d -name "*" -mmin +10 -exec echo "{}" \; + # example2: find /mnt/backups/ -mindepth 1 -maxdepth 1 -type d -name "*" -mmin +10 -exec sh -c 'printf " - Deleting: {} \n" && echo "{}" >> "/mnt/backups/.delete-lock-pm1" && rm -rf "{}"' \; + # find "$backup_location" -mindepth 1 -maxdepth 1 -type d -name "*" -mmin +$retention_days -exec sh -c 'printf " - Deleting: {} \n" && echo "{}" >> "'"${backup_location}${retention_file_lock_name}"'" && rm -rf "{}"' \; + find "$backup_location" -mindepth 1 -maxdepth 1 -type d -name "*" -mtime +$retention_days -exec sh -c 'printf " - Deleting: {} \n" && echo "{}" >> "'"${backup_location}${retention_file_lock_name}"'" && rm -rf "{}"' \; + + # Cleanup the lock file + rm "${backup_location}${retention_file_lock_name}" + printf " - Deleted: ${backup_location}${retention_file_lock_name}\n" - delete_backup=false - folder=$(echo "$line" | awk '{print substr($2, 1, length($2)-1)}') - date_time=$(s3ls "${backup_bucket}/${folder}/restore --recursive" | awk '{print $1,$2}') - - if [[ -n "$date_time" ]]; then + elif [ $storage == "S3" ]; then - # Parse the date - backup_date=$(date -d "$date_time" +%s) - - # Calculate the difference in days - days_diff=$(( (current_date - backup_date) / (60*60*24) )) - # echo "line: $line" - # echo "date_time: $date_time" - # echo "backup_date: $backup_date" - # echo "days_diff: $days_diff" + # Create a lock file to prevent other PMs from deleting backups + echo "most_recent_backup: $most_recent_backup" >> "${retention_file_lock_name}" + s3cp "${retention_file_lock_name}" "${backup_bucket}/${retention_file_lock_name}" - if [ $days_diff -gt "$retention_days" ]; then + current_date=$(date +%s) + backups=$(s3ls $backup_bucket) + + while IFS= read -r line; do + + delete_backup=false + folder=$(echo "$line" | awk '{print substr($2, 1, length($2)-1)}') + date_time=$(s3ls "${backup_bucket}/${folder}/restore --recursive" | awk '{print $1,$2}') + + if [[ -n "$date_time" ]]; then + + # Parse the date + backup_date=$(date -d "$date_time" +%s) + + # Calculate the difference in days + days_diff=$(( (current_date - backup_date) / (60*60*24) )) + # echo "line: $line" + # echo "date_time: $date_time" + # echo "backup_date: $backup_date" + # echo "days_diff: $days_diff" + + if [ $days_diff -gt "$retention_days" ]; then + delete_backup=true + fi + else delete_backup=true fi - else - delete_backup=true - fi + + if $delete_backup; then + echo "${backup_bucket}/${folder}" >> "${retention_file_lock_name}" + s3cp "${retention_file_lock_name}" "${backup_bucket}/${retention_file_lock_name}" + s3rm "${backup_bucket}/${folder}" + #printf "\n - Deleting: ${backup_bucket}/${folder}" + fi + printf "." + done <<< "$backups" - if $delete_backup; then - s3rm "${backup_bucket}/${folder}" - #echo "Deleting ${backup_bucket}/${folder}" - fi - printf "." + # Cleanup the lock file + s3rm "${backup_bucket}/${retention_file_lock_name}" + fi + + applying_retention_end=$(date +%s) + printf "Done $(human_readable_time $((applying_retention_end-applying_retention_start))) \n" + else - done <<< "$backups" + local retry_limit=2160 + local retry_counter=0 + local retry_sleep_interval=5 + if [ $storage == "LocalStorage" ]; then + + if [ -f "${backup_location}${retention_file_lock_name}-pm1" ]; then + printf " - Waiting for PM1 to finish deleting backups (max: $retry_limit intervals of $retry_sleep_interval sec ) ... \n" + fi; + while [ -f "${backup_location}${retention_file_lock_name}-pm1" ]; do + + if [ -f "${backup_location}${retention_file_lock_name}-pm1" ]; then + printf " - Waiting on PM1 to finish deleting %s \n" "$(tail -n 1 "${backup_location}${retention_file_lock_name}-pm1" 2>/dev/null )" + fi; + + if [ $retry_counter -ge $retry_limit ]; then + handle_early_exit_on_backup "\n [!] PM1 lock file still exists after $retry_limit sec, deleting backups via retention policy might have failed or taking too long, please investigate, exiting\n" true + fi + sleep $retry_sleep_interval + ((retry_counter++)) + done + elif [ $storage == "S3" ]; then + if [[ $(s3ls "${backup_bucket}/${retention_file_lock_name}") ]]; then + printf " - Waiting for PM1 to finish deleting backups (max: $retry_limit intervals of $retry_sleep_interval sec ) ... \n" + fi; + while [[ $(s3ls "${backup_bucket}/${retention_file_lock_name}") ]]; do + if [[ $(s3ls "${backup_bucket}/${retention_file_lock_name}") ]]; then + printf " - Waiting on PM1 to finish deleting %s \n" "$( s3cp "${backup_bucket}/${retention_file_lock_name}" "-" | tail -n 1 2>/dev/null )" + fi; + if [ $retry_counter -ge $retry_limit ]; then + handle_early_exit_on_backup "\n [!] PM1 lock file still exists after $retry_limit sec, deleting backups via retention policy might have failed or taking too long, please investigate, exiting\n" true + fi + sleep $retry_sleep_interval + ((retry_counter++)) + done + fi + applying_retention_end=$(date +%s) + printf "Done $(human_readable_time $((applying_retention_end-applying_retention_start))) \n" fi - printf " Done\n" } @@ -994,7 +1089,8 @@ issue_write_locks() } poll_check_no_active_sql_writes() { - printf " - Polling For Active Writes ... " + poll_active_sql_writes_start=$(date +%s) + printf " - Polling for active Writes ... " query="SELECT COUNT(*) FROM information_schema.processlist where user != 'root' AND command = 'Query' AND ( info LIKE '%INSERT%' OR info LIKE '%UPDATE%' OR info LIKE '%DELETE%' OR info LIKE '%LOAD DATA%');" attempts=0 no_writes=false @@ -1005,34 +1101,41 @@ poll_check_no_active_sql_writes() { fi if [ "$active_writes" -le 0 ]; then - printf "Done\n" + poll_active_sql_writes_end=$(date +%s) + printf "Done $(human_readable_time $((poll_active_sql_writes_start-poll_active_sql_writes_end))) \n" no_writes=true break else - # extra_info=$( mariadb -e "SELECT * FROM information_schema.processlist where user != 'root' AND command = 'Query' AND info LIKE '%INSERT%' OR info LIKE '%UPDATE%' OR info LIKE '%DELETE%' " ) - # echo "Active write operations detected: $active_writes" - # echo "$extra_info" - printf "." - if ! $quiet; then printf "\nActive write operations detected: $active_writes"; fi; + + if ! $quiet; then + printf "\nActive write operations detected: $active_writes\n"; + mariadb -e "select ID,USER,TIME,LEFT(INFO, 50) from information_schema.processlist where user != 'root' AND command = 'Query' AND ( info LIKE '%INSERT%' OR info LIKE '%UPDATE%' OR info LIKE '%DELETE%' OR info LIKE '%LOAD DATA%');" + else + printf "." + fi; sleep "$poll_interval" ((attempts++)) fi done if ! $no_writes; then + mariadb -e "select ID,USER,TIME,INFO from information_schema.processlist where user != 'root' AND command = 'Query' AND ( info LIKE '%INSERT%' OR info LIKE '%UPDATE%' OR info LIKE '%DELETE%' OR info LIKE '%LOAD DATA%');" handle_early_exit_on_backup "\n[X] Exceeded poll_max_wait: $poll_max_wait minutes\nActive write operations detected: $active_writes \n" true fi; + } poll_check_no_active_cpimports() { - printf " - Polling For Active cpimports ... " + poll_active_cpimports_start=$(date +%s) + printf " - Polling for active cpimports ... " attempts=0 no_cpimports=false if ! $columnstore_online ; then printf "Skip since offline\n"; return ; fi; while [ "$attempts" -lt "$max_poll_attempts" ]; do active_cpimports=$(viewtablelock 2>/dev/null ) if [[ "$active_cpimports" == *"No tables are locked"* ]]; then - printf "Done\n" + poll_active_cpimports_end=$(date +%s) + printf "Done $(human_readable_time $((poll_active_cpimports_end-poll_active_cpimports_start))) \n" no_cpimports=true break else @@ -1358,7 +1461,39 @@ deepParallelRsync() { wait } -run_backup() { +human_readable_time() { + local total_seconds=$1 + local days=$((total_seconds / 86400)) + local hours=$(( (total_seconds % 86400) / 3600 )) + local minutes=$(( (total_seconds % 3600) / 60 )) + local seconds=$(( total_seconds % 60 )) + + local output="" + + # Special case for 0 seconds + if (( total_seconds == 0 )); then + printf "<1 sec" + return + fi + + if (( days > 0 )); then + output+=" ${days} days" + fi + if (( hours > 0 )); then + output+=" ${hours} hours" + fi + if (( minutes > 0 )); then + output+=" ${minutes} min" + fi + if (( seconds > 0 || total_seconds == 0 )); then + output+=" ${seconds} sec" + fi + + printf "${output}" | xargs +} + +run_backup() { + backup_start=$(date +%s) if [ $storage == "LocalStorage" ]; then if [ $backup_destination == "Local" ]; then @@ -1383,6 +1518,7 @@ run_backup() { printf " Done\n" # Backup Columnstore data + columnstore_backup_start=$(date +%s) i=1 while [ $i -le $DBROOT_COUNT ]; do if [[ $ASSIGNED_DBROOT == "$i" || $HA == true ]]; then @@ -1393,15 +1529,19 @@ run_backup() { elif $parrallel_rsync ; then printf " - Parallel Rsync CS Data$i... \n" initiate_rsyncs $i - printf " Done\n" + columnstore_backup_end=$(date +%s) + printf " Done $(human_readable_time $((columnstore_backup_end-columnstore_backup_start))) \n" else printf " - Syncing Columnstore Data$i... " eval "rsync $additional_rysnc_flags /var/lib/columnstore/data$i/* $backup_location$today/data$i/ $xtra_cmd_args"; - printf " Done\n" + columnstore_backup_end=$(date +%s) + printf " Done $(human_readable_time $((columnstore_backup_end-columnstore_backup_start))) \n" + fi; fi ((i++)) done + # Backup MariaDB data if [ $ASSIGNED_DBROOT == "1" ]; then @@ -1419,6 +1559,7 @@ run_backup() { fi # Run mariadb-backup + mariadb_backup_start=$(date +%s) if ! $skip_mdb; then if [[ -n "$compress_format" ]]; then printf " - Compressing MariaDB Data... " @@ -1426,9 +1567,11 @@ run_backup() { case $compress_format in pigz) # Handle Cloud - if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $mbd_prefix 2>> $logfile; then + mariadb_backup_start=$(date +%s) + if ! mariabackup --user=root --backup --slave-info --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $mbd_prefix 2>> $logfile; then handle_early_exit_on_backup "\nFailed mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $mbd_prefix 2>> $logfile \n" fi + mariadb_backup_end=$(date +%s) printf " Done @ $mbd_prefix\n" ;; *) # unknown option @@ -1436,7 +1579,12 @@ run_backup() { esac else printf " - Copying MariaDB Data... " - if eval "mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root $xtra_cmd_args" ; then printf " Done \n"; else printf "\n Failed: mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root\n"; handle_early_exit_on_backup; fi + if eval "mariadb-backup --backup --slave-info --target-dir=$backup_location$today/mysql --user=root $xtra_cmd_args" ; then + mariadb_backup_end=$(date +%s) + printf " Done $(human_readable_time $((mariadb_backup_end-mariadb_backup_start))) \n" + else + handle_early_exit_on_backup "\n Failed: mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root\n"; + fi fi else echo "[!] Skipping mariadb-backup" @@ -1469,8 +1617,23 @@ run_backup() { else compress_paths+=" $MARIADB_SERVER_CONFIGS_PATH/*" fi + + # Backup mariadb server replication details + slave_status=$(mariadb -e "show slave status\G" 2>/dev/null) + if [[ $? -eq 0 ]]; then + if [[ -n "$slave_status" ]]; then + if [[ -z "$compress_format" ]]; then + mariadb -e "show slave status\G" | grep -E "Master_Host|Master_User|Master_Port|Gtid_IO_Pos" > $backup_location$today/configs/mysql/$pm/replication_details.txt + else + mariadb -e "show slave status\G" | grep -E "Master_Host|Master_User|Master_Port|Gtid_IO_Pos" > replication_details.txt + compress_paths+=" replication_details.txt" + fi + fi + else + handle_early_exit_on_backup "[!] - Failed to execute the MariaDB command: mariadb -e 'show slave status\G' \n" + fi fi - + # Handle compression for Columnstore Data & Configs if [[ -n "$compress_format" ]]; then cs_prefix="$backup_location$today/$split_file_cs_prefix.$compress_format" @@ -1478,9 +1641,31 @@ run_backup() { case $compress_format in pigz) printf " - Compressing CS Data & Configs... " + columnstore_backup_start=$(date +%s) if ! eval "tar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $cs_prefix 2>> $logfile"; then handle_early_exit_on_backup "[!] - Compression Failed \ntar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $cs_prefix 2>> $logfile \n" fi + columnstore_backup_end=$(date +%s) + + # Create summary info for list backup + get_latest_em_from_dbrm_directory "$DBRM_PATH" + em_file_name=$(basename "$em_file_full_path") + version_prefix=${em_file_name::-3} + journal_file=$(ls -la "${subdir_dbrms}/BRM_saves_journal" 2>/dev/null | awk 'NR==1 {print $5}' ) + vbbm_file=$(ls -la "${subdir_dbrms}/${version_prefix}_vbbm" 2>/dev/null | awk 'NR==1 {print $5}' ) + vss_file=$(ls -la "${subdir_dbrms}/${version_prefix}_vss" 2>/dev/null | awk 'NR==1 {print $5}' ) + echo "em_file_created: $em_file_created" >> $backup_location$today/backup_summary_info.txt + echo "em_file_name: $em_file_name" >> $backup_location$today/backup_summary_info.txt + echo "em_file_size: $em_file_size" >> $backup_location$today/backup_summary_info.txt + echo "journal_file: $journal_file" >> $backup_location$today/backup_summary_info.txt + echo "vbbm_file: $vbbm_file" >> $backup_location$today/backup_summary_info.txt + echo "vss_file: $vss_file" >> $backup_location$today/backup_summary_info.txt + echo "compression: $compress_format" >> $backup_location$today/backup_summary_info.txt + + if [ -f replication_details.txt ]; then + rm -rf replication_details.txt; + fi + printf " Done @ $cs_prefix\n" ;; *) # unknown option @@ -1488,18 +1673,6 @@ run_backup() { esac fi - # Save replication details - slave_status=$(mariadb -e "show slave status\G" 2>/dev/null) - if [[ $? -eq 0 ]]; then - if [[ -n "$slave_status" ]]; then - mariadb -e "show slave status\G" | grep -E "Master_Host|Master_User|Master_Port" > $backup_location$today/replication_details.txt - else - echo "No replication details to save" - fi - else - handle_early_exit_on_backup "[!] - Failed to execute the MariaDB command: mariadb -e 'show slave status\G' \n" - fi - if $incremental ; then # Log each incremental run now=$(date "+%m-%d-%Y %H:%M:%S"); echo "$pm updated on $now" >> $backup_location$today/incrementallyUpdated.txt @@ -1662,12 +1835,12 @@ run_backup() { pigz) # Handle Cloud if [ $cloud == "gcp" ]; then - if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter="gsutil cp - ${mbd_prefix}_\$FILE 2>$logfile" - chunk 2>$logfile; then + if ! mariabackup --user=root --backup --slave-info --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter="gsutil cp - ${mbd_prefix}_\$FILE 2>$logfile" - chunk 2>$logfile; then handle_early_exit_on_backup "\nFailed mariadb-backup --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter=\"gsutil cp - ${mbd_prefix}_\$FILE 2>$logfile\" - chunk 2>$logfile \n" fi elif [ $cloud == "aws" ]; then - if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter="aws s3 cp - ${mbd_prefix}_\$FILE 2>$logfile 1>&2" - chunk 2>$logfile; then + if ! mariabackup --user=root --backup --slave-info --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter="aws s3 cp - ${mbd_prefix}_\$FILE 2>$logfile 1>&2" - chunk 2>$logfile; then handle_early_exit_on_backup "\nFailed mariadb-backup --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter=\"aws s3 cp - ${mbd_prefix}_\$FILE 2>$logfile 1>&2\" - chunk 2>$logfile\n" fi fi @@ -1682,7 +1855,7 @@ run_backup() { printf " - Syncing MariaDB data ... \n" rm -rf $backup_location$today/mysql if mkdir -p $backup_location$today/mysql ; then - if eval "mariabackup --user=root --backup --target-dir=$backup_location$today/mysql/ $xtra_cmd_args"; then + if eval "mariabackup --user=root --backup --slave-info --target-dir=$backup_location$today/mysql/ $xtra_cmd_args"; then printf " + mariadb-backup @ $backup_location$today/mysql/ \n" else handle_early_exit_on_backup "\nFailed mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root\n" true @@ -1708,6 +1881,22 @@ run_backup() { else compress_paths+="$MARIADB_SERVER_CONFIGS_PATH" fi + + # Backup mariadb server replication details + slave_status=$(mariadb -e "show slave status\G" 2>/dev/null) + if [[ $? -eq 0 ]]; then + if [[ -n "$slave_status" ]]; then + mariadb -e "show slave status\G" | grep -E "Master_Host|Master_User|Master_Port|Gtid_IO_Pos" > replication_details.txt + if [[ -z "$compress_format" ]]; then + s3cp replication_details.txt $backup_bucket/$today/configs/mysql/$pm/replication_details.txt + else + compress_paths+=" replication_details.txt" + fi + fi + else + handle_early_exit_on_backup "[!] - Failed to execute the MariaDB command: mariadb -e 'show slave status\G' \n" + fi + fi # Handles streaming the compressed columnstore local files @@ -1727,6 +1916,12 @@ run_backup() { handle_early_exit_on_backup "[!] - Compression/Split/Upload Failed \n" fi fi + + # Cleanup temp files + if [ -f replication_details.txt ]; then + rm -rf replication_details.txt + fi + printf " Done @ $cs_prefix\n" ;; *) # unknown option @@ -1765,8 +1960,52 @@ run_backup() { clear_read_lock end=$(date +%s) - runtime=$((end-start)) - printf "\nRuntime: $runtime\n" + total_runtime=$((end - start)) + other=$total_runtime + + printf "\nRuntime Summary\n" + echo "--------------------" + if [[ -n "$applying_retention_end" && $applying_retention_end -ne 0 ]]; then + applying_retention_time=$((applying_retention_end - applying_retention_start)) + printf "%-20s %-15s\n" "Applying Retention:" "$(human_readable_time $applying_retention_time)" + other=$((other - applying_retention_time)) + fi + + if [[ -n "$poll_active_sql_writes_end" && $poll_active_sql_writes_end -ne 0 ]]; then + poll_sql_writes_time=$((poll_active_sql_writes_end - poll_active_sql_writes_start)) + printf "%-20s %-15s\n" "Polling SQL Writes:" "$(human_readable_time $poll_sql_writes_time)" + other=$((other - poll_sql_writes_time)) + fi + + if [[ -n "$poll_active_cpimports_end" && $poll_active_cpimports_end -ne 0 ]]; then + poll_cpimports_time=$((poll_active_cpimports_end - poll_active_cpimports_start)) + printf "%-20s %-15s\n" "Polling cpimports:" "$(human_readable_time $poll_cpimports_time)" + other=$((other - poll_cpimports_time)) + fi + + if [[ -n "$columnstore_backup_end" && $columnstore_backup_end -ne 0 ]]; then + columnstore_backup_time=$((columnstore_backup_end - columnstore_backup_start)) + printf "%-20s %-15s\n" "Columnstore Backup:" "$(human_readable_time $columnstore_backup_time)" + other=$((other - columnstore_backup_time)) + fi + + if [[ -n "$mariadb_backup_end" && $mariadb_backup_end -ne 0 ]]; then + mariadb_backup_time=$((mariadb_backup_end - mariadb_backup_start)) + printf "%-20s %-15s\n" "MariaDB Backup:" "$(human_readable_time $mariadb_backup_time)" + other=$((other - mariadb_backup_time)) + fi + + if (( other < 0 )); then + printf "%-20s %-15s\n" "Other:" "N/A (invalid data)" + else + if [ $other -gt 0 ]; then + printf "%-20s %-15s\n" "Other:" "<$(human_readable_time $other)" + else + printf "%-20s %-15s\n" "Other:" "<1 sec" + fi + fi + + printf "\nTotal Runtime: $(human_readable_time $((end-start))) \n" printf "$final_message\n\n" } @@ -2353,7 +2592,7 @@ validation_prechecks_for_restore() { fi fi; else - handle_early_exit_on_restore " Failed to list backup contents...\n$check5\n" + handle_early_exit_on_restore " Failed to list backup contents...\n$check5\n" fi; fi; @@ -2734,10 +2973,12 @@ run_restore() if [ $DBROOT_COUNT -gt 1 ]; then echo -e "[!!] Check Replicas to manually configure mariadb replication"; fi echo -e "systemctl start mariadb " echo -e "systemctl start mariadb-columnstore-cmapi " - echo -e "mcs cluster start \n\n" + echo -e "mcs cluster start" + echo -e "mariadb -e \"show variables like '%gtid_current_pos%';\"\n\n" else echo -e "[!!] Check this replica to manually configure mariadb replication after primary node is ready" echo -e "Example:" + echo -e "mariadb -e \"SET GLOBAL gtid_slave_pos = '0-1-14'\"" echo -e "mariadb -e \"stop slave;CHANGE MASTER TO MASTER_HOST='\$pm1' , MASTER_USER='repuser' , MASTER_PASSWORD='aBcd123%123%aBc' , MASTER_USE_GTID = slave_pos;start slave;show slave status\G\"" echo -e "mariadb -e \"show slave status\G\" | grep \"Slave_\" \n" fi @@ -3321,7 +3562,7 @@ get_latest_em_from_dbrm_directory() { fi else subdir_dbrms="$1" - latest_em_file=$(find "${subdir_dbrms}" -maxdepth 1 -type f -name "BRM_saves*_em" -exec ls -lat {} + | awk 'NR==1 {printf "%-12s %-4s %-2s %-5s %s\n", $5, $6, $7, $8, $9}'| head -n 1) + latest_em_file=$(find "${subdir_dbrms}" -maxdepth 1 -type f -name "BRM_saves*_em" -exec ls -lat {} + 2>/dev/null | awk 'NR==1 {printf "%-12s %-4s %-2s %-5s %s\n", $5, $6, $7, $8, $9}'| head -n 1) em_file_size=$(echo "$latest_em_file" | awk '{print $1}' ) em_file_created=$(echo "$latest_em_file" | awk '{print $2,$3,$4}' ) em_file_full_path=$(echo $latest_em_file | awk '{print $NF}' ) @@ -3339,18 +3580,32 @@ list_restore_options_from_backups() { echo "Retention Policy: $retention_days days" fi; echo "--------------------------------------------------------------------------" - printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s %-10s\n" "Options" "Last-Updated" "Extent Map" "EM-Size" "Journal-Size" "VBBM-Size" "VSS-Size" "Days Old" + printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s %-10s\n" "Options" "EM-Updated" "Extent Map" "EM-Size" "Journal-Size" "VBBM-Size" "VSS-Size" "Backup Days Old" # Iterate over subdirectories for subdir in "${backup_location}"/*; do if [ -f "${subdir}/cs-localfiles.tar.pigz" ]; then em_file_created=$( date -r "$subdir" +"%b %d %H:%M" ) - em_file_size="N/A" - em_file_name="N/A" - journal_file="N/A" - vbbm_file="N/A" - vss_file="N/A" + + # parse from ${subdir}/backup_summary_info.txt + if [ -f "${subdir}/backup_summary_info.txt" ]; then + em_file_created="$(grep -m 1 "em_file_created" "${subdir}/backup_summary_info.txt" | awk -F': ' '{print $2}')" + em_file_name="$(grep -m 1 "em_file_name" "${subdir}/backup_summary_info.txt" | awk '{print $2}')" + em_file_size="$(grep -m 1 "em_file_size" "${subdir}/backup_summary_info.txt" | awk '{print $2}')" + journal_file="$(grep -m 1 "journal_file" "${subdir}/backup_summary_info.txt" | awk '{print $2}')" + vbbm_file="$(grep -m 1 "vbbm_file" "${subdir}/backup_summary_info.txt" | awk '{print $2}')" + vss_file="$(grep -m 1 "vss_file" "${subdir}/backup_summary_info.txt" | awk '{print $2}')" + compression="$(grep -m 1 "compression" "${subdir}/backup_summary_info.txt" | awk '{print $2}')" + else + em_file_created="N/A" + em_file_size="N/A" + em_file_name="N/A" + journal_file="N/A" + vbbm_file="N/A" + vss_file="N/A" + compression="N/A" + fi file_age=$(($(date +%s) - $(date -r "$subdir" +%s))) file_age_days=$((file_age / 86400)) @@ -3359,13 +3614,12 @@ list_restore_options_from_backups() { file_age_days="$file_age_days (Will Delete)" fi - printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s %-10s\n" "$(basename "$subdir")" "$em_file_created" "$em_file_name" "$em_file_size" "$journal_file" "$vbbm_file" "$vss_file" "$file_age_days" + printf "%-35s %-9s %-13s %-15s %-12s %-12s %-10s %-10s %-10s\n" "$(basename "$subdir")" "$compression" "$em_file_created" "$em_file_name" "$em_file_size" "$journal_file" "$vbbm_file" "$vss_file" "$file_age_days" continue fi get_latest_em_from_dbrm_directory "$subdir/data1/systemFiles/dbrm/" - if [ -f "${subdir_dbrms}/BRM_saves_journal" ]; then em_file_name=$(basename "$em_file_full_path") version_prefix=${em_file_name::-3} @@ -3373,12 +3627,13 @@ list_restore_options_from_backups() { vbbm_file=$(ls -la "${subdir_dbrms}/${version_prefix}_vbbm" 2>/dev/null | awk 'NR==1 {print $5}' ) vss_file=$(ls -la "${subdir_dbrms}/${version_prefix}_vss" 2>/dev/null | awk 'NR==1 {print $5}' ) - file_age=$(($(date +%s) - $(date -r "$subdir/data1/systemFiles" +%s))) + file_age=$(($(date +%s) - $(date -r "$subdir" +%s))) file_age_days=$((file_age / 86400)) # Check if the backup will be deleted given the retention policy defined if [ -n "$retention_days" ] && [ $retention_days -ne 0 ]; then - will_delete="$(find "$subdir/data1/systemFiles" -mindepth 1 -maxdepth 1 -type d -name "*" -mtime +$retention_days -exec echo {} \;)" + # file_age_days=$((file_age / 60)); will_delete="$(find "$subdir" -mindepth 1 -maxdepth 1 -type d -name "*" -mmin +$retention_days -exec echo {} \;)" + will_delete="$(find "$subdir" -mindepth 1 -maxdepth 1 -type d -name "*" -mtime +$retention_days -exec echo {} \;)" if [ -n "$will_delete" ]; then file_age_days="$file_age_days (Will Delete)" @@ -3400,7 +3655,7 @@ list_dbrm_restore_options_from_backups() { echo "Retention Policy: $retention_days days" fi; echo "--------------------------------------------------------------------------" - printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s\n" "Options" "Last-Updated" "Extent Map" "EM-Size" "Journal-Size" "VBBM-Size" "VSS-Size" + printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s\n" "Options" "EM-Updated" "Extent Map" "EM-Size" "Journal-Size" "VBBM-Size" "VSS-Size" # Iterate over subdirectories for subdir in "${backup_location}"/*; do @@ -3783,6 +4038,7 @@ dynamic_list_backups() { } + handle_list_backups () { if $list_backups ; then @@ -3799,16 +4055,42 @@ handle_empty_restore_folder() { fi; } +handle_apply_retention_only() { + + if $apply_retention_only; then + s1=20 + s2=20 + printf "\nApplying Retention Only Variables\n" + echo "--------------------------------------------------------------------------" + if [ -n "$config_file" ] && [ -f "$config_file" ]; then + printf "%-${s1}s %-${s2}s\n" "Configuration File:" "$config_file"; + fi + echo "--------------------------------------------------------------------------" + printf "%-${s1}s %-${s2}s\n" "Backup Location:" "$backup_location"; + printf "%-${s1}s %-${s2}s\n" "Timestamp:" "$(date +%m-%d-%Y-%H:%M:%S)"; + printf "%-${s1}s %-${s2}s\n" "Retention:" "$retention_days"; + echo "--------------------------------------------------------------------------" + + if [ $retention_days -eq 0 ]; then + printf "\n[!] Are you sure retention should be set to 0?\n" + fi; + + apply_backup_retention_policy + exit 0 + fi; +} + process_backup() { load_default_backup_variables; parse_backup_variables "$@"; handle_list_backups "$@" + handle_apply_retention_only "$@" print_backup_variables; check_for_dependancies "backup"; validation_prechecks_for_backup; - apply_backup_retention_policy + apply_backup_retention_policy; issue_write_locks; run_save_brm; run_backup; @@ -3826,11 +4108,20 @@ process_restore() run_restore; } +global_dependencies() { + if ! command -v curl &> /dev/null; then + printf "\n[!] curl not found. Please install curl\n\n" + exit 1; + fi +} + print_mcs_bk_mgr_version_info() { echo "MariaDB Columnstore Backup Manager" echo "Version: $mcs_bk_manager_version" } +global_dependencies + case "$action" in 'help' | '--help' | '-help' | '-h') print_action_help_text @@ -3859,4 +4150,4 @@ case "$action" in ;; esac -exit 0; +exit 0; \ No newline at end of file From 6a6db672db82a8aa8de740ea6582d7d5439b6717 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Fri, 14 Mar 2025 14:39:06 +0300 Subject: [PATCH 56/65] feat(FDB): [FDB BlobHandler] Add `writeOrUpdateBlob`, update `read` and `remove`. (#3373) This patch: 1) Adds a `writeOrUpdateBlob` function. 2) Updates `read` and `remove` to take in account the size of the `keys` and `values` for one FDB transaction. --- storage-manager/src/MetadataFile.cpp | 6 +- utils/fdb_wrapper_cpp/include/fdbcs.hpp | 28 +++ utils/fdb_wrapper_cpp/src/fdbcs.cpp | 235 +++++++++++++------- utils/fdb_wrapper_cpp/test/test_fdb_api.cpp | 6 +- 4 files changed, 196 insertions(+), 79 deletions(-) diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index 8cf964c69..025565203 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -390,10 +390,10 @@ int MetadataFile::writeMetadata() stringstream stream; write_json(stream, *jsontree); - if (!blobWriter.writeBlob(kvStorage, metaKVName_, stream.str())) + if (!blobWriter.writeOrUpdateBlob(kvStorage, metaKVName_, stream.str())) { - SMLogging::get()->log(LOG_CRIT, "Metadatafile: cannot commit tnx set()."); - throw runtime_error("Metadatafile: cannot commit tnx set()."); + SMLogging::get()->log(LOG_CRIT, "Metadatafile: cannot write metadata to storage."); + throw runtime_error("Metadatafile: cannot write metadata to storage."); } } diff --git a/utils/fdb_wrapper_cpp/include/fdbcs.hpp b/utils/fdb_wrapper_cpp/include/fdbcs.hpp index 4298dfba9..b083ecb88 100644 --- a/utils/fdb_wrapper_cpp/include/fdbcs.hpp +++ b/utils/fdb_wrapper_cpp/include/fdbcs.hpp @@ -50,6 +50,8 @@ class Transaction explicit Transaction(FDBTransaction* tnx); ~Transaction(); + // Tries to atomically (during one transaction) swap keys. + bool swap(const ByteArray &key1, const ByteArray &key2); // Sets a given `key` and given `value`. void set(const ByteArray& key, const ByteArray& value) const; // Gets a `value` by the given `key`. @@ -117,6 +119,7 @@ using Keys = std::vector; using KeyBlockMap = std::unordered_map; using TreeLevelNumKeysMap = std::unordered_map; +// Represents an abstract class for key generators. class KeyGenerator { public: @@ -152,12 +155,37 @@ class BlobHandler } // Writes the given `blob` with given `key`. + // The semantic of this `write` is not atomic, it splits the data into multiple fdb transactions, if one of + // this transaction fails, we should call `removeBlob` to clear data which were written partially, and then + // try to `writeBlob` again. bool writeBlob(std::shared_ptr database, const ByteArray& key, const ByteArray& blob); + + // This function: + // 1) Checks if blob with the same key exists if not, uses `writeBlob` function. + // 2) Creates a new tree with a new key. + // 3) Atomically (during one fdb transaction) swaps root nodes for the original tree and new tree. + // 4) Removes original tree. + bool writeOrUpdateBlob(std::shared_ptr database, const ByteArray& key, + const ByteArray& blob); + // Reads `blob` by the given `key`, on error returns false. std::pair readBlob(std::shared_ptr database, const ByteArray& key); + + // Read blocks of the data based on the given `keys` starting from `index` position in keys vector + // and taking in account the max size of transaction. + // If `DataBlock` reached in a tree (leaf nodes), sets `dataBlockReached` flag to true. + std::pair> readBlocks(std::shared_ptr database, + const std::vector& keys, uint32_t& index, + bool& dataBlockReached); + // Removes a `blob` by the given `key`, on error returns false. + // The semantic of this `remove` is not atomic, it splits keys to remove into multiple fdb transactions, if + // one of this transaction fails, we should call `removeBlob` again to remove keys, which were not removed. bool removeBlob(std::shared_ptr database, const ByteArray& key); + // Checks if key exis. + bool keyExists(std::shared_ptr database, const ByteArray& key); + private: size_t insertData(Block& block, const std::string& blob, const size_t offset); void insertKey(Block& block, const std::string& value); diff --git a/utils/fdb_wrapper_cpp/src/fdbcs.cpp b/utils/fdb_wrapper_cpp/src/fdbcs.cpp index a6de89676..e6e681b02 100644 --- a/utils/fdb_wrapper_cpp/src/fdbcs.cpp +++ b/utils/fdb_wrapper_cpp/src/fdbcs.cpp @@ -50,6 +50,24 @@ Transaction::~Transaction() } } +bool Transaction::swap(const ByteArray& key1, const ByteArray& key2) +{ + if (!tnx_) + return false; + + auto resultPair1 = get(key1); + if (!resultPair1.first) + return false; + + auto resultPair2 = get(key2); + if (!resultPair2.first) + return false; + + set(key1, resultPair2.second); + set(key2, resultPair1.second); + return true; +} + void Transaction::set(const ByteArray& key, const ByteArray& value) const { if (tnx_) @@ -71,6 +89,14 @@ std::pair Transaction::get(const ByteArray& key) const return {false, {}}; } + err = fdb_future_get_error(future); + if (err) + { + fdb_future_destroy(future); + std::cerr << "fdb_future_get_error error, code: " << (int)err << std::endl; + return {false, {}}; + } + const uint8_t* outValue; int outValueLength; fdb_bool_t present; @@ -85,13 +111,9 @@ std::pair Transaction::get(const ByteArray& key) const fdb_future_destroy(future); if (present) - { return {true, ByteArray(outValue, outValue + outValueLength)}; - } else - { return {false, {}}; - } } return {false, {}}; } @@ -115,25 +137,25 @@ void Transaction::removeRange(const ByteArray& beginKey, const ByteArray& endKey bool Transaction::commit() const { - if (tnx_) + if (!tnx_) + return false; + + FDBFuture* future = fdb_transaction_commit(tnx_); + auto err = fdb_future_block_until_ready(future); + if (err) { - FDBFuture* future = fdb_transaction_commit(tnx_); - auto err = fdb_future_block_until_ready(future); - if (err) - { - fdb_future_destroy(future); - std::cerr << "fdb_future_block_until_ready error, code: " << (int)err << std::endl; - return false; - } - err = fdb_future_get_error(future); - if (err) - { - fdb_future_destroy(future); - std::cerr << "fdb_future_get_error(), code: " << (int)err << std::endl; - return false; - } fdb_future_destroy(future); + std::cerr << "fdb_future_block_until_ready error, code: " << (int)err << std::endl; + return false; } + err = fdb_future_get_error(future); + if (err) + { + fdb_future_destroy(future); + std::cerr << "fdb_future_get_error(), code: " << (int)err << std::endl; + return false; + } + fdb_future_destroy(future); return true; } @@ -320,6 +342,9 @@ TreeLevelNumKeysMap BlobHandler::computeNumKeysForEachTreeLevel(const int32_t tr bool BlobHandler::writeBlob(std::shared_ptr dataBase, const ByteArray& key, const ByteArray& blob) { + if (!dataBase) + return false; + const size_t blobSizeInBytes = blob.size(); if (!blobSizeInBytes) return commitKey(dataBase, key, ""); @@ -383,6 +408,37 @@ bool BlobHandler::writeBlob(std::shared_ptr dataBase, const return true; } +bool BlobHandler::keyExists(std::shared_ptr database, const ByteArray& key) +{ + auto tnx = database->createTransaction(); + if (!tnx) + return false; + return tnx->get(key).first; +} + +bool BlobHandler::writeOrUpdateBlob(std::shared_ptr dataBase, const ByteArray& key, + const ByteArray& blob) +{ + if (!dataBase) + return false; + + if (keyExists(dataBase, key)) + { + auto newKey = key + "new"; + if (!writeBlob(dataBase, newKey, blob)) + return false; + // Tnx destructor calls destroy on transaction if it fails/not commited. + { + auto tnx = dataBase->createTransaction(); + if (!tnx->swap(newKey, key) || !tnx->commit()) + return false; + } + return removeBlob(dataBase, newKey); + } + else + return writeBlob(dataBase, key, blob); +} + std::pair BlobHandler::getKeysFromBlock(const Block& block) { Keys keys; @@ -406,66 +462,88 @@ bool BlobHandler::isDataBlock(const Block& block) return block.second.compare(0, keyBlockIdentifier.size(), keyBlockIdentifier) != 0; } +std::pair> BlobHandler::readBlocks(std::shared_ptr database, + const std::vector& keys, + uint32_t& index, bool& dataBlockReached) +{ + if (!database) + return {false, {}}; + + auto tnx = database->createTransaction(); + if (!tnx) + return {false, {}}; + + size_t currentTnxSize = 0; + std::vector blocks; + // Take in account the size of the data that was read. + while ((index < keys.size()) && (currentTnxSize + blockSizeInBytes_ < maxTnxSize_)) + { + const auto& key = keys[index]; + auto p = tnx->get(key); + if (!p.first) + return {false, {}}; + currentTnxSize += key.size() + p.second.size(); + + Block block{0, p.second}; + if (!dataBlockReached && isDataBlock(block)) + dataBlockReached = true; + + blocks.push_back(block); + ++index; + } + + return {true, blocks}; +} + std::pair BlobHandler::readBlob(std::shared_ptr database, const ByteArray& key) { + if (!database) + return {false, ""}; + Keys currentKeys{key}; bool dataBlockReached = false; + std::string blob; while (!dataBlockReached) { - auto tnx = database->createTransaction(); - if (!tnx) - return {false, ""}; - std::vector blocks; - for (const auto& key : currentKeys) + blocks.reserve(currentKeys.size()); + + uint32_t index = 0; + while (index < currentKeys.size()) { - auto p = tnx->get(key); + const auto p = readBlocks(database, currentKeys, index, dataBlockReached); if (!p.first) return {false, ""}; - - Block block{0, p.second}; - if (isDataBlock(block)) - { - dataBlockReached = true; - break; - } - blocks.push_back(block); + blocks.insert(blocks.end(), p.second.begin(), p.second.end()); } - if (dataBlockReached) - break; - - Keys nextKeys; - for (const auto& block : blocks) + if (!dataBlockReached) [[likely]] { - auto keysPair = getKeysFromBlock(block); - if (!keysPair.first) - return {false, ""}; + Keys nextKeys; + for (const auto& block : blocks) + { + auto keysPair = getKeysFromBlock(block); + if (!keysPair.first) + return {false, ""}; - auto& keys = keysPair.second; - nextKeys.insert(nextKeys.end(), keys.begin(), keys.end()); + auto& keys = keysPair.second; + nextKeys.insert(nextKeys.end(), keys.begin(), keys.end()); + } + currentKeys = std::move(nextKeys); + } + else + { + blob.reserve(blocks.size() * dataBlockSizeInBytes_); + for (const auto& dataBlock : blocks) + { + if (!dataBlock.second.size()) + return {false, ""}; + blob.insert(blob.end(), dataBlock.second.begin() + dataBlockIdentifier.size(), + dataBlock.second.end()); + } } - currentKeys = std::move(nextKeys); - } - - std::string blob; - for (const auto& key : currentKeys) - { - auto tnx = database->createTransaction(); - if (!tnx) - return {false, ""}; - - auto resultPair = tnx->get(key); - if (!resultPair.first) - return {false, ""}; - - auto& dataBlock = resultPair.second; - if (!dataBlock.size()) - return {false, ""}; - - blob.insert(blob.end(), dataBlock.begin() + dataBlockIdentifier.size(), dataBlock.end()); } return {true, blob}; @@ -488,29 +566,30 @@ bool BlobHandler::removeKeys(std::shared_ptr database, const bool BlobHandler::removeBlob(std::shared_ptr database, const Key& key) { std::unordered_map treeLevel; - auto tnx = database->createTransaction(); - if (!tnx) - return false; - - uint32_t currentLevel = 0; + int32_t currentLevel = 0; treeLevel[0] = {key}; + bool dataBlockReached = false; while (true) { const auto& currentKeys = treeLevel[currentLevel]; std::vector blocks; - for (const auto& key : currentKeys) + blocks.reserve(currentKeys.size()); + + uint32_t index = 0; + while (index < currentKeys.size()) { - auto p = tnx->get(key); + const auto p = readBlocks(database, currentKeys, index, dataBlockReached); if (!p.first) return false; - blocks.push_back({0, p.second}); + blocks.insert(blocks.end(), p.second.begin(), p.second.end()); } - if (isDataBlock(blocks.front()) || !currentKeys.size()) + if (dataBlockReached) break; Keys nextKeys; + nextKeys.reserve(blocks.size() * numKeysInBlock_); for (const auto& block : blocks) { auto keysPair = getKeysFromBlock(block); @@ -525,8 +604,14 @@ bool BlobHandler::removeBlob(std::shared_ptr database, const treeLevel[currentLevel] = std::move(nextKeys); } - for (uint32_t level = 0; level <= currentLevel; ++level) - RETURN_ON_ERROR(removeKeys(database, treeLevel[level])); + // Start to remove keys from the bottom of the tree. + // If we fail in the middle of removing operation, we can try to remove a unremoved data again, because a + // tree is still in a valid state, even if the bottom levels are removed. + while (currentLevel >= 0) + { + RETURN_ON_ERROR(removeKeys(database, treeLevel[currentLevel])); + --currentLevel; + } return true; } @@ -536,4 +621,4 @@ bool setAPIVersion() auto err = fdb_select_api_version(FDB_API_VERSION); return err ? false : true; } -} // namespace FDBCS +} // namespace FDBCS \ No newline at end of file diff --git a/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp b/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp index 4f8f1decd..f6bae3df6 100644 --- a/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp +++ b/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp @@ -54,7 +54,7 @@ static void testBlobHandler(std::shared_ptr db) std::vector blobSizes{0, 1, 11, 101, 1001, 10001, 100001, 1000001, 10000001, 100000001}; std::vector blockSizes{10000, 100000}; #else - std::vector blobSizes{0, 1, 10001, 100001, 10000001, 100000001}; + std::vector blobSizes{0, 1, 100001, 10000001, 100000001}; std::vector blockSizes{100000}; #endif @@ -87,6 +87,10 @@ static void testBlobHandler(std::shared_ptr db) std::cout << "Read blob time: " << ms_int.count() << std::endl; t1 = high_resolution_clock::now(); #endif + auto blobB = generateBlob(blobSize + 1); + assert_internal(handler.writeOrUpdateBlob(db, rootKey, blobB), "Remove blob error"); + p = handler.readBlob(db, rootKey); + assert_internal(p.second == blobB, "Blobs not equal"); assert_internal(handler.removeBlob(db, rootKey), "Remove blob error"); #ifdef TEST_PERF t2 = high_resolution_clock::now(); From 75ed571f0913d470ca6c96d020fde3dcff4bee87 Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Fri, 31 Jan 2025 15:43:35 +0300 Subject: [PATCH 57/65] feat(cmapi): MCOL-5133: Stage3 stand alone cli tool. MAJOR: Some logic inside node remove changed significantly using active nodes list from Columnstore.xml to broadcast config after remove. [fix] TransactionManager passsing extra, remove and optional nodes arguments to start_transaction function [fix] commit and rollback methods of TransactionManager adding nodes argument [fix] TransactionManager using success_txn_nodes inside [fix] remove node logic to use Transaction manager [fix] cluster set api key call using totp on a top level cli call [add] missed docstrings [fix] cluster shutdown timeout for next release --- cmapi/cmapi_server/controllers/api_clients.py | 34 +++++++-- cmapi/cmapi_server/controllers/endpoints.py | 44 +++++++++--- cmapi/cmapi_server/handlers/cluster.py | 70 ++++--------------- cmapi/cmapi_server/helpers.py | 4 +- cmapi/cmapi_server/managers/transaction.py | 70 ++++++++++++++----- cmapi/cmapi_server/node_manipulation.py | 54 +++++++++----- .../cmapi_server/test/config_apply_example.py | 9 +-- cmapi/mcs_cluster_tool/cluster_app.py | 22 +++--- 8 files changed, 183 insertions(+), 124 deletions(-) diff --git a/cmapi/cmapi_server/controllers/api_clients.py b/cmapi/cmapi_server/controllers/api_clients.py index c03d0b263..dd449258a 100644 --- a/cmapi/cmapi_server/controllers/api_clients.py +++ b/cmapi/cmapi_server/controllers/api_clients.py @@ -1,7 +1,13 @@ import requests from typing import Any, Dict, Optional, Union + +import pyotp + from cmapi_server.controllers.dispatcher import _version -from cmapi_server.constants import CURRENT_NODE_CMAPI_URL +from cmapi_server.constants import ( + CMAPI_CONF_PATH, CURRENT_NODE_CMAPI_URL, SECRET_KEY, +) +from cmapi_server.helpers import get_config_parser, get_current_key class ClusterControllerClient: @@ -11,6 +17,10 @@ class ClusterControllerClient: ): """Initialize the ClusterControllerClient with the base URL. + WARNING: This class only handles the API requests, it does not + handle the transaction management. So it should be started + at level above using TransactionManager (decorator or context manager). + :param base_url: The base URL for the API endpoints, defaults to CURRENT_NODE_CMAPI_URL :type base_url: str, optional @@ -59,14 +69,14 @@ class ClusterControllerClient: return self._request('PUT', 'node', {**node_info, **extra}) def remove_node( - self, node_id: str, extra: Dict[str, Any] = dict() + self, node: str, extra: Dict[str, Any] = dict() ) -> Union[Dict[str, Any], Dict[str, str]]: """Remove a node from the cluster. - :param node_id: The ID of the node to remove. + :param node: node IP, name or FQDN. :return: The response from the API. """ - return self._request('DELETE', 'node', {'node_id': node_id}) + return self._request('DELETE', 'node', {'node': node, **extra}) def get_status(self) -> Union[Dict[str, Any], Dict[str, str]]: """Get the status of the cluster. @@ -83,7 +93,12 @@ class ClusterControllerClient: :param api_key: The API key to set. :return: The response from the API. """ - return self._request('put', 'apikey-set', {'api_key': api_key}) + totp = pyotp.TOTP(SECRET_KEY) + payload = { + 'api_key': api_key, + 'verification_key': totp.now() + } + return self._request('put', 'apikey-set', payload) def set_log_level( self, log_level: str @@ -117,9 +132,16 @@ class ClusterControllerClient: :return: The response from the API. """ url = f'{self.base_url}/cmapi/{_version}/cluster/{endpoint}' + cmapi_cfg_parser = get_config_parser(CMAPI_CONF_PATH) + key = get_current_key(cmapi_cfg_parser) + headers = {'x-api-key': key} + if method in ['PUT', 'POST', 'DELETE']: + headers['Content-Type'] = 'application/json' + data = {'in_transaction': True, **(data or {})} try: response = requests.request( - method, url, json=data, timeout=self.request_timeout + method, url, headers=headers, json=data, + timeout=self.request_timeout, verify=False ) response.raise_for_status() return response.json() diff --git a/cmapi/cmapi_server/controllers/endpoints.py b/cmapi/cmapi_server/controllers/endpoints.py index 53adcfc9c..c6b38b5d3 100644 --- a/cmapi/cmapi_server/controllers/endpoints.py +++ b/cmapi/cmapi_server/controllers/endpoints.py @@ -14,15 +14,16 @@ import requests from cmapi_server.exceptions import CMAPIBasicError from cmapi_server.constants import ( - DEFAULT_SM_CONF_PATH, EM_PATH_SUFFIX, DEFAULT_MCS_CONF_PATH, MCS_EM_PATH, - MCS_BRM_CURRENT_PATH, S3_BRM_CURRENT_PATH, CMAPI_CONF_PATH, SECRET_KEY, + DEFAULT_MCS_CONF_PATH, DEFAULT_SM_CONF_PATH, EM_PATH_SUFFIX, + MCS_BRM_CURRENT_PATH, MCS_EM_PATH, S3_BRM_CURRENT_PATH, SECRET_KEY, ) from cmapi_server.controllers.error import APIError from cmapi_server.handlers.cej import CEJError from cmapi_server.handlers.cluster import ClusterHandler from cmapi_server.helpers import ( - cmapi_config_check, get_config_parser, get_current_key, get_dbroots, - system_ready, save_cmapi_conf_file, dequote, in_maintenance_state, + cmapi_config_check, dequote, get_active_nodes, get_config_parser, + get_current_key, get_dbroots, in_maintenance_state, save_cmapi_conf_file, + system_ready, ) from cmapi_server.logging_management import change_loggers_level from cmapi_server.managers.application import AppManager @@ -60,6 +61,9 @@ def raise_422_error( :type exc_info: bool :raises APIError: everytime with custom error message """ + # TODO: change: + # - func name to inspect.stack(0)[1][3] + # - make something to logger, seems passing here is useless logger.error(f'{func_name} {err_msg}', exc_info=exc_info) raise APIError(422, err_msg) @@ -146,7 +150,21 @@ def active_operation(): if txn_section is not None: txn_manager_address = app.config['txn'].get('manager_address', None) if txn_manager_address is not None and len(txn_manager_address) > 0: - raise APIError(422, "There is an active operation.") + raise_422_error( + module_logger, 'active_operation', 'There is an active operation.' + ) + + +@cherrypy.tools.register('before_handler', priority=82) +def has_active_nodes(): + """Check if there are any active nodes in the cluster.""" + active_nodes = get_active_nodes() + + if len(active_nodes) == 0: + raise_422_error( + module_logger, 'has_active_nodes', + 'No active nodes in the cluster.' + ) class TimingTool(cherrypy.Tool): @@ -816,19 +834,22 @@ class ClusterController: @cherrypy.tools.json_in() @cherrypy.tools.json_out() @cherrypy.tools.validate_api_key() # pylint: disable=no-member + @cherrypy.tools.has_active_nodes() # pylint: disable=no-member def put_shutdown(self): func_name = 'put_shutdown' log_begin(module_logger, func_name) request = cherrypy.request request_body = request.json + timeout = request_body.get('timeout', None) + force = request_body.get('force', False) config = request_body.get('config', DEFAULT_MCS_CONF_PATH) in_transaction = request_body.get('in_transaction', False) try: if not in_transaction: with TransactionManager(): - response = ClusterHandler.shutdown(config) + response = ClusterHandler.shutdown(config, timeout) else: response = ClusterHandler.shutdown(config) except CMAPIBasicError as err: @@ -882,7 +903,7 @@ class ClusterController: try: if not in_transaction: - with TransactionManager(): + with TransactionManager(extra_nodes=[node]): response = ClusterHandler.add_node(node, config) else: response = ClusterHandler.add_node(node, config) @@ -903,7 +924,6 @@ class ClusterController: request_body = request.json node = request_body.get('node', None) config = request_body.get('config', DEFAULT_MCS_CONF_PATH) - #TODO: for next release in_transaction = request_body.get('in_transaction', False) #TODO: add arguments verification decorator @@ -911,7 +931,11 @@ class ClusterController: raise_422_error(module_logger, func_name, 'missing node argument') try: - response = ClusterHandler.remove_node(node, config) + if not in_transaction: + with TransactionManager(remove_nodes=[node]): + response = ClusterHandler.remove_node(node, config) + else: + response = ClusterHandler.remove_node(node, config) except CMAPIBasicError as err: raise_422_error(module_logger, func_name, err.message) @@ -1021,7 +1045,7 @@ class ClusterController: if not totp_key or not new_api_key: # not show which arguments in error message because endpoint for - # internal usage only + # cli tool or internal usage only raise_422_error( module_logger, func_name, 'Missing required arguments.' ) diff --git a/cmapi/cmapi_server/handlers/cluster.py b/cmapi/cmapi_server/handlers/cluster.py index 88c381d07..b4012a733 100644 --- a/cmapi/cmapi_server/handlers/cluster.py +++ b/cmapi/cmapi_server/handlers/cluster.py @@ -11,11 +11,9 @@ from cmapi_server.constants import ( ) from cmapi_server.exceptions import CMAPIBasicError from cmapi_server.helpers import ( - broadcast_new_config, commit_transaction, get_active_nodes, get_dbroots, - get_config_parser, get_current_key, get_id, get_version, start_transaction, - rollback_transaction, update_revision_and_manager, + broadcast_new_config, get_active_nodes, get_dbroots, get_config_parser, + get_current_key, get_version, update_revision_and_manager, ) -from cmapi_server.managers.transaction import TransactionManager from cmapi_server.node_manipulation import ( add_node, add_dbroot, remove_node, switch_node_maintenance, ) @@ -28,9 +26,9 @@ class ClusterAction(Enum): STOP = 'stop' -def toggle_cluster_state(action: ClusterAction, config: str) -> dict: - """ - Toggle the state of the cluster (start or stop). +def toggle_cluster_state( + action: ClusterAction, config: str) -> dict: + """Toggle the state of the cluster (start or stop). :param action: The cluster action to perform. (ClusterAction.START or ClusterAction.STOP). @@ -127,16 +125,16 @@ class ClusterHandler(): @staticmethod def shutdown( - config: str = DEFAULT_MCS_CONF_PATH, timeout: int = 15 + config: str = DEFAULT_MCS_CONF_PATH, timeout: Optional[int] = None ) -> dict: """Method to stop the MCS Cluster. :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param timeout: timeout in seconds to gracefully stop DMLProc - TODO: for next releases - :type timeout: int + :param timeout: timeout in seconds to gracefully stop DMLProc, + defaults to None + :type timeout: Optional[int], optional :raises CMAPIBasicError: if no nodes in the cluster :return: start timestamp :rtype: dict @@ -229,21 +227,6 @@ class ClusterHandler(): f'Cluster remove node command called. Removing node {node}.' ) response = {'timestamp': str(datetime.now())} - transaction_id = get_id() - - try: - suceeded, transaction_id, txn_nodes = start_transaction( - cs_config_filename=config, remove_nodes=[node], - txn_id=transaction_id - ) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while starting the transaction.' - ) from err - if not suceeded: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError('Starting transaction isn\'t successful.') try: remove_node( @@ -251,50 +234,31 @@ class ClusterHandler(): output_config_filename=config ) except Exception as err: - rollback_transaction( - transaction_id, nodes=txn_nodes, cs_config_filename=config - ) raise CMAPIBasicError('Error while removing node.') from err response['node_id'] = node - if len(txn_nodes) > 0: + active_nodes = get_active_nodes(config) + if len(active_nodes) > 0: update_revision_and_manager( input_config_filename=config, output_config_filename=config ) try: broadcast_successful = broadcast_new_config( - config, nodes=txn_nodes + config, nodes=active_nodes ) except Exception as err: - rollback_transaction( - transaction_id, nodes=txn_nodes, cs_config_filename=config - ) raise CMAPIBasicError( 'Error while distributing config file.' ) from err if not broadcast_successful: - rollback_transaction( - transaction_id, nodes=txn_nodes, cs_config_filename=config - ) raise CMAPIBasicError('Config distribution isn\'t successful.') - try: - commit_transaction(transaction_id, cs_config_filename=config) - except Exception as err: - rollback_transaction( - transaction_id, nodes=txn_nodes, cs_config_filename=config - ) - raise CMAPIBasicError( - 'Error while committing transaction.' - ) from err - logger.debug(f'Successfully finished removing node {node}.') return response @staticmethod def set_mode( - mode: str, timeout:int = 60, config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server') + mode: str, timeout: int = 60, config: str = DEFAULT_MCS_CONF_PATH, ) -> dict: """Method to set MCS CLuster mode. @@ -303,8 +267,6 @@ class ClusterHandler(): :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional :raises CMAPIBasicError: if no master found in the cluster :raises CMAPIBasicError: on exception while starting transaction :raises CMAPIBasicError: if transaction start isn't successful @@ -315,6 +277,7 @@ class ClusterHandler(): :return: result of adding node :rtype: dict """ + logger: logging.Logger = logging.getLogger('cmapi_server') logger.debug( f'Cluster mode set command called. Setting mode to {mode}.' ) @@ -323,7 +286,6 @@ class ClusterHandler(): cmapi_cfg_parser = get_config_parser(CMAPI_CONF_PATH) api_key = get_current_key(cmapi_cfg_parser) headers = {'x-api-key': api_key} - transaction_id = get_id() master = None if len(get_active_nodes(config)) != 0: @@ -359,7 +321,6 @@ class ClusterHandler(): def set_api_key( api_key: str, verification_key: str, config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server') ) -> dict: """Method to set API key for each CMAPI node in cluster. @@ -370,13 +331,12 @@ class ClusterHandler(): :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional :raises CMAPIBasicError: if catch some exception while setting API key to each node :return: status result :rtype: dict """ + logger: logging.Logger = logging.getLogger('cmapi_server') logger.debug('Cluster set API key command called.') active_nodes = get_active_nodes(config) diff --git a/cmapi/cmapi_server/helpers.py b/cmapi/cmapi_server/helpers.py index 9348393ee..dc12c55f8 100644 --- a/cmapi/cmapi_server/helpers.py +++ b/cmapi/cmapi_server/helpers.py @@ -290,7 +290,7 @@ def broadcast_new_config( sm_config_filename: str = DEFAULT_SM_CONF_PATH, test_mode: bool = False, nodes: Optional[list] = None, - timeout: int = 10 + timeout: Optional[int] = None ) -> bool: """Send new config to nodes. Now in async way. @@ -491,7 +491,7 @@ def save_cmapi_conf_file(cfg_parser, config_filepath: str = CMAPI_CONF_PATH): ) -def get_active_nodes(config:str = DEFAULT_MCS_CONF_PATH) -> list: +def get_active_nodes(config: str = DEFAULT_MCS_CONF_PATH) -> list: """Get active nodes from Columnstore.xml. Actually this is only names of nodes by which node have been added. diff --git a/cmapi/cmapi_server/managers/transaction.py b/cmapi/cmapi_server/managers/transaction.py index cffb71386..68bb7bc77 100644 --- a/cmapi/cmapi_server/managers/transaction.py +++ b/cmapi/cmapi_server/managers/transaction.py @@ -25,16 +25,34 @@ class TransactionManager(ContextDecorator): :type txn_id: Optional[int], optional :param handle_signals: handle specific signals or not, defaults to False :type handle_signals: bool, optional + :param extra_nodes: extra nodes to start transaction at, defaults to None + :type extra_nodes: Optional[list], optional + :param remove_nodes: nodes to remove from transaction, defaults to None + :type remove_nodes: Optional[list], optional + :param optional_nodes: nodes to add to transaction, defaults to None + :type optional_nodes: Optional[list], optional + :raises CMAPIBasicError: if there are no nodes in the cluster + :raises CMAPIBasicError: if starting transaction isn't succesful + :raises Exception: if error while starting the transaction + :raises Exception: if error while committing transaction + :raises Exception: if error while rollback transaction """ def __init__( self, timeout: float = TRANSACTION_TIMEOUT, - txn_id: Optional[int] = None, handle_signals: bool = False + txn_id: Optional[int] = None, handle_signals: bool = False, + extra_nodes: Optional[list] = None, + remove_nodes: Optional[list] = None, + optional_nodes: Optional[list] = None, ): self.timeout = timeout self.txn_id = txn_id or get_id() self.handle_signals = handle_signals self.active_transaction = False + self.extra_nodes = extra_nodes + self.remove_nodes = remove_nodes + self.optional_nodes = optional_nodes + self.success_txn_nodes = None def _handle_exception( self, exc: Optional[Type[Exception]] = None, @@ -53,7 +71,7 @@ class TransactionManager(ContextDecorator): """ # message = 'Got exception in transaction manager' if (exc or signum) and self.active_transaction: - self.rollback_transaction() + self.rollback_transaction(nodes=self.success_txn_nodes) self.set_default_signals() raise exc @@ -79,10 +97,14 @@ class TransactionManager(ContextDecorator): signal(SIGTERM, SIG_DFL) signal(SIGHUP, SIG_DFL) - def rollback_transaction(self) -> None: - """Rollback transaction.""" + def rollback_transaction(self, nodes: Optional[list] = None) -> None: + """Rollback transaction. + + :param nodes: nodes to rollback transaction, defaults to None + :type nodes: Optional[list], optional + """ try: - rollback_transaction(self.txn_id) + rollback_transaction(self.txn_id, nodes=nodes) self.active_transaction = False logging.debug(f'Success rollback of transaction "{self.txn_id}".') except Exception: @@ -91,15 +113,20 @@ class TransactionManager(ContextDecorator): exc_info=True ) - def commit_transaction(self): - """Commit transaction.""" + def commit_transaction(self, nodes: Optional[list] = None) -> None: + """Commit transaction. + + :param nodes: nodes to commit transaction, defaults to None + :type nodes: Optional[list], optional + """ try: commit_transaction( - self.txn_id, cs_config_filename=DEFAULT_MCS_CONF_PATH + self.txn_id, cs_config_filename=DEFAULT_MCS_CONF_PATH, + nodes=nodes ) except Exception: logging.error(f'Error while committing transaction {self.txn_id}') - self.rollback_transaction() + self.rollback_transaction(nodes=self.success_txn_nodes) self.set_default_signals() raise @@ -107,9 +134,11 @@ class TransactionManager(ContextDecorator): if self.handle_signals: self.set_custom_signals() try: - suceeded, _transaction_id, successes = start_transaction( + suceeded, _, success_txn_nodes = start_transaction( cs_config_filename=DEFAULT_MCS_CONF_PATH, - txn_id=self.txn_id, timeout=self.timeout + extra_nodes=self.extra_nodes, remove_nodes=self.remove_nodes, + optional_nodes=self.optional_nodes, + txn_id=self.txn_id, timeout=self.timeout, ) except Exception as exc: logging.error('Error while starting the transaction.') @@ -118,19 +147,26 @@ class TransactionManager(ContextDecorator): self._handle_exception( exc=CMAPIBasicError('Starting transaction isn\'t succesful.') ) - if suceeded and len(successes) == 0: - self._handle_exception( - exc=CMAPIBasicError('There are no nodes in the cluster.') - ) + if suceeded and len(success_txn_nodes) == 0: + # corner case when deleting last node in the cluster + # TODO: remove node mechanics potentially has a vulnerability + # because no transaction started for removing node. + # Probably in some cases rollback never works for removing + # node, because it never exist in success_txn_nodes. + if not self.remove_nodes: + self._handle_exception( + exc=CMAPIBasicError('There are no nodes in the cluster.') + ) self.active_transaction = True + self.success_txn_nodes = success_txn_nodes return self def __exit__(self, *exc): if exc[0] and self.active_transaction: - self.rollback_transaction() + self.rollback_transaction(nodes=self.success_txn_nodes) self.set_default_signals() return False if self.active_transaction: - self.commit_transaction() + self.commit_transaction(nodes=self.success_txn_nodes) self.set_default_signals() return True diff --git a/cmapi/cmapi_server/node_manipulation.py b/cmapi/cmapi_server/node_manipulation.py index d4e9240ed..756b49a74 100644 --- a/cmapi/cmapi_server/node_manipulation.py +++ b/cmapi/cmapi_server/node_manipulation.py @@ -61,7 +61,7 @@ def switch_node_maintenance( def add_node( node: str, input_config_filename: str = DEFAULT_MCS_CONF_PATH, output_config_filename: Optional[str] = None, - rebalance_dbroots: bool = True + use_rebalance_dbroots: bool = True ): """Add node to a cluster. @@ -86,8 +86,8 @@ def add_node( :type input_config_filename: str, optional :param output_config_filename: mcs output config path, defaults to None :type output_config_filename: Optional[str], optional - :param rebalance_dbroots: rebalance dbroots or not, defaults to True - :type rebalance_dbroots: bool, optional + :param use_rebalance_dbroots: rebalance dbroots or not, defaults to True + :type use_rebalance_dbroots: bool, optional """ node_config = NodeConfig() c_root = node_config.get_current_config_root(input_config_filename) @@ -100,7 +100,7 @@ def add_node( _add_Module_entries(c_root, node) _add_active_node(c_root, node) _add_node_to_ExeMgrs(c_root, node) - if rebalance_dbroots: + if use_rebalance_dbroots: _rebalance_dbroots(c_root) _move_primary_node(c_root) except Exception: @@ -116,25 +116,41 @@ def add_node( node_config.write_config(c_root, filename=output_config_filename) -# deactivate_only is a bool that indicates whether the node is being removed completely from -# the cluster, or whether it has gone offline and should still be monitored in case it comes back. -# Note! this does not pick a new primary node, use the move_primary_node() fcn to change that. def remove_node( - node, input_config_filename=DEFAULT_MCS_CONF_PATH, - output_config_filename=None, deactivate_only=False, - rebalance_dbroots = True, **kwargs + node: str, input_config_filename: str = DEFAULT_MCS_CONF_PATH, + output_config_filename: Optional[str] = None, + deactivate_only: bool = True, + use_rebalance_dbroots: bool = True, **kwargs ): + """Remove node from a cluster. + + - Rebuild the PMS section w/o node + - Remove the DBRM_Worker entry + - Remove the WES entry + - Rebuild the "Module*" entries w/o node + - Update the list of active / inactive / desired nodes + + :param node: node address or hostname + :type node: str + :param input_config_filename: mcs input config path, + defaults to DEFAULT_MCS_CONF_PATH + :type input_config_filename: str, optional + :param output_config_filename: mcs output config path, defaults to None + :type output_config_filename: Optional[str], optional + :param deactivate_only: indicates whether the node is being removed + completely from the cluster, or whether it has gone + offline and should still be monitored in case it + comes back. + Note! this does not pick a new primary node, + use the move_primary_node() fcn to change that., + defaults to True + :type deactivate_only: bool, optional + :param use_rebalance_dbroots: rebalance dbroots or not, defaults to True + :type use_rebalance_dbroots: bool, optional + """ node_config = NodeConfig() c_root = node_config.get_current_config_root(input_config_filename) - ''' - Rebuild the PMS section w/o node - Remove the DBRM_Worker entry - Remove the WES entry - Rebuild the "Module*" entries w/o node - Update the list of active / inactive / desired nodes - ''' - try: active_nodes = helpers.get_active_nodes(input_config_filename) @@ -151,7 +167,7 @@ def remove_node( # TODO: unspecific name, need to think of a better one _remove_node(c_root, node) - if rebalance_dbroots: + if use_rebalance_dbroots: _rebalance_dbroots(c_root) _move_primary_node(c_root) else: diff --git a/cmapi/cmapi_server/test/config_apply_example.py b/cmapi/cmapi_server/test/config_apply_example.py index 54cea2214..bc1c59dd1 100644 --- a/cmapi/cmapi_server/test/config_apply_example.py +++ b/cmapi/cmapi_server/test/config_apply_example.py @@ -7,11 +7,11 @@ from datetime import datetime from cmapi_server.controllers.dispatcher import _version config_filename = './cmapi_server/cmapi_server.conf' - + url = f"https://localhost:8640/cmapi/{_version}/node/config" begin_url = f"https://localhost:8640/cmapi/{_version}/node/begin" config_path = './cmapi_server/test/Columnstore_apply_config.xml' - + # create tmp dir tmp_prefix = '/tmp/mcs_config_test' tmp_path = Path(tmp_prefix) @@ -43,8 +43,3 @@ body = { 'timeout': 0, 'config': config, } - -#print(config) - -#r = requests.put(url, verify=False, headers=headers, json=body) - diff --git a/cmapi/mcs_cluster_tool/cluster_app.py b/cmapi/mcs_cluster_tool/cluster_app.py index 0b23ff021..6f4ddc85f 100644 --- a/cmapi/mcs_cluster_tool/cluster_app.py +++ b/cmapi/mcs_cluster_tool/cluster_app.py @@ -22,7 +22,7 @@ from cmapi_server.helpers import ( from cmapi_server.managers.transaction import TransactionManager from mcs_cluster_tool.decorators import handle_output from mcs_node_control.models.node_config import NodeConfig -from cmapi.cmapi_server.controllers.api_clients import ClusterControllerClient +from cmapi_server.controllers.api_clients import ClusterControllerClient logger = logging.getLogger('mcs_cli') @@ -191,9 +191,6 @@ def restart(): @node_app.command(rich_help_panel='cluster node commands') @handle_output -@TransactionManager( - timeout=timedelta(days=1).total_seconds(), handle_signals=True -) def add( nodes: Optional[List[str]] = typer.Option( ..., @@ -206,8 +203,12 @@ def add( ): """Add nodes to the Columnstore cluster.""" result = [] - for node in nodes: - result.append(client.add_node({'node': node})) + with TransactionManager( + timeout=timedelta(days=1).total_seconds(), handle_signals=True, + extra_nodes=nodes + ): + for node in nodes: + result.append(client.add_node({'node': node})) return result @@ -224,8 +225,12 @@ def remove(nodes: Optional[List[str]] = typer.Option( ): """Remove nodes from the Columnstore cluster.""" result = [] - for node in nodes: - result.append(client.remove_node(node)) + with TransactionManager( + timeout=timedelta(days=1).total_seconds(), handle_signals=True, + remove_nodes=nodes + ): + for node in nodes: + result.append(client.remove_node(node)) return result @@ -265,6 +270,7 @@ def api_key(key: str = typer.Option(..., help='API key to set.')): return client.set_api_key(key) +#TODO: remove in next releases @set_app.command() @handle_output def log_level(level: str = typer.Option(..., help='Logging level to set.')): From d198ff77f2dedd833cfae3f311f80eb9694dd13b Mon Sep 17 00:00:00 2001 From: Allen Herrera Date: Fri, 14 Mar 2025 12:04:25 -0400 Subject: [PATCH 58/65] adjusted list flag to --list in mcs_backup_manager.sh --- cmapi/scripts/mcs_backup_manager.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/cmapi/scripts/mcs_backup_manager.sh b/cmapi/scripts/mcs_backup_manager.sh index af1a1f2c0..3146db304 100644 --- a/cmapi/scripts/mcs_backup_manager.sh +++ b/cmapi/scripts/mcs_backup_manager.sh @@ -13,7 +13,7 @@ # ######################################################################## # Documentation: bash mcs_backup_manager.sh help -# Version: 3.12 +# Version: 3.13 # # Backup Example # LocalStorage: sudo ./mcs_backup_manager.sh backup @@ -26,7 +26,7 @@ # S3: sudo ./mcs_backup_manager.sh restore -bb s3://my-cs-backups -l # ######################################################################## -mcs_bk_manager_version="3.12" +mcs_bk_manager_version="3.13" start=$(date +%s) action=$1 @@ -328,7 +328,7 @@ parse_backup_variables() { apply_retention_only=true shift # past argument ;; - "list") + -li | --list) list_backups=true shift # past argument ;; @@ -378,6 +378,7 @@ print_backup_help_text() { -nb | --name-backup Define the name of the backup - default: date +%m-%d-%Y -r | --retention-days Retain backups created within the last X days, the rest are deleted, default 0 = keep all backups -aro | --apply-retention-only Only apply retention policy to existing backups, does not run a backup + -li | --list List all backups available in the backup location -ha | --highavilability Hint wether shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) HA S3 ( /var/lib/columnstore/storagemanager/ ) @@ -386,6 +387,7 @@ print_backup_help_text() { ./$0 backup -bl /tmp/backups/ ./$0 backup -bl /tmp/backups/ -P 8 ./$0 backup -bl /tmp/backups/ --incremental auto_most_recent + ./$0 backup -bl /tmp/backups/ --list ./$0 backup -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -s LocalStorage S3 Examples: @@ -2227,7 +2229,7 @@ parse_restore_variables() { no_verify_ssl=true shift # past argument ;; - "list") + -li | --list) list_backups=true shift # past argument ;; @@ -2274,6 +2276,7 @@ print_restore_help_text() -sb | --skip-bucket-data Skip restoring columnstore data in the bucket - ideal if looking to only restore mariadb server -q | --quiet Silence verbose copy command outputs -c | --compress Hint that the backup is compressed in X format - Options: [ pigz ] + -li | --list List available backups in the backup location -ha | --highavilability Hint for if shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) HA S3 ( /var/lib/columnstore/storagemanager/ ) @@ -3030,10 +3033,12 @@ print_dbrm_backup_help_text() { -bl | --backup-location Path of where to save the dbrm backups on disk -nb | --name-backup Define the prefix of the backup - default: dbrm_backup+date +%Y%m%d_%H%M%S -ssm | --skip-storage-manager skip backing up storagemanager directory + -li | --list List available dbrm backups in the backup location Default: ./$0 dbrm_backup -m once --retention-days 0 --backup-location /tmp/dbrm_backups Examples: + ./$0 dbrm_backup --list ./$0 dbrm_backup --backup-location /mnt/columnstore/dbrm_backups ./$0 dbrm_backup --retention-days 7 --backup-location /mnt/dbrm_backups --mode once -nb my-one-off-backup-before-upgrade ./$0 dbrm_backup --retention-days 7 --backup-location /mnt/dbrm_backups --mode loop --interval 90 @@ -3052,10 +3057,12 @@ print_dbrm_restore_help_text() { -ns | --no-start Do not attempt columnstore startup post dbrm_restore -sdbk| --skip-dbrm-backup Skip backing up dbrms brefore restoring -ssm | --skip-storage-manager Skip restoring storagemanager directory + -li | --list List available dbrm backups in the backup location Default: ./$0 dbrm_restore --backup-location /tmp/dbrm_backups Examples: + ./$0 dbrm_restore --list ./$0 dbrm_restore --backup-location /tmp/dbrm_backups --load dbrm_backup_20240318_172842 ./$0 dbrm_restore --backup-location /tmp/dbrm_backups --load dbrm_backup_20240318_172842 --no-start "; @@ -3103,7 +3110,7 @@ parse_dbrms_variables() { quiet=true shift # past argument ;; - "list") + -li | --list) list_dbrm_backups=true shift # past argument ;; @@ -3150,7 +3157,7 @@ parse_dbrm_restore_variables() { skip_storage_manager=true shift # past argument ;; - "list") + -li | --list) list_dbrm_backups=true shift # past argument ;; From 7c751ea53e293685107fd60d88ad282a997c9fd4 Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Wed, 19 Mar 2025 18:26:13 +0300 Subject: [PATCH 59/65] fix(cmapi): MCOL-5133 maintainance flag typo. --- cmapi/cmapi_server/handlers/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmapi/cmapi_server/handlers/cluster.py b/cmapi/cmapi_server/handlers/cluster.py index b4012a733..8b9c94882 100644 --- a/cmapi/cmapi_server/handlers/cluster.py +++ b/cmapi/cmapi_server/handlers/cluster.py @@ -37,7 +37,7 @@ def toggle_cluster_state( :type config: str """ if action == ClusterAction.START: - maintainance_flag = True + maintainance_flag = False elif action == ClusterAction.STOP: maintainance_flag = True else: From a60b278ee6ae45221fa20f996c68cc3a506d6fd7 Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Thu, 20 Mar 2025 15:14:40 +0300 Subject: [PATCH 60/65] fix(cmapi tests): MCOL-5133 test mode set without nodes now returns adequate message --- cmapi/cmapi_server/test/test_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmapi/cmapi_server/test/test_cluster.py b/cmapi/cmapi_server/test/test_cluster.py index f174f72ad..7af9226c8 100644 --- a/cmapi/cmapi_server/test/test_cluster.py +++ b/cmapi/cmapi_server/test/test_cluster.py @@ -147,7 +147,7 @@ class ClusterModesetTestCase(BaseClusterTestCase): ) error = resp.json()['error'] self.assertEqual(resp.status_code, 422) - self.assertEqual(error, 'No master found in the cluster.') + self.assertEqual(error, 'There are no nodes in the cluster.') def test_add_node_and_set_readonly(self): payload = {'node': socket.gethostname()} From 6a57537cb1fe4ecca55d19c2dd4741015f223898 Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Thu, 20 Mar 2025 15:14:40 +0300 Subject: [PATCH 61/65] fix(cmapi): MCOL-5133 remove has_active_nodes handler for cluster shutdown because TransactionManager has advance variant of this check --- cmapi/cmapi_server/controllers/endpoints.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmapi/cmapi_server/controllers/endpoints.py b/cmapi/cmapi_server/controllers/endpoints.py index c6b38b5d3..9727da9ca 100644 --- a/cmapi/cmapi_server/controllers/endpoints.py +++ b/cmapi/cmapi_server/controllers/endpoints.py @@ -157,7 +157,13 @@ def active_operation(): @cherrypy.tools.register('before_handler', priority=82) def has_active_nodes(): - """Check if there are any active nodes in the cluster.""" + """Check if there are any active nodes in the cluster. + + TODO: Remove in next releases due to never used. + Now TransactionManager has this check inside. + Before removing, have to check all API endpoints without transaction + mechanics to potential use of this handler. + """ active_nodes = get_active_nodes() if len(active_nodes) == 0: @@ -834,7 +840,6 @@ class ClusterController: @cherrypy.tools.json_in() @cherrypy.tools.json_out() @cherrypy.tools.validate_api_key() # pylint: disable=no-member - @cherrypy.tools.has_active_nodes() # pylint: disable=no-member def put_shutdown(self): func_name = 'put_shutdown' log_begin(module_logger, func_name) From 6e64d3a38dca11f38939820e07086398db47956c Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Fri, 14 Mar 2025 17:58:23 +0300 Subject: [PATCH 62/65] fix(mcs): MCOL-5618: change list option to --list and refactor. fix(mcs): list option to -li/--list for mcs backup, restore and dbrm_restore commands fix(mcs): add missed option "aro" fix(mcs): resort options for backup and dbrm_backup to keep same ordering as in original scrypt fix(mcs docs): updated README.md + mcs.1 --- cmapi/mcs_cluster_tool/README.md | 33 +++-- cmapi/mcs_cluster_tool/backup_commands.py | 155 ++++++++++++--------- cmapi/mcs_cluster_tool/mcs.1 | 52 ++++--- cmapi/mcs_cluster_tool/restore_commands.py | 24 ++-- 4 files changed, 148 insertions(+), 116 deletions(-) diff --git a/cmapi/mcs_cluster_tool/README.md b/cmapi/mcs_cluster_tool/README.md index 3fd286d51..57088117f 100644 --- a/cmapi/mcs_cluster_tool/README.md +++ b/cmapi/mcs_cluster_tool/README.md @@ -53,26 +53,28 @@ Name of the bucket to store the columnstore backups. Example: "s3://my-cs-backups" * `-url, --endpoint-url TEXT`: Used by on premise S3 vendors. Example: "http://127.0.0.1:8000" -* `-nv-ssl, --no-verify-ssl / -v-ssl, --verify-ssl`: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v-ssl] * `-s, --storage TEXT`: What storage topogoly is being used by Columnstore - found in /etc/columnstore/storagemanager.cnf. Options: "LocalStorage" or "S3" [default: LocalStorage] * `-i, --incremental TEXT`: Adds columnstore deltas to an existing full backup. Backup folder to apply increment could be a value or "auto_most_recent" - the incremental backup applies to last full backup. +* `-P, --parallel INTEGER`: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if "-c/--compress" argument not set. [default: 4] * `-ha, --highavilability / -no-ha, --no-highavilability`: Hint wether shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) HA S3 ( /var/lib/columnstore/storagemanager/ ) [default: no-ha] -* `-f, --config-file TEXT`: Path to backup configuration file to load variables from. +* `-f, --config-file TEXT`: Path to backup configuration file to load variables from - relative or full path accepted. * `-sbrm, --skip-save-brm / -no-sbrm, --no-skip-save-brm`: Skip saving brm prior to running a backup - ideal for dirty backups. [default: no-sbrm] * `-spoll, --skip-polls / -no-spoll, --no-skip-polls`: Skip sql checks confirming no write/cpimports running. [default: no-spoll] * `-slock, --skip-locks / -no-slock, --no-skip-locks`: Skip issuing write locks - ideal for dirty backups. [default: no-slock] * `-smdb, --skip-mariadb-backup / -no-smdb, --no-skip-mariadb-backup`: Skip running a mariadb-backup for innodb data - ideal for incremental dirty backups. [default: no-smdb] * `-sb, --skip-bucket-data / -no-sb, --no-skip-bucket-data`: Skip taking a copy of the columnstore data in the bucket. [default: no-sb] +* `-nb, --name-backup TEXT`: Define the name of the backup - default: $(date +%m-%d-%Y) [default: 03-20-2025] +* `-c, --compress TEXT`: Compress backup in X format - Options: [ pigz ]. +* `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] +* `-nv-ssl, --no-verify-ssl / -v-ssl, --verify-ssl`: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v-ssl] * `-pi, --poll-interval INTEGER`: Number of seconds between poll checks for active writes & cpimports. [default: 5] * `-pmw, --poll-max-wait INTEGER`: Max number of minutes for polling checks for writes to wait before exiting as a failed backup attempt. [default: 60] -* `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] -* `-c, --compress TEXT`: Compress backup in X format - Options: [ pigz ]. -* `-P, --parallel INTEGER`: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if "-c/--compress" argument not set. [default: 4] -* `-nb, --name-backup TEXT`: Define the name of the backup - default: $(date +%m-%d-%Y) [default: 03-06-2025] * `-r, --retention-days INTEGER`: Retain backups created within the last X days, default 0 == keep all backups. [default: 0] +* `-aro, --apply-retention-only`: Only apply retention policy to existing backups, does not run a backup. +* `-li, --list`: List backups. * `--help`: Show this message and exit. ## `mcs dbrm_backup` @@ -87,13 +89,14 @@ $ mcs dbrm_backup [OPTIONS] **Options**: -* `-m, --mode TEXT`: "loop" or "once" ; Determines if this script runs in a forever loop sleeping -i minutes or just once. [default: once] * `-i, --interval INTEGER`: Number of minutes to sleep when --mode=loop. [default: 90] * `-r, --retention-days INTEGER`: Retain dbrm backups created within the last X days, the rest are deleted [default: 7] -* `-p, --path TEXT`: Path of where to save the dbrm backups on disk. [default: /tmp/dbrm_backups] -* `-nb, --name-backup TEXT`: Custom name to prefex dbrm backups with. [default: dbrm_backup] -* `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] +* `-bl, --backup-location TEXT`: Path of where to save the dbrm backups on disk. [default: /tmp/dbrm_backups] +* `-m, --mode TEXT`: "loop" or "once" ; Determines if this script runs in a forever loop sleeping -i minutes or just once. [default: once] +* `-nb, --name-backup TEXT`: Define the prefix of the backup - default: dbrm_backup+date +%Y%m%d_%H%M%S [default: dbrm_backup] * `-ssm, --skip-storage-manager / -no-ssm, --no-skip-storage-manager`: Skip backing up storagemanager directory. [default: no-ssm] +* `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] +* `-li, --list`: List backups. * `--help`: Show this message and exit. ## `mcs restore` @@ -127,15 +130,16 @@ Options: "LocalStorage" or "S3" [default: LocalStorage] * `-nr, --new-region TEXT`: Defines the region of the new bucket to copy the s3 data to from the backup bucket. * `-nk, --new-key TEXT`: Defines the aws key to connect to the new_bucket. * `-ns, --new-secret TEXT`: Defines the aws secret of the aws key to connect to the new_bucket. +* `-P, --parallel INTEGER`: Determines number of decompression and mdbstream threads. Ignored if "-c/--compress" argument not set. [default: 4] * `-ha, --highavilability / -no-ha, --no-highavilability`: Flag for high available systems (meaning shared storage exists supporting the topology so that each node sees all data) [default: no-ha] * `-cont, --continue / -no-cont, --no-continue`: This acknowledges data in your --new_bucket is ok to delete when restoring S3. When set to true skips the enforcement that new_bucket should be empty prior to starting a restore. [default: no-cont] -* `-f, --config-file TEXT`: Path to backup configuration file to load variables from. +* `-f, --config-file TEXT`: Path to backup configuration file to load variables from - relative or full path accepted. * `-smdb, --skip-mariadb-backup / -no-smdb, --no-skip-mariadb-backup`: Skip restoring mariadb server via mariadb-backup - ideal for only restoring columnstore. [default: no-smdb] * `-sb, --skip-bucket-data / -no-sb, --no-skip-bucket-data`: Skip restoring columnstore data in the bucket - ideal if looking to only restore mariadb server. [default: no-sb] * `-c, --compress TEXT`: Hint that the backup is compressed in X format. Options: [ pigz ]. -* `-P, --parallel INTEGER`: Determines number of decompression and mdbstream threads. Ignored if "-c/--compress" argument not set. [default: 4] * `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] * `-nv-ssl, --no-verify-ssl / -v-ssl, --verify-ssl`: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v-ssl] +* `-li, --list`: List backups. * `--help`: Show this message and exit. ## `mcs dbrm_restore` @@ -150,11 +154,12 @@ $ mcs dbrm_restore [OPTIONS] **Options**: -* `-p, --path TEXT`: Path of where dbrm backups stored on disk. [default: /tmp/dbrm_backups] -* `-d, --directory TEXT`: Date or directory chose to restore from. +* `-bl, --backup-location TEXT`: Path of where dbrm backups exist on disk. [default: /tmp/dbrm_backups] +* `-l, --load TEXT`: Name of the directory to restore from -bl * `-ns, --no-start`: Do not attempt columnstore startup post dbrm_restore. * `-sdbk, --skip-dbrm-backup / -no-sdbk, --no-skip-dbrm-backup`: Skip backing up dbrms before restoring. [default: sdbk] * `-ssm, --skip-storage-manager / -no-ssm, --no-skip-storage-manager`: Skip backing up storagemanager directory. [default: ssm] +* `-li, --list`: List backups. * `--help`: Show this message and exit. ## `mcs help-all` diff --git a/cmapi/mcs_cluster_tool/backup_commands.py b/cmapi/mcs_cluster_tool/backup_commands.py index f9496dd94..39e4c734b 100644 --- a/cmapi/mcs_cluster_tool/backup_commands.py +++ b/cmapi/mcs_cluster_tool/backup_commands.py @@ -76,13 +76,6 @@ def backup( ) ) ] = '', - nv_ssl: Annotated[ - bool, - typer.Option( - '-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl', - help='Skips verifying ssl certs, useful for onpremise s3 storage.' - ) - ] = False, s: Annotated[ str, typer.Option( @@ -107,6 +100,18 @@ def backup( show_default=False ) ] = '', + P: Annotated[ + int, + typer.Option( + '-P', '--parallel', + help=( + 'Determines if columnstore data directories will have ' + 'multiple rsync running at the same time for different ' + 'subfolders to parallelize writes. ' + 'Ignored if "-c/--compress" argument not set.' + ) + ) + ] = 4, ha: Annotated[ bool, typer.Option( @@ -171,53 +176,6 @@ def backup( help='Skip taking a copy of the columnstore data in the bucket.' ) ] = False, - pi: Annotated[ - int, - typer.Option( - '-pi', '--poll-interval', - help=( - 'Number of seconds between poll checks for active writes & ' - 'cpimports.' - ) - ) - ] = 5, - pmw: Annotated[ - int, - typer.Option( - '-pmw', '--poll-max-wait', - help=( - 'Max number of minutes for polling checks for writes to wait ' - 'before exiting as a failed backup attempt.' - ) - ) - ] = 60, - q: Annotated[ - bool, - typer.Option( - '-q/-no-q', '--quiet/--no-quiet', - help='Silence verbose copy command outputs.' - ) - ] = False, - c: Annotated[ - str, - typer.Option( - '-c', '--compress', - help='Compress backup in X format - Options: [ pigz ].', - show_default=False - ) - ] = '', - P: Annotated[ - int, - typer.Option( - '-P', '--parallel', - help=( - 'Determines if columnstore data directories will have ' - 'multiple rsync running at the same time for different ' - 'subfolders to parallelize writes. ' - 'Ignored if "-c/--compress" argument not set.' - ) - ) - ] = 4, nb: Annotated[ str, typer.Option( @@ -238,6 +196,48 @@ def backup( hidden=True ) ] = 'direct', + c: Annotated[ + str, + typer.Option( + '-c', '--compress', + help='Compress backup in X format - Options: [ pigz ].', + show_default=False + ) + ] = '', + q: Annotated[ + bool, + typer.Option( + '-q/-no-q', '--quiet/--no-quiet', + help='Silence verbose copy command outputs.' + ) + ] = False, + nv_ssl: Annotated[ + bool, + typer.Option( + '-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl', + help='Skips verifying ssl certs, useful for onpremise s3 storage.' + ) + ] = False, + pi: Annotated[ + int, + typer.Option( + '-pi', '--poll-interval', + help=( + 'Number of seconds between poll checks for active writes & ' + 'cpimports.' + ) + ) + ] = 5, + pmw: Annotated[ + int, + typer.Option( + '-pmw', '--poll-max-wait', + help=( + 'Max number of minutes for polling checks for writes to wait ' + 'before exiting as a failed backup attempt.' + ) + ) + ] = 60, r: Annotated[ int, typer.Option( @@ -248,13 +248,23 @@ def backup( ) ) ] = 0, + aro: Annotated[ + bool, + typer.Option( + '-aro', '--apply-retention-only', + help=( + 'Only apply retention policy to existing backups, ' + 'does not run a backup.' + ) + ) + ] = False, list: Annotated[ bool, typer.Option( - 'list', + '-li', '--list', help='List backups.' ) - ] = False + ] = False, ): """Backup Columnstore and/or MariDB data.""" @@ -286,16 +296,6 @@ def backup( @handle_output def dbrm_backup( - m: Annotated[ - str, - typer.Option( - '-m', '--mode', - help=( - '"loop" or "once" ; Determines if this script runs in a ' - 'forever loop sleeping -i minutes or just once.' - ), - ) - ] = 'once', i: Annotated[ int, typer.Option( @@ -320,6 +320,16 @@ def dbrm_backup( help='Path of where to save the dbrm backups on disk.' ) ] = '/tmp/dbrm_backups', + m: Annotated[ + str, + typer.Option( + '-m', '--mode', + help=( + '"loop" or "once" ; Determines if this script runs in a ' + 'forever loop sleeping -i minutes or just once.' + ), + ) + ] = 'once', nb: Annotated[ str, typer.Option( @@ -330,6 +340,13 @@ def dbrm_backup( ) ) ] = 'dbrm_backup', + ssm: Annotated[ + bool, + typer.Option( + '-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager', + help='Skip backing up storagemanager directory.' + ) + ] = False, q: Annotated[ bool, typer.Option( @@ -337,11 +354,11 @@ def dbrm_backup( help='Silence verbose copy command outputs.' ) ] = False, - ssm: Annotated[ + list: Annotated[ bool, typer.Option( - '-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager', - help='Skip backing up storagemanager directory.' + '-li', '--list', + help='List backups.' ) ] = False, ): diff --git a/cmapi/mcs_cluster_tool/mcs.1 b/cmapi/mcs_cluster_tool/mcs.1 index 6b1e32ed2..c465f4a71 100644 --- a/cmapi/mcs_cluster_tool/mcs.1 +++ b/cmapi/mcs_cluster_tool/mcs.1 @@ -78,18 +78,18 @@ Example: \[dq]s3://my\-cs\-backups\[dq] \fB\fC\-url, \-\-endpoint\-url TEXT\fR: Used by on premise S3 vendors. Example: \[dq]\[la]http://127.0.0.1:8000\[ra]\[dq] .IP \(bu 2 -\fB\fC\-nv\-ssl, \-\-no\-verify\-ssl / \-v\-ssl, \-\-verify\-ssl\fR: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v\-ssl] -.IP \(bu 2 \fB\fC\-s, \-\-storage TEXT\fR: What storage topogoly is being used by Columnstore \- found in /etc/columnstore/storagemanager.cnf. Options: \[dq]LocalStorage\[dq] or \[dq]S3\[dq] [default: LocalStorage] .IP \(bu 2 \fB\fC\-i, \-\-incremental TEXT\fR: Adds columnstore deltas to an existing full backup. Backup folder to apply increment could be a value or \[dq]auto\fImost\fPrecent\[dq] \- the incremental backup applies to last full backup. .IP \(bu 2 +\fB\fC\-P, \-\-parallel INTEGER\fR: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if \[dq]\-c/\-\-compress\[dq] argument not set. [default: 4] +.IP \(bu 2 \fB\fC\-ha, \-\-highavilability / \-no\-ha, \-\-no\-highavilability\fR: Hint wether shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) HA S3 ( /var/lib/columnstore/storagemanager/ ) [default: no\-ha] .IP \(bu 2 -\fB\fC\-f, \-\-config\-file TEXT\fR: Path to backup configuration file to load variables from. +\fB\fC\-f, \-\-config\-file TEXT\fR: Path to backup configuration file to load variables from \- relative or full path accepted. .IP \(bu 2 \fB\fC\-sbrm, \-\-skip\-save\-brm / \-no\-sbrm, \-\-no\-skip\-save\-brm\fR: Skip saving brm prior to running a backup \- ideal for dirty backups. [default: no\-sbrm] .IP \(bu 2 @@ -101,20 +101,24 @@ HA S3 ( /var/lib/columnstore/storagemanager/ ) [default: no\-ha] .IP \(bu 2 \fB\fC\-sb, \-\-skip\-bucket\-data / \-no\-sb, \-\-no\-skip\-bucket\-data\fR: Skip taking a copy of the columnstore data in the bucket. [default: no\-sb] .IP \(bu 2 +\fB\fC\-nb, \-\-name\-backup TEXT\fR: Define the name of the backup \- default: $(date +%m\-%d\-%Y) [default: 03\-20\-2025] +.IP \(bu 2 +\fB\fC\-c, \-\-compress TEXT\fR: Compress backup in X format \- Options: [ pigz ]. +.IP \(bu 2 +\fB\fC\-q, \-\-quiet / \-no\-q, \-\-no\-quiet\fR: Silence verbose copy command outputs. [default: no\-q] +.IP \(bu 2 +\fB\fC\-nv\-ssl, \-\-no\-verify\-ssl / \-v\-ssl, \-\-verify\-ssl\fR: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v\-ssl] +.IP \(bu 2 \fB\fC\-pi, \-\-poll\-interval INTEGER\fR: Number of seconds between poll checks for active writes & cpimports. [default: 5] .IP \(bu 2 \fB\fC\-pmw, \-\-poll\-max\-wait INTEGER\fR: Max number of minutes for polling checks for writes to wait before exiting as a failed backup attempt. [default: 60] .IP \(bu 2 -\fB\fC\-q, \-\-quiet / \-no\-q, \-\-no\-quiet\fR: Silence verbose copy command outputs. [default: no\-q] -.IP \(bu 2 -\fB\fC\-c, \-\-compress TEXT\fR: Compress backup in X format \- Options: [ pigz ]. -.IP \(bu 2 -\fB\fC\-P, \-\-parallel INTEGER\fR: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if \[dq]\-c/\-\-compress\[dq] argument not set. [default: 4] -.IP \(bu 2 -\fB\fC\-nb, \-\-name\-backup TEXT\fR: Define the name of the backup \- default: $(date +%m\-%d\-%Y) [default: 03\-06\-2025] -.IP \(bu 2 \fB\fC\-r, \-\-retention\-days INTEGER\fR: Retain backups created within the last X days, default 0 == keep all backups. [default: 0] .IP \(bu 2 +\fB\fC\-aro, \-\-apply\-retention\-only\fR: Only apply retention policy to existing backups, does not run a backup. +.IP \(bu 2 +\fB\fC\-li, \-\-list\fR: List backups. +.IP \(bu 2 \fB\fC\-\-help\fR: Show this message and exit. .RE .SH \fB\fCmcs dbrm_backup\fR @@ -132,19 +136,21 @@ $ mcs dbrm_backup [OPTIONS] \fBOptions\fP: .RS .IP \(bu 2 -\fB\fC\-m, \-\-mode TEXT\fR: \[dq]loop\[dq] or \[dq]once\[dq] ; Determines if this script runs in a forever loop sleeping \-i minutes or just once. [default: once] -.IP \(bu 2 \fB\fC\-i, \-\-interval INTEGER\fR: Number of minutes to sleep when \-\-mode=loop. [default: 90] .IP \(bu 2 \fB\fC\-r, \-\-retention\-days INTEGER\fR: Retain dbrm backups created within the last X days, the rest are deleted [default: 7] .IP \(bu 2 -\fB\fC\-p, \-\-path TEXT\fR: Path of where to save the dbrm backups on disk. [default: /tmp/dbrm_backups] +\fB\fC\-bl, \-\-backup\-location TEXT\fR: Path of where to save the dbrm backups on disk. [default: /tmp/dbrm_backups] .IP \(bu 2 -\fB\fC\-nb, \-\-name\-backup TEXT\fR: Custom name to prefex dbrm backups with. [default: dbrm_backup] +\fB\fC\-m, \-\-mode TEXT\fR: \[dq]loop\[dq] or \[dq]once\[dq] ; Determines if this script runs in a forever loop sleeping \-i minutes or just once. [default: once] +.IP \(bu 2 +\fB\fC\-nb, \-\-name\-backup TEXT\fR: Define the prefix of the backup \- default: dbrm\fIbackup+date +%Y%m%d\fP%H%M%S [default: dbrm_backup] +.IP \(bu 2 +\fB\fC\-ssm, \-\-skip\-storage\-manager / \-no\-ssm, \-\-no\-skip\-storage\-manager\fR: Skip backing up storagemanager directory. [default: no\-ssm] .IP \(bu 2 \fB\fC\-q, \-\-quiet / \-no\-q, \-\-no\-quiet\fR: Silence verbose copy command outputs. [default: no\-q] .IP \(bu 2 -\fB\fC\-ssm, \-\-skip\-storage\-manager / \-no\-ssm, \-\-no\-skip\-storage\-manager\fR: Skip backing up storagemanager directory. [default: no\-ssm] +\fB\fC\-li, \-\-list\fR: List backups. .IP \(bu 2 \fB\fC\-\-help\fR: Show this message and exit. .RE @@ -195,11 +201,13 @@ Options: \[dq]LocalStorage\[dq] or \[dq]S3\[dq] [default: LocalStorage] .IP \(bu 2 \fB\fC\-ns, \-\-new\-secret TEXT\fR: Defines the aws secret of the aws key to connect to the new_bucket. .IP \(bu 2 +\fB\fC\-P, \-\-parallel INTEGER\fR: Determines number of decompression and mdbstream threads. Ignored if \[dq]\-c/\-\-compress\[dq] argument not set. [default: 4] +.IP \(bu 2 \fB\fC\-ha, \-\-highavilability / \-no\-ha, \-\-no\-highavilability\fR: Flag for high available systems (meaning shared storage exists supporting the topology so that each node sees all data) [default: no\-ha] .IP \(bu 2 \fB\fC\-cont, \-\-continue / \-no\-cont, \-\-no\-continue\fR: This acknowledges data in your \-\-new\fIbucket is ok to delete when restoring S3. When set to true skips the enforcement that new\fPbucket should be empty prior to starting a restore. [default: no\-cont] .IP \(bu 2 -\fB\fC\-f, \-\-config\-file TEXT\fR: Path to backup configuration file to load variables from. +\fB\fC\-f, \-\-config\-file TEXT\fR: Path to backup configuration file to load variables from \- relative or full path accepted. .IP \(bu 2 \fB\fC\-smdb, \-\-skip\-mariadb\-backup / \-no\-smdb, \-\-no\-skip\-mariadb\-backup\fR: Skip restoring mariadb server via mariadb\-backup \- ideal for only restoring columnstore. [default: no\-smdb] .IP \(bu 2 @@ -207,12 +215,12 @@ Options: \[dq]LocalStorage\[dq] or \[dq]S3\[dq] [default: LocalStorage] .IP \(bu 2 \fB\fC\-c, \-\-compress TEXT\fR: Hint that the backup is compressed in X format. Options: [ pigz ]. .IP \(bu 2 -\fB\fC\-P, \-\-parallel INTEGER\fR: Determines number of decompression and mdbstream threads. Ignored if \[dq]\-c/\-\-compress\[dq] argument not set. [default: 4] -.IP \(bu 2 \fB\fC\-q, \-\-quiet / \-no\-q, \-\-no\-quiet\fR: Silence verbose copy command outputs. [default: no\-q] .IP \(bu 2 \fB\fC\-nv\-ssl, \-\-no\-verify\-ssl / \-v\-ssl, \-\-verify\-ssl\fR: Skips verifying ssl certs, useful for onpremise s3 storage. [default: v\-ssl] .IP \(bu 2 +\fB\fC\-li, \-\-list\fR: List backups. +.IP \(bu 2 \fB\fC\-\-help\fR: Show this message and exit. .RE .SH \fB\fCmcs dbrm_restore\fR @@ -230,9 +238,9 @@ $ mcs dbrm_restore [OPTIONS] \fBOptions\fP: .RS .IP \(bu 2 -\fB\fC\-p, \-\-path TEXT\fR: Path of where dbrm backups stored on disk. [default: /tmp/dbrm_backups] +\fB\fC\-bl, \-\-backup\-location TEXT\fR: Path of where dbrm backups exist on disk. [default: /tmp/dbrm_backups] .IP \(bu 2 -\fB\fC\-d, \-\-directory TEXT\fR: Date or directory chose to restore from. +\fB\fC\-l, \-\-load TEXT\fR: Name of the directory to restore from \-bl .IP \(bu 2 \fB\fC\-ns, \-\-no\-start\fR: Do not attempt columnstore startup post dbrm_restore. .IP \(bu 2 @@ -240,6 +248,8 @@ $ mcs dbrm_restore [OPTIONS] .IP \(bu 2 \fB\fC\-ssm, \-\-skip\-storage\-manager / \-no\-ssm, \-\-no\-skip\-storage\-manager\fR: Skip backing up storagemanager directory. [default: ssm] .IP \(bu 2 +\fB\fC\-li, \-\-list\fR: List backups. +.IP \(bu 2 \fB\fC\-\-help\fR: Show this message and exit. .RE .SH \fB\fCmcs help\-all\fR diff --git a/cmapi/mcs_cluster_tool/restore_commands.py b/cmapi/mcs_cluster_tool/restore_commands.py index ba0ad3533..288998537 100644 --- a/cmapi/mcs_cluster_tool/restore_commands.py +++ b/cmapi/mcs_cluster_tool/restore_commands.py @@ -144,6 +144,16 @@ def restore( ) ) ] = '', + P: Annotated[ + int, + typer.Option( + '-P', '--parallel', + help=( + 'Determines number of decompression and mdbstream threads. ' + 'Ignored if "-c/--compress" argument not set.' + ) + ) + ] = 4, ha: Annotated[ bool, typer.Option( @@ -221,16 +231,6 @@ def restore( show_default=False ) ] = '', - P: Annotated[ - int, - typer.Option( - '-P', '--parallel', - help=( - 'Determines number of decompression and mdbstream threads. ' - 'Ignored if "-c/--compress" argument not set.' - ) - ) - ] = 4, q: Annotated[ bool, typer.Option( @@ -248,7 +248,7 @@ def restore( list: Annotated[ bool, typer.Option( - 'list', + '-li', '--list', help='List backups.' ) ] = False @@ -318,7 +318,7 @@ def dbrm_restore( list: Annotated[ bool, typer.Option( - 'list', + '-li', '--list', help='List backups.' ) ] = False From 61b8973fa8223a07c4620a3249f45b8f0dccf16f Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Sat, 22 Mar 2025 02:40:21 +0200 Subject: [PATCH 63/65] add branches info to buildinfo file --- .drone.jsonnet | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.drone.jsonnet b/.drone.jsonnet index 0334bcbd3..00bc2e7de 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -844,11 +844,19 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') status: ['success', 'failure'], }, volumes: [pipeline._volumes.mdb], + environment: { + SERVER_REF: '${SERVER_REF:-' + server + '}', + SERVER_REMOTE: '${SERVER_REMOTE:-' + server_remote + '}', + }, commands: [ 'cd /mdb/' + builddir, 'echo "engine: $DRONE_COMMIT" > buildinfo.txt', 'echo "server: $$(git rev-parse HEAD)" >> buildinfo.txt', 'echo "buildNo: $DRONE_BUILD_NUMBER" >> buildinfo.txt', + 'echo "serverBranch: $$SERVER_REF" >> buildinfo.txt', + 'echo "serverRepo: $$SERVER_REMOTE" >> buildinfo.txt', + 'echo "engineBranch: $DRONE_SOURCE_BRANCH" >> buildinfo.txt', + 'echo "engineRepo: https://github.com/$DRONE_REPO" >> buildinfo.txt', 'mv buildinfo.txt ./%s/' % result, 'yes | cp -vr ./%s/. /drone/src/%s/' % [result, result], 'ls -l /drone/src/' + result, From 66778933ceb2194f32aa447d68f7d4040506c9de Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Tue, 21 Nov 2023 16:11:29 +0300 Subject: [PATCH 64/65] A start --- docs/formal-review.md | 146 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 docs/formal-review.md diff --git a/docs/formal-review.md b/docs/formal-review.md new file mode 100644 index 000000000..c248020d9 --- /dev/null +++ b/docs/formal-review.md @@ -0,0 +1,146 @@ +# Formal review checklist and considerations + +> "The cost of fixing a defect is proportional to the time between defect introduction and discovery" - main axiom of softwaer engineering. + +## Purpose of the document and intented audience + +The purpose of this document is to list common defects and their sources to prevent their introduction into a code base. + +It is advisory for Columnstore developers to consult this document while doing personal erview of the code to be published. We also should extend this document as new sources of defects come to light. + + +# Performance related issues + +## Assignment: x = y; + +If x is a std::string and y is a pointer, y is treated as an ASCIIZ string. There is implicit strlen call in there. + + +# Side effects + + +## Copy-paste errors + +Copy-pasted code can contain obvious and non-obvious side effects. These side effects should be controlled for, especially when logging. + +### Obvious side effects + +Pre- and post-increments: avoid copying these verbatim. + +### Non-obvious side effects + +Getters and attribute computations may have side-effecting computations inside them, although rare. + +One of the effects can be different implicit conversions within different contexts. + + +## Changed code errors + +Code changes can result in defects. + +I should look at the semantics of the code that is moved or otherwise refactored. It is possible to have some non-trivial implicit transformations changed and/or deleted. + +If you changed the method, review use sites of this method. + +# New code errors + +## Side effects + +### Data invariants + +Objects should be constructed with invariants satisfied. If possible, there should be no invariant exceptions for “uninitialized” objects. + +### Create do not reuse + +If it is possible, I should construct new objects instead of reusing existing ones. + +### Separate inputs and outputs + +If possible, signify outputs in the result and inputs in arguments. Reference arguments should be avoided as much as possible, especially in new code. + +### Type (conversion) errors + +C++ has a standardized type constraint solution engine and it is possible to have a program logic error with type conversions and assignments. Myself, I prefer not to implement assignment and conversion operators (and operators at all, if that’s possible) but other’s code can contain these. + + +### Change of structure of one of the types + +Usually, the deletion or change of field is flagged in compiler errors. The addition of fields is not tracked at all, thus, this is the source of errors in assignment operators and other structure-copying mechanisms. + +Tracking can be implemented with the change of class/type’s name, then reverting the change back when everything is done. + +# Structure of code - design + +## Execution should avoid interpretation + +Execution of some action should avoid conditions on the data as much as possible. + +This warrants a concrete example. + +Instead of + +``` +if (condition on the fields) + { auto p = createThisObj(...); p-> bubu(); } +else { auto p = createThatObj(...); p->gaga() } +``` + +We should create object with either executor (or both) and execute them conditionally: + +``` +If (thisObjBubuExec) { thisObjBubuExec->bubu(); } +If (thatObjGagaExec) { thatObjGagaExec->gaga(); } +``` + +First variant pushes optimizations and rewrites of execution down to the execution phase, instead of pulling them up into optimization and rewrite phase. + +## Execution must be stupid. It is the optimization that must be smart. + +### Templates execute, not decide + +It is hard to inspect control decisions inside templated code. For example, not all template parametrizations may have serialization to streams implemented. Because of that, templated code should have as few decisions as possible. + +It should be preferred to call templates with all decisions made elsewhere and for templates to execute only simple decisions like loops. + +## Inheritance should be shallow + +Liskov substitution principle usually can be realized several ways. Classical example is the Square and Rectangle inheritance - which one should inherit the other? One can argue both ways, that Rectangle adds dimension and should inherit Square and that Square is a constrained Rectangle and thus should be inherited from. + +Usually inheritance direction depends on the use cases present at the time of the design. But, use cases after a while can change - some can be no longer valid, some can be added. + +Inheritance adds constraints through LOGICAL AND: inheriting class supports all constraints of the parent class AND adds its own. Classes at the same level of inheritance combine constraints through the LOGICAL OR: each child of a parent class can be used as a parent class. + +The structures of these AND and OR connections are fixed in hierarchies, often quite sturdily. + +It is much easier to design shallow hierarchies. It is much easier to modify shallow hierarchies. + +The most simple hierarchy is a base class, preferably abstract, and exactly one level of inheritance. It is a direct analogue of an Algebraic Data Types from many functional languages and it expresses a SUM (LOGICAL OR) of PRODUCTS (LOGICAL AND). + +I should prefer not to use class hierarchy of any kind + +Learning from MCS’ code, I see a great deal of copy-paste. + +# Miscellanea + +## Looping and conditionals - use code blocks + +Loops and conditional statements should always delimit their bodies with curly brackets. E.g., there should not be single statements in bodies, there should be compound statements or code blocks. + +This makes conditions and loops heavier but it is one of the points: you should make less conditionals and loops. + +This also helps with the refactoring as the conditional or loop body is clearly delimited. + +## (De)serialization + +Serialization and deserialization should go through intermediate types. + +This stems from having adhoc serdes code, where I can receive strings from NullStrings sent. If I have a separate message type where I can change strings to NullString, I will get compilation errors indicating changes needed instead of tests failing for inability to find the table it just created. + +## Same errors + +When fixing an error, look around for the presence of the error of the same type. Addition may not be added everywhere it needed, deletion may not delete everything that has to be deleted. + +## No generic getters/setters + +These are often superfluous or may violate class contracts when used incorrectly. Also, there is usually now way to find the correct way to use them. + From f877b1784120da4deee846f5252470167fcdae06 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Tue, 19 Nov 2024 13:38:54 +0000 Subject: [PATCH 65/65] MCOL-5722 Tool to move meta from files to fdb for existing installation. --- tools/CMakeLists.txt | 3 +- tools/updateMeta/CMakeLists.txt | 6 ++ tools/updateMeta/updateMeta.cpp | 141 ++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 tools/updateMeta/CMakeLists.txt create mode 100644 tools/updateMeta/updateMeta.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index fee47158a..bba29ca4e 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -14,4 +14,5 @@ add_subdirectory(rebuildEM) add_subdirectory(passwd) add_subdirectory(configMgt) add_subdirectory(parquetGen) -add_subdirectory(parquetDDL) \ No newline at end of file +add_subdirectory(parquetDDL) +add_subdirectory(updateMeta) diff --git a/tools/updateMeta/CMakeLists.txt b/tools/updateMeta/CMakeLists.txt new file mode 100644 index 000000000..f4f923fc1 --- /dev/null +++ b/tools/updateMeta/CMakeLists.txt @@ -0,0 +1,6 @@ +include_directories(${ENGINE_COMMON_INCLUDES}) + +set(updateMeta_SRCS updateMeta.cpp) +add_executable(updateMeta ${updateMeta_SRCS}) +target_link_libraries(updateMeta ${ENGINE_LDFLAGS} fdbcs) +install(TARGETS updateMeta DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/tools/updateMeta/updateMeta.cpp b/tools/updateMeta/updateMeta.cpp new file mode 100644 index 000000000..ba79daad1 --- /dev/null +++ b/tools/updateMeta/updateMeta.cpp @@ -0,0 +1,141 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#include +#include "fdbcs.hpp" +#include +#include + +using namespace std; +std::unique_ptr fdbNetworkInstance; + +static std::shared_ptr getStorageInstance() +{ + if (!FDBCS::setAPIVersion()) + { + cerr << "Update meta: FDB setAPIVersion failed." << endl; + return nullptr; + } + + fdbNetworkInstance = std::make_unique(); + if (!fdbNetworkInstance->setUpAndRunNetwork()) + { + cerr << "Update meta: FDB setUpAndRunNetwork failed." << endl; + return nullptr; + } + + std::string clusterFilePath = "/etc/foundationdb/fdb.cluster"; + auto fdbDataBaseInstance = FDBCS::DataBaseCreator::createDataBase(clusterFilePath); + if (!fdbDataBaseInstance) + { + cerr << "Update meta: FDB createDataBase failed." << endl; + return nullptr; + } + + return fdbDataBaseInstance; +} + +class MetaCollector +{ + public: + MetaCollector(const std::string& metaPath) : metaPath(metaPath) + { + } + + bool collect(const std::string& dbRoot) + { + try + { + auto path = metaPath + dbRoot; + for (auto const& file : std::filesystem::recursive_directory_iterator{path}) + { + if (std::filesystem::is_regular_file(file)) + { + files.push_back(file.path()); + } + } + } + catch (std::exception& e) + { + std::cerr << e.what() << std::endl; + return false; + } + return true; + } + + bool commit() + { + auto kvStorage = getStorageInstance(); + if (!kvStorage) + { + cerr << "Cannot get a storage instance" << endl; + return false; + } + auto keyGen = std::make_shared(); + + for (const auto& file : files) + { + auto fileName = file.string(); + cout << fileName << endl; + ifstream iFile(fileName); + if (!iFile.is_open()) + return false; + + std::stringstream metaDataStream; + metaDataStream << iFile.rdbuf(); + FDBCS::BlobHandler blobWriter(keyGen); + auto metaKey = metaPrefix + fileName; + if (!blobWriter.writeBlob(kvStorage, metaKey, metaDataStream.str())) + { + return false; + } + } + return true; + } + + private: + std::vector files; + string metaPrefix = "SM_M_"; + string metaPath; + string dbRoot; +}; + +int main(int argc, char** argv) +{ + if (argc != 2) + { + cout << "Have to provide dbroot name. For example: updateMeta data1 " << endl; + return 0; + } + std::string metaPath = "/var/lib/columnstore/storagemanager/metadata/"; + MetaCollector collector(metaPath); + string dbRootName = argv[1]; + if (!collector.collect(dbRootName)) + { + cerr << "Cannot collect metadata files " << endl; + return 1; + } + if (!collector.commit()) + { + cerr << "Cannot commit metadata to kv storage " << endl; + return 1; + } + return 0; +}