From e5d76e142be7edcbbf0817f5bea594ddffc84973 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 28 Mar 2019 15:25:49 -0600 Subject: [PATCH 01/40] MCOL-1559 Some string trailing blank stuff --- dbcon/joblist/jlf_execplantojoblist.cpp | 2 ++ dbcon/joblist/lbidlist.cpp | 1 + dbcon/mysql/ha_calpont_execplan.cpp | 2 ++ primitives/primproc/dictstep.cpp | 2 ++ primitives/primproc/primitiveserver.cpp | 2 ++ 5 files changed, 9 insertions(+) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index f3782c9d5..24ec7f50e 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -36,6 +36,7 @@ using namespace std; #include #include #include +#include namespace ba = boost::algorithm; #include "calpontexecutionplan.h" @@ -1635,6 +1636,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); + boost::trim_right_if(constval, boost::is_any_of(" ")); CalpontSystemCatalog::OID dictOid = 0; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index c317defc9..7852562ef 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -749,6 +749,7 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, int64_t tMax = Max; dataconvert::DataConvert::trimWhitespace(tMin); dataconvert::DataConvert::trimWhitespace(tMax); + dataconvert::DataConvert::trimWhitespace(value); scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value), op, lcf); diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 1094ffc37..e3bf14ad2 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -49,6 +49,7 @@ using namespace std; #include #include +#include #include #include @@ -4868,6 +4869,7 @@ void gp_walk(const Item* item, void* arg) { cval.assign(str->ptr(), str->length()); } +// boost::trim_right_if(cval, boost::is_any_of(" ")); gwip->rcWorkStack.push(new ConstantColumn(cval)); break; diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index abd99ada3..47bfeac25 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -30,6 +30,7 @@ #include #include +#include #include "bpp.h" #include "primitiveserver.h" @@ -93,6 +94,7 @@ void DictStep::createCommand(ByteStream& bs) for (uint32_t i = 0; i < filterCount; i++) { bs >> strTmp; + boost::trim_right_if(strTmp, boost::is_any_of(" ")); //cout << " " << strTmp << endl; eqFilter->insert(strTmp); } diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index 76ff8ef3c..9359556e9 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -28,6 +28,7 @@ #include #include #include +#include //#define NDEBUG #include #include @@ -1804,6 +1805,7 @@ private: for (i = 0; i < count; i++) { *bs >> str; + boost::trim_right_if(str, boost::is_any_of(" ")); filter->insert(str); } From 90d16dced463236ae70df0a0cdc9d61729a39791 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 29 Mar 2019 13:11:34 +0000 Subject: [PATCH 02/40] Bump version to 1.2.4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ebe283cd3..91bc7ea08 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=2 -COLUMNSTORE_VERSION_PATCH=3 +COLUMNSTORE_VERSION_PATCH=4 COLUMNSTORE_VERSION_RELEASE=1 From aa802f44c534a21ff447b3025ae3e07ac323e337 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 29 Mar 2019 08:37:16 -0600 Subject: [PATCH 03/40] MCOL-1559 trailing white space comparison --- dbcon/execplan/predicateoperator.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index b83931394..f81f189bb 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -36,6 +36,7 @@ #endif #include #include +#include #include "expressionparser.h" #include "returnedcolumn.h" @@ -460,11 +461,17 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return false; const std::string& val1 = lop->getStrVal(row, isNull); - if (isNull) return false; - return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; + const std::string& val2 = rop->getStrVal(row, isNull); + if (isNull) + return false; + + boost::trim_right_if(val1, boost::is_any_of(" ")); + boost::trim_right_if(val2, boost::is_any_of(" ")); + + return strCompare(val1, val2); } //FIXME: ??? From 8d553ae9fbbe8ddcc984966a45f632a1896fdf81 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 2 Apr 2019 11:14:40 -0600 Subject: [PATCH 04/40] MCOL-1559 trim spaces before compare --- dbcon/execplan/predicateoperator.h | 4 ++-- dbcon/mysql/ha_calpont_execplan.cpp | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index f81f189bb..70209df5b 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -460,11 +460,11 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu if (isNull) return false; - const std::string& val1 = lop->getStrVal(row, isNull); + std::string val1 = lop->getStrVal(row, isNull); if (isNull) return false; - const std::string& val2 = rop->getStrVal(row, isNull); + std::string val2 = rop->getStrVal(row, isNull); if (isNull) return false; diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index e3bf14ad2..1094ffc37 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -49,7 +49,6 @@ using namespace std; #include #include -#include #include #include @@ -4869,7 +4868,6 @@ void gp_walk(const Item* item, void* arg) { cval.assign(str->ptr(), str->length()); } -// boost::trim_right_if(cval, boost::is_any_of(" ")); gwip->rcWorkStack.push(new ConstantColumn(cval)); break; From 5f76cee5f43750e09aa9d2dc97b21d05f3261bd1 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 2 Apr 2019 11:36:56 -0600 Subject: [PATCH 05/40] MCOL-2267 allow for long double in row::equals() --- utils/rowgroup/rowgroup.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 27f0b98a4..b91e0f0ef 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1176,7 +1176,12 @@ inline bool Row::equals(const Row& r2, const std::vector& keyCols) con if (!isLongString(col)) { - if (getUintField(col) != r2.getUintField(col)) + if (getColType(i) == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + if (getLongDoubleField(i) != r2.getLongDoubleField(i)) + return false; + } + else if (getUintField(col) != r2.getUintField(col)) return false; } else @@ -1204,7 +1209,12 @@ inline bool Row::equals(const Row& r2, uint32_t lastCol) const for (uint32_t i = 0; i <= lastCol; i++) if (!isLongString(i)) { - if (getUintField(i) != r2.getUintField(i)) + if (getColType(i) == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + if (getLongDoubleField(i) != r2.getLongDoubleField(i)) + return false; + } + else if (getUintField(i) != r2.getUintField(i)) return false; } else From 6f15c97591b8557615bf1b7c246477c47477e929 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 4 Apr 2019 15:32:38 +0100 Subject: [PATCH 06/40] MCOL-2061 Add upgrade path to rebuild FRM files A major upgrade (1.1 -> 1.2 for example) may have issues due to stale FRM table IDs. This commit adds a stored procedure that changes the table comment to empty on every ColumnStore table to repair the IDs. The user should run this as part of the upgrade procedure between major versions. --- dbcon/ddlpackageproc/altertableprocessor.cpp | 3 ++- dbcon/mysql/columnstore_info.sql | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index a398e656f..118d6be54 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -2064,7 +2064,8 @@ void AlterTableProcessor::tableComment(uint32_t sessionID, execplan::CalpontSyst } else { - throw std::runtime_error("Invalid table comment"); + // Generic table comment, we don't need to do anything + return; } // Get the OID for autoinc (if exists) diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql index d0433a0d9..7655b5f16 100644 --- a/dbcon/mysql/columnstore_info.sql +++ b/dbcon/mysql/columnstore_info.sql @@ -98,4 +98,23 @@ BEGIN SELECT CONCAT((SELECT SUM(data_size) FROM information_schema.columnstore_extents ce left join information_schema.columnstore_columns cc on ce.object_id = cc.object_id where compression_type='Snappy') / (SELECT SUM(compressed_data_size) FROM information_schema.columnstore_files WHERE compressed_data_size IS NOT NULL), ':1') COMPRESSION_RATIO; END // +create procedure columnstore_upgrade() +`columnstore_upgrade`: BEGIN + DECLARE done INTEGER DEFAULT 0; + DECLARE schema_table VARCHAR(100) DEFAULT ""; + DECLARE table_list CURSOR FOR select concat('`', table_schema,'`.`',table_name,'`') from information_schema.tables where engine='columnstore'; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1; + OPEN table_list; + tlist: LOOP + FETCH table_list INTO schema_table; + IF done = 1 THEN LEAVE tlist; + END IF; + SET @sql_query = concat('ALTER TABLE ', schema_table, ' COMMENT=\'\''); + PREPARE stmt FROM @sql_query; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + END LOOP; +END // +delimiter ; + DELIMITER ; From 5b581f53cbe1c0a7b27ac63fdddcffeccd866103 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 9 Apr 2019 13:09:29 -0500 Subject: [PATCH 07/40] MCOL-1559 trailing space compare --- dbcon/execplan/predicateoperator.h | 12 ++++++++++-- dbcon/joblist/jlf_execplantojoblist.cpp | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 70209df5b..daa078948 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -456,11 +456,19 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu isNull = false; return !ret; } +#if 0 + if (isNull) + return false; + + const std::string& val1 = lop->getStrVal(row, isNull); if (isNull) return false; - std::string val1 = lop->getStrVal(row, isNull); + return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; +#endif + // MCOL-1559 + std::string val1 = lop->getStrVal(row, isNull); if (isNull) return false; @@ -472,7 +480,7 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu boost::trim_right_if(val2, boost::is_any_of(" ")); return strCompare(val1, val2); - } + } //FIXME: ??? case execplan::CalpontSystemCatalog::VARBINARY: diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 24ec7f50e..4bd41530d 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1636,7 +1636,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); - boost::trim_right_if(constval, boost::is_any_of(" ")); +// boost::trim_right_if(constval, boost::is_any_of(" ")); CalpontSystemCatalog::OID dictOid = 0; From e2cb6444843494e62ae8eacb0a1c3425400b7f69 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 9 Apr 2019 16:34:45 -0500 Subject: [PATCH 08/40] MCOL-1559 trailing space compare --- dbcon/execplan/predicateoperator.h | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index daa078948..08f0c40cf 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -456,18 +456,8 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu isNull = false; return !ret; } -#if 0 - if (isNull) - return false; - const std::string& val1 = lop->getStrVal(row, isNull); - - if (isNull) - return false; - - return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; -#endif - // MCOL-1559 + // MCOL-1559 std::string val1 = lop->getStrVal(row, isNull); if (isNull) return false; From b6484dda4aa5703606cf74fe77e29101625cff33 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 9 Apr 2019 16:57:21 -0500 Subject: [PATCH 09/40] MCOL-1559 trim constant varchar string before adding filter --- dbcon/joblist/jlf_execplantojoblist.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 4bd41530d..2cad5870e 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1636,8 +1636,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); -// boost::trim_right_if(constval, boost::is_any_of(" ")); - CalpontSystemCatalog::OID dictOid = 0; CalpontSystemCatalog::ColType ct = sc->colType(); @@ -1676,6 +1674,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) pds->cardinality(sc->cardinality()); //Add the filter + // MCOL-1559 trim before adding. + boost::trim_right_if(constval, boost::is_any_of(" ")); pds->addFilter(cop, constval); // data list for pcolstep output From d78944d9ff894f677b63726d3e4f0a89763c8521 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 9 Apr 2019 17:08:03 -0500 Subject: [PATCH 10/40] MCOL-1559 backout trim before setting compare. Shouldn't do this in all cases. --- dbcon/joblist/jlf_execplantojoblist.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 2cad5870e..e9068b5e8 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1674,8 +1674,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) pds->cardinality(sc->cardinality()); //Add the filter - // MCOL-1559 trim before adding. - boost::trim_right_if(constval, boost::is_any_of(" ")); pds->addFilter(cop, constval); // data list for pcolstep output From 28e743bf38e6cc143d21e047cfcc7af105a665c5 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 10 Apr 2019 08:46:15 -0500 Subject: [PATCH 11/40] MCOL-1559 remove unused boost/trim header --- dbcon/joblist/jlf_execplantojoblist.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index e9068b5e8..fff1b12fb 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -36,7 +36,6 @@ using namespace std; #include #include #include -#include namespace ba = boost::algorithm; #include "calpontexecutionplan.h" From f1b908abeb6ab2a7212851079a2b5414caa07767 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 11 Apr 2019 15:44:46 -0500 Subject: [PATCH 12/40] MCOL-2091 Special UDAF reset code for multi-distinct queries --- utils/rowgroup/rowaggregation.cpp | 61 ++++++++++++++++++++++--------- utils/rowgroup/rowaggregation.h | 5 ++- 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index e550a3f04..f42c89bbd 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -714,11 +714,8 @@ void RowAggregation::setJoinRowGroups(vector* pSmallSideRG, RowGroup* // threads on the PM and by multple threads on the UM. It must remain // thread safe. //------------------------------------------------------------------------------ -void RowAggregation::resetUDAF(uint64_t funcColID) +void RowAggregation::resetUDAF(RowUDAFFunctionCol* rowUDAF) { - // Get the UDAF class pointer and store in the row definition object. - RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[funcColID].get()); - // RowAggregation and it's functions need to be re-entrant which means // each instance (thread) needs its own copy of the context object. // Note: operator=() doesn't copy userData. @@ -786,7 +783,7 @@ void RowAggregation::initialize() { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(i); + resetUDAF(dynamic_cast(fFunctionCols[i].get())); } } } @@ -838,7 +835,7 @@ void RowAggregation::aggReset() { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(i); + resetUDAF(dynamic_cast(fFunctionCols[i].get())); } } } @@ -885,14 +882,28 @@ void RowAggregationUM::aggregateRowWithRemap(Row& row) inserted.first->second = RowPosition(fResultDataVec.size() - 1, fRowGroupOut->getRowCount() - 1); // If there's UDAF involved, reset the user data. - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + if (fOrigFunctionCols) { - if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + // This is a multi-distinct query and fFunctionCols may not + // contain all the UDAF we need to reset + for (uint64_t i = 0; i < fOrigFunctionCols->size(); i++) { - resetUDAF(i); + if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) + { + resetUDAF(dynamic_cast((*fOrigFunctionCols)[i].get())); + } + } + } + else + { + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + { + resetUDAF(dynamic_cast(fFunctionCols[i].get())); + } } } - // replace the key value with an equivalent copy, yes this is OK const_cast((inserted.first->first)) = pos; } @@ -946,14 +957,28 @@ void RowAggregation::aggregateRow(Row& row) RowPosition(fResultDataVec.size() - 1, fRowGroupOut->getRowCount() - 1); // If there's UDAF involved, reset the user data. - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + if (fOrigFunctionCols) { - if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + // This is a multi-distinct query and fFunctionCols may not + // contain all the UDAF we need to reset + for (uint64_t i = 0; i < fOrigFunctionCols->size(); i++) { - resetUDAF(i); + if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) + { + resetUDAF(dynamic_cast((*fOrigFunctionCols)[i].get())); + } + } + } + else + { + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + { + resetUDAF(dynamic_cast(fFunctionCols[i].get())); + } } } - } else { @@ -4699,7 +4724,7 @@ void RowAggregationMultiDistinct::doDistinctAggregation() { // backup the function column vector for finalize(). vector origFunctionCols = fFunctionCols; - + fOrigFunctionCols = &origFunctionCols; // aggregate data from each sub-aggregator to distinct aggregator for (uint64_t i = 0; i < fSubAggregators.size(); ++i) { @@ -4727,6 +4752,7 @@ void RowAggregationMultiDistinct::doDistinctAggregation() // restore the function column vector fFunctionCols = origFunctionCols; + fOrigFunctionCols = NULL; } @@ -4734,7 +4760,8 @@ void RowAggregationMultiDistinct::doDistinctAggregation_rowVec(vector origFunctionCols = fFunctionCols; - + fOrigFunctionCols = &origFunctionCols; + // aggregate data from each sub-aggregator to distinct aggregator for (uint64_t i = 0; i < fSubAggregators.size(); ++i) { @@ -4751,9 +4778,9 @@ void RowAggregationMultiDistinct::doDistinctAggregation_rowVec(vectorclear(); } - void resetUDAF(uint64_t funcColID); + void resetUDAF(RowUDAFFunctionCol* rowUDAF); inline bool isNull(const RowGroup* pRowGroup, const Row& row, int64_t col); inline void makeAggFieldsNull(Row& row); @@ -710,6 +710,9 @@ protected: static const static_any::any& doubleTypeId; static const static_any::any& longdoubleTypeId; static const static_any::any& strTypeId; + + // For UDAF along with with multiple distinct columns + vector* fOrigFunctionCols = NULL; }; //------------------------------------------------------------------------------ From 34b1d44563fa72907cf3c423f308f905696b9f5f Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 15 Apr 2019 12:37:04 +0300 Subject: [PATCH 13/40] MCOL-3247 Add two utility functions to simplify the code: to log the messages and to forcely close the FEP connection. --- dbcon/mysql/ha_calpont_impl.cpp | 379 ++++++++++---------------------- 1 file changed, 116 insertions(+), 263 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 02164e239..ae1904dee 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -179,6 +179,82 @@ inline uint32_t tid2sid(const uint32_t tid) return CalpontSystemCatalog::idb_tid2sid(tid); } + +/** + @brief + Wrapper around logging facility. + + @details + Reduces the boiler plate code. + + Called from number of places(mostly DML) in + ha_calpont_impl.cpp(). +*/ +void log_this(THD *thd, const char *message, + logging::LOG_TYPE log_type, unsigned sid) +{ + // corresponds with dbcon in SubsystemID vector + // in messagelog.cpp + unsigned int subSystemId = 24; + logging::LoggingID logid( subSystemId, sid, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add(message); + msg.format( args1 ); + Logger logger(logid.fSubsysID); + logger.logMessage(log_type, msg, logid); +} + +/** + @brief + Forcely close a FEP connection. + + @details + Plugin code opens network connection with ExMgr to + get: + the result of meta-data queries + the result of DML or DQL query in any mode + statistics + This code allows to explicitly close the connection + if any error happens using a non-existing protocol + code 0. This causes ExeMgr loop to drop the + connection. + + Called from many places in ha_calpont_impl.cpp(). +*/ +void force_close_fep_conn(THD *thd, cal_connection_info* ci, bool check_prev_rc = false) +{ + if (!ci->cal_conn_hndl) + { + return; + } + + if(check_prev_rc && !ci->rc) + { + return; + } + + // send ExeMgr an unknown signal to force him to close + // the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // Add details into the message. + log_this(thd, "Exception in force_close_fep_conn().", + logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; +} + void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& ct) { // unset null bit first @@ -2119,26 +2195,7 @@ int ha_calpont_impl_rnd_init(TABLE* table) if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // canceling query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -2650,26 +2707,7 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) // @bug 3078 if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before cloing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // cancel query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -2767,27 +2805,7 @@ int ha_calpont_impl_rnd_end(TABLE* table) if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) { - // @bug 4022. error handling for select part of dml - if (ci->cal_conn_hndl && ci->rc) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is error handling, so ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - return rc; - } + force_close_fep_conn(thd, ci, true); // checking prev command rc } if (!ci) @@ -2802,29 +2820,9 @@ int ha_calpont_impl_rnd_end(TABLE* table) (thd->lex)->sql_command != SQLCOM_INSERT_SELECT && thd->variables.select_limit != (uint64_t) - 1)) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is the end of query. Ignore the exception if exemgr connection failed - // for whatever reason. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - // clear querystats because no query stats available for cancelled query - ci->queryStats = ""; - } - + force_close_fep_conn(thd, ci); + // clear querystats because no query stats available for cancelled query + ci->queryStats = ""; return 0; } @@ -2906,6 +2904,7 @@ int ha_calpont_impl_rnd_end(TABLE* table) ci->warningMsg.clear(); // reset expressionId just in case ci->expressionId = 0; + return rc; } @@ -3243,16 +3242,11 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) } //Log the statement to debug.log - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("Start SQL statement: "); - ostringstream oss; - oss << idb_mysql_query_str(thd) << "; |" << table->s->db.str << "|"; - args1.add(oss.str()); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + { + ostringstream oss; + oss << "Start SQL statement: " << idb_mysql_query_str(thd) << "; |" << table->s->db.str << "|"; + log_this(thd, oss.str().c_str(), logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } //start process cpimport mode 1 ci->mysqld_pid = getpid(); @@ -3290,13 +3284,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) { setError(current_thd, ER_INTERNAL_ERROR, logging::IDBErrorInfo::instance()->errorMsg(ERR_LOCAL_QUERY_UM)); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, + tid2sid(thd->thread_id)); return; } else @@ -3470,16 +3459,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) errnum << "); " << errmsg; setError(current_thd, ER_INTERNAL_ERROR, oss.str()); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1, args2; - logging::Message emsg(1), msg(1); - args1.add(oss.str()); - emsg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_ERROR, emsg, logid); - args2.add("End SQL statement"); - msg.format( args2 ); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, oss.str(), logging::LOG_TYPE_ERROR, tid2sid(thd->thread_id)); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); return; } @@ -3500,13 +3481,7 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) errnum << "); " << strerror(errnum); setError(current_thd, ER_INTERNAL_ERROR, oss.str()); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); return; } @@ -3522,13 +3497,7 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) errnum << "); " << strerror(errnum); setError(current_thd, ER_INTERNAL_ERROR, oss.str()); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); return; } else if (aChPid == 0) // we are in child @@ -3565,13 +3534,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) setError(current_thd, ER_INTERNAL_ERROR, "Forking process cpimport failed."); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, + tid2sid(thd->thread_id)); exit(1); } else // parent @@ -3736,13 +3700,7 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, errnum, 0, errmsg, 512, NULL); ostringstream oss; oss << "GenerateConsoleCtrlEvent: (errno-" << errnum << "); " << errmsg; - LoggingID logid( 24, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add(oss.str()); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, oss.str(), logging::LOG_TYPE_DEBUG,0); } // Close handles to the cpimport process and its primary thread. @@ -3846,18 +3804,15 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) } #endif - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - if ( rc == 0) - args1.add("End SQL statement"); + { + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } else - args1.add("End SQL statement with error"); + { + log_this(thd, "End SQL statement with error", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); ci->columnTypes.clear(); //get extra warning count if any ifstream dmlFile; @@ -4183,26 +4138,7 @@ int ha_calpont_impl_external_lock(THD* thd, TABLE* table, int lock_type) if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before cloing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is the end of the query. Ignore connetion failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -4357,26 +4293,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // canceling query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -4851,26 +4768,7 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE // @bug 3078 if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before cloing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // cancel query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -4932,7 +4830,6 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* THD* thd = current_thd; cal_connection_info* ci = NULL; - if (thd->slave_thread && ( thd->lex->sql_command == SQLCOM_INSERT || thd->lex->sql_command == SQLCOM_INSERT_SELECT || @@ -4950,37 +4847,11 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* if (get_fe_conn_info_ptr() != NULL) ci = reinterpret_cast(get_fe_conn_info_ptr()); - // MCOL-1052 - //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) - //{ - // thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE;// flip back to normal state - // return rc; - //} - - if (((thd->lex)->sql_command == SQLCOM_INSERT) || + if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) { - // @bug 4022. error handling for select part of dml - if (ci->cal_conn_hndl && ci->rc) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is error handling, so ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - return rc; - } + force_close_fep_conn(thd, ci, true); // with checking prev command rc + return rc; } if (!ci) @@ -4995,33 +4866,15 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* (thd->lex)->sql_command != SQLCOM_INSERT_SELECT && thd->variables.select_limit != (uint64_t) - 1)) { - if (ci->cal_conn_hndl) + force_close_fep_conn(thd, ci); + // clear querystats because no query stats available for cancelled query + ci->queryStats = ""; + // Poping next ExeMgr connection out of the stack + if ( ci->cal_conn_hndl_st.size() ) { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is the end of query. Ignore the exception if exemgr connection failed - // for whatever reason. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - // clear querystats because no query stats available for cancelled query - ci->queryStats = ""; + ci->cal_conn_hndl_st.pop(); if ( ci->cal_conn_hndl_st.size() ) - { - ci->cal_conn_hndl_st.pop(); - if ( ci->cal_conn_hndl_st.size() ) - ci->cal_conn_hndl = ci->cal_conn_hndl_st.top(); - } + ci->cal_conn_hndl = ci->cal_conn_hndl_st.top(); } return 0; From f9f966fe962e36169f83d453ff66cad05b9e1908 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 15 Apr 2019 14:45:34 +0100 Subject: [PATCH 14/40] MCOL-593 Add optional MariaDB replication support This patch will allow MariaDB replication into UM1 when enabling the following is added to the SystemConfig section of Columnstore.xml: Y The intended use case is to replication from an InnoDB MariaDB server into ColumnStore. You would need to create the tables on the ColumnStore slave as "ColumnStore" and the same tables in the master as InnoDB. At the moment the use case is narrow and could be prone to problems so this will use the hidden flag until we can improve it. --- dbcon/mysql/ha_calpont_ddl.cpp | 9 +- dbcon/mysql/ha_calpont_dml.cpp | 3 +- dbcon/mysql/ha_calpont_impl.cpp | 136 +++++++++++++++++++------------ dbcon/mysql/ha_calpont_impl_if.h | 11 ++- 4 files changed, 97 insertions(+), 62 deletions(-) diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 20398fe3d..0f0f24a1b 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -2078,8 +2078,7 @@ int ha_calpont_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* if ( schemaSyncOnly && isCreate) return rc; - //this is replcated DDL, treat it just like SSO - if (thd->slave_thread) + if (thd->slave_thread && !ci.replicationEnabled) return rc; //@bug 5660. Error out REAL DDL/DML on slave node. @@ -2277,8 +2276,7 @@ int ha_calpont_impl_delete_table_(const char* db, const char* name, cal_connecti return 0; } - //this is replcated DDL, treat it just like SSO - if (thd->slave_thread) + if (thd->slave_thread && !ci.replicationEnabled) return 0; //@bug 5660. Error out REAL DDL/DML on slave node. @@ -2417,8 +2415,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti pair toPair; string stmt; - //this is replicated DDL, treat it just like SSO - if (thd->slave_thread) + if (thd->slave_thread && !ci.replicationEnabled) return 0; //@bug 5660. Error out REAL DDL/DML on slave node. diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 8f5a5f21c..0e2bce76c 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -2078,7 +2078,8 @@ int ha_calpont_impl_commit_ (handlerton* hton, THD* thd, bool all, cal_connectio thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE ) return rc; - if (thd->slave_thread) return 0; + if (thd->slave_thread && !ci.replicationEnabled) + return 0; std::string command("COMMIT"); #ifdef INFINIDB_DEBUG diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 52c270cf0..7dffcd36f 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -925,7 +925,6 @@ uint32_t doUpdateDelete(THD* thd) cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); - //@bug 5660. Error out DDL/DML on slave node, or on local query node if (ci->isSlaveNode && !thd->slave_thread) { string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); @@ -947,7 +946,14 @@ uint32_t doUpdateDelete(THD* thd) // stats start ci->stats.reset(); ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } if (thd->main_security_ctx.host) ci->stats.fHost = thd->main_security_ctx.host; @@ -2871,8 +2877,9 @@ int ha_calpont_impl_rnd_init(TABLE* table) // prevent "create table as select" from running on slave thd->infinidb_vtable.hasInfiniDBTable = true; - /* If this node is the slave, ignore DML to IDB tables */ - if (thd->slave_thread && ( + cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + if (thd->slave_thread && !ci->replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || thd->lex->sql_command == SQLCOM_INSERT_SELECT || thd->lex->sql_command == SQLCOM_UPDATE || @@ -2925,8 +2932,6 @@ int ha_calpont_impl_rnd_init(TABLE* table) if (!thd->infinidb_vtable.cal_conn_info) thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); - idbassert(ci != 0); // MySQL sometimes calls rnd_init multiple times, plan should only be @@ -3057,7 +3062,14 @@ int ha_calpont_impl_rnd_init(TABLE* table) { ci->stats.reset(); // reset query stats ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } if (thd->main_security_ctx.host) ci->stats.fHost = thd->main_security_ctx.host; @@ -3422,8 +3434,9 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) { THD* thd = current_thd; - /* If this node is the slave, ignore DML to IDB tables */ - if (thd->slave_thread && ( + cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + if (thd->slave_thread && !ci->replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || thd->lex->sql_command == SQLCOM_INSERT_SELECT || thd->lex->sql_command == SQLCOM_UPDATE || @@ -3434,7 +3447,6 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) thd->lex->sql_command == SQLCOM_LOAD)) return 0; - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) return ER_INTERNAL_ERROR; @@ -3463,8 +3475,6 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) if (!thd->infinidb_vtable.cal_conn_info) thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); - // @bug 3078 if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { @@ -3547,8 +3557,17 @@ int ha_calpont_impl_rnd_end(TABLE* table) int rc = 0; THD* thd = current_thd; cal_connection_info* ci = NULL; + bool replicationEnabled = false; - if (thd->slave_thread && ( + if (thd->infinidb_vtable.cal_conn_info) + ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + if (ci && ci->replicationEnabled) + { + replicationEnabled = true; + } + + if (thd->slave_thread && !replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || thd->lex->sql_command == SQLCOM_INSERT_SELECT || thd->lex->sql_command == SQLCOM_UPDATE || @@ -3561,9 +3580,6 @@ int ha_calpont_impl_rnd_end(TABLE* table) thd->infinidb_vtable.isNewQuery = true; - if (thd->infinidb_vtable.cal_conn_info) - ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) { thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE; // flip back to normal state @@ -3861,9 +3877,8 @@ int ha_calpont_impl_write_row(uchar* buf, TABLE* table) cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); - if (thd->slave_thread) return 0; - - + if (thd->slave_thread && !ci->replicationEnabled) + return 0; if (ci->alterTableState > 0) return 0; @@ -3948,7 +3963,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) if (thd->infinidb_vtable.vtable_state != THD::INFINIDB_ALTER_VTABLE) thd->infinidb_vtable.isInfiniDBDML = true; - if (thd->slave_thread) return; + if (thd->slave_thread && !ci->replicationEnabled) + return; //@bug 5660. Error out DDL/DML on slave node, or on local query node if (ci->isSlaveNode && thd->infinidb_vtable.vtable_state != THD::INFINIDB_ALTER_VTABLE) @@ -4422,7 +4438,14 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) // query stats. only collect execution time and rows inserted for insert/load_data_infile ci->stats.reset(); ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } if (thd->main_security_ctx.host) ci->stats.fHost = thd->main_security_ctx.host; @@ -4508,7 +4531,8 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); - if (thd->slave_thread) return 0; + if (thd->slave_thread && !ci->replicationEnabled) + return 0; int rc = 0; @@ -5209,7 +5233,14 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE { ci->stats.reset(); // reset query stats ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } if (thd->main_security_ctx.host) ci->stats.fHost = thd->main_security_ctx.host; @@ -5628,19 +5659,6 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE { THD* thd = current_thd; - /* If this node is the slave, ignore DML to IDB tables */ - if (thd->slave_thread && ( - thd->lex->sql_command == SQLCOM_INSERT || - thd->lex->sql_command == SQLCOM_INSERT_SELECT || - thd->lex->sql_command == SQLCOM_UPDATE || - thd->lex->sql_command == SQLCOM_UPDATE_MULTI || - thd->lex->sql_command == SQLCOM_DELETE || - thd->lex->sql_command == SQLCOM_DELETE_MULTI || - thd->lex->sql_command == SQLCOM_TRUNCATE || - thd->lex->sql_command == SQLCOM_LOAD)) - return 0; - - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) return ER_INTERNAL_ERROR; @@ -5666,6 +5684,17 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + if (thd->slave_thread && !ci->replicationEnabled && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + // @bug 3078 if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { @@ -5750,18 +5779,6 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* THD* thd = current_thd; cal_connection_info* ci = NULL; - - if (thd->slave_thread && ( - thd->lex->sql_command == SQLCOM_INSERT || - thd->lex->sql_command == SQLCOM_INSERT_SELECT || - thd->lex->sql_command == SQLCOM_UPDATE || - thd->lex->sql_command == SQLCOM_UPDATE_MULTI || - thd->lex->sql_command == SQLCOM_DELETE || - thd->lex->sql_command == SQLCOM_DELETE_MULTI || - thd->lex->sql_command == SQLCOM_TRUNCATE || - thd->lex->sql_command == SQLCOM_LOAD)) - return 0; - thd->infinidb_vtable.isNewQuery = true; thd->infinidb_vtable.isUnion = false; @@ -5775,6 +5792,23 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* // return rc; //} + if (!ci) + { + thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); + ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + } + + if (thd->slave_thread && !ci->replicationEnabled && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) { @@ -5801,12 +5835,6 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* } } - if (!ci) - { - thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); - ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); - } - // @bug 3078. Also session limit variable works the same as ctrl+c if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD || ((thd->lex)->sql_command != SQLCOM_INSERT && diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 72579111b..f107bac06 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -251,10 +251,18 @@ struct cal_connection_info useXbit(false), utf8(false), useCpimport(1), - delimiter('\7') + delimiter('\7'), + replicationEnabled(false) { // check if this is a slave mysql daemon isSlaveNode = checkSlave(); + + std::string option = config::Config::makeConfig()->getConfig("SystemConfig", "ReplicationEnabled"); + + if (!option.compare("Y")) + { + replicationEnabled = true; + } } static bool checkSlave() @@ -319,6 +327,7 @@ struct cal_connection_info char delimiter; char enclosed_by; std::vector columnTypes; + bool replicationEnabled; }; typedef std::tr1::unordered_map CalConnMap; From 287b2ba7bdd6cc767e3422de5ddaa6f58a65f7d7 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 15 Apr 2019 19:31:12 +0300 Subject: [PATCH 15/40] MCOL-3247 CS uses thread plugin variables since MCOL-1101 that aren't good for transmiting cal_connection_info to handler::close_connection(). This variables are then erased in THD::cleanup() and the cal_connection_info is replaced in handler::delete_table with an empty structure. handler::close_connection() recevies cal_connection_info with empty cal_conn_hndl. This patch uses THD::ha_data that is supposed to be used in such occasions. --- dbcon/mysql/ha_calpont.cpp | 81 --------------------------------- dbcon/mysql/ha_calpont_impl.cpp | 17 +++++-- 2 files changed, 13 insertions(+), 85 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index d4ff366bd..552a895d6 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -181,87 +181,6 @@ static int infinidb_done_func(void* p) DBUG_RETURN(0); } -#if 0 -/** - @brief - Example of simple lock controls. The "share" it creates is a - structure we will pass to each example handler. Do you have to have - one of these? Well, you have pieces that are used for locking, and - they are needed to function. -*/ - -static CALPONT_SHARE* get_share(const char* table_name, TABLE* table) -{ - CALPONT_SHARE* share; - uint32_t length; - char* tmp_name; -#ifndef _MSC_VER - pthread_mutex_lock(&calpont_mutex); -#endif - length = (uint32_t) strlen(table_name); - - if (!(share = (CALPONT_SHARE*) hash_search(&calpont_open_tables, - (uchar*) table_name, - length))) - { - if (!(share = (CALPONT_SHARE*) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), - &share, sizeof(*share), - &tmp_name, length + 1, - NullS))) - { - pthread_mutex_unlock(&calpont_mutex); - return NULL; - } - - share->use_count = 0; - share->table_name_length = length; - share->table_name = tmp_name; - strmov(share->table_name, table_name); - - if (my_hash_insert(&calpont_open_tables, (uchar*) share)) - goto error; - - thr_lock_init(&share->lock); - pthread_mutex_init(&share->mutex, MY_MUTEX_INIT_FAST); - } - - share->use_count++; - pthread_mutex_unlock(&calpont_mutex); - - return share; - -error: - pthread_mutex_destroy(&share->mutex); - my_free(share, MYF(0)); - - return NULL; -} - -/** - @brief - Free lock controls. We call this whenever we close a table. If the table had - the last reference to the share, then we free memory associated with it. -*/ - -static int free_share(CALPONT_SHARE* share) -{ - pthread_mutex_lock(&calpont_mutex); - - if (!--share->use_count) - { - hash_delete(&calpont_open_tables, (uchar*) share); - thr_lock_delete(&share->lock); - pthread_mutex_destroy(&share->mutex); - my_free(share, MYF(0)); - } - - pthread_mutex_unlock(&calpont_mutex); - - return 0; -} -#endif - static handler* calpont_create_handler(handlerton* hton, TABLE_SHARE* table, MEM_ROOT* mem_root) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index ae1904dee..57c17c6e7 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -3969,10 +3969,16 @@ int ha_calpont_impl_close_connection (handlerton* hton, THD* thd) execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + // MCOL-3247 Use THD::ha_data as a per-plugin per-session + // storage. Filled in external_lock when we remove a lock + // from vtable(lock_type = 2) + // An ugly way. I will use ha_data w/o external_lock. + // This in MCOL-2178 + cal_connection_info* ci = NULL; + if(thd_get_ha_data(thd, hton)) + { + ci = reinterpret_cast(thd_get_ha_data(thd, hton)); + } if (!ci) return 0; @@ -4195,6 +4201,9 @@ int ha_calpont_impl_external_lock(THD* thd, TABLE* table, int lock_type) ci->miniStats = ci->cal_conn_hndl->miniStats; ci->queryState = 0; thd->infinidb_vtable.override_largeside_estimate = false; + // MCOL-3247 Use THD::ha_data as a per-plugin per-session + // storage for cal_conn_hndl to use it later in close_connection + thd_set_ha_data(thd, calpont_hton, get_fe_conn_info_ptr()); } } From 853dc2a2c1965c565e0a239dd7558b890425ff3f Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 15 Apr 2019 12:49:43 -0500 Subject: [PATCH 16/40] MCOL-2091 Don't use in-line initializers, a C++x11 feature. --- utils/rowgroup/rowaggregation.cpp | 9 +++++---- utils/rowgroup/rowaggregation.h | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index f42c89bbd..df5c1c146 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -218,7 +218,6 @@ inline string getStringNullValue() namespace rowgroup { - const std::string typeStr(""); const static_any::any& RowAggregation::charTypeId((char)1); const static_any::any& RowAggregation::scharTypeId((signed char)1); @@ -590,7 +589,8 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in RowAggregation::RowAggregation() : fAggMapPtr(NULL), fRowGroupOut(NULL), fTotalRowCount(0), fMaxTotalRowCount(AGG_ROWGROUP_SIZE), - fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0) + fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0), + fOrigFunctionCols(NULL) { } @@ -599,7 +599,8 @@ RowAggregation::RowAggregation(const vector& rowAggGroupByCol const vector& rowAggFunctionCols) : fAggMapPtr(NULL), fRowGroupOut(NULL), fTotalRowCount(0), fMaxTotalRowCount(AGG_ROWGROUP_SIZE), - fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0) + fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0), + fOrigFunctionCols(NULL) { fGroupByCols.assign(rowAggGroupByCols.begin(), rowAggGroupByCols.end()); fFunctionCols.assign(rowAggFunctionCols.begin(), rowAggFunctionCols.end()); @@ -610,7 +611,7 @@ RowAggregation::RowAggregation(const RowAggregation& rhs): fAggMapPtr(NULL), fRowGroupOut(NULL), fTotalRowCount(0), fMaxTotalRowCount(AGG_ROWGROUP_SIZE), fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0), - fRGContext(rhs.fRGContext) + fRGContext(rhs.fRGContext), fOrigFunctionCols(NULL) { //fGroupByCols.clear(); //fFunctionCols.clear(); diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index ef8686c0e..d1547d387 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -712,7 +712,7 @@ protected: static const static_any::any& strTypeId; // For UDAF along with with multiple distinct columns - vector* fOrigFunctionCols = NULL; + vector* fOrigFunctionCols; }; //------------------------------------------------------------------------------ From 9846d6b595d5742e94df6b85af9214052642a015 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 15 Apr 2019 12:52:00 -0500 Subject: [PATCH 17/40] MCOL-2091 align margin --- utils/rowgroup/rowaggregation.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index d1547d387..c829cdefc 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -630,7 +630,7 @@ protected: if (fAggMapPtr) fAggMapPtr->clear(); } - void resetUDAF(RowUDAFFunctionCol* rowUDAF); + void resetUDAF(RowUDAFFunctionCol* rowUDAF); inline bool isNull(const RowGroup* pRowGroup, const Row& row, int64_t col); inline void makeAggFieldsNull(Row& row); From c17e32d5e013149cfec2b084ba7a7e8b905e2c61 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 15 Apr 2019 16:36:27 -0500 Subject: [PATCH 18/40] MCOL-1985 Modify regrmysql to use long double where practical and to use the latest algorithms for regr_xxx functions. --- utils/regr/regr_sxx.cpp | 2 +- utils/regr/regr_syy.cpp | 2 +- utils/regr/regrmysql.cpp | 431 +++++++++++++++++++++++---------------- 3 files changed, 257 insertions(+), 178 deletions(-) diff --git a/utils/regr/regr_sxx.cpp b/utils/regr/regr_sxx.cpp index 5769a227b..4d5fac370 100644 --- a/utils/regr/regr_sxx.cpp +++ b/utils/regr/regr_sxx.cpp @@ -132,7 +132,7 @@ mcsv1_UDAF::ReturnCode regr_sxx::evaluate(mcsv1Context* context, static_any::any long double sumx2 = data->sumx2; long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - valOut = static_cast(data->cnt * var_popx); + valOut = static_cast(N * var_popx); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_syy.cpp b/utils/regr/regr_syy.cpp index 014a28389..6febb9579 100644 --- a/utils/regr/regr_syy.cpp +++ b/utils/regr/regr_syy.cpp @@ -132,7 +132,7 @@ mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any long double sumy2 = data->sumy2; long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - valOut = static_cast(data->cnt * var_popy); + valOut = static_cast(N * var_popy); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 4980108e3..5faed3b0b 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -147,7 +147,7 @@ extern "C" */ struct regr_avgx_data { - double sumx; + long double sumx; int64_t cnt; }; @@ -159,21 +159,26 @@ extern "C" struct regr_avgx_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_avgx() requires two arguments"); - return 1; + strcpy(message,"regr_avgx() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[1], args->attributes[1]))) { strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); return 1; } + + if (args->arg_type[1] == DECIMAL_RESULT) + { + initid->decimals +=4; + } - if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) + if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } - data->sumx = 0; + data->sumx = 0; data->cnt = 0; initid->ptr = (char*)data; @@ -226,7 +231,17 @@ extern "C" char* is_null, char* error __attribute__((unused))) { struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; - return data->sumx / data->cnt; + double valOut = 0; + if (data->cnt > 0) + { + valOut = static_cast(data->sumx / data->cnt); + } + else + { + *is_null = 1; + } + + return valOut; } //======================================================================= @@ -236,8 +251,8 @@ extern "C" */ struct regr_avgy_data { - double sumy; - int64_t cnt; + long double sumy; + int64_t cnt; }; #ifdef _MSC_VER @@ -248,8 +263,8 @@ extern "C" struct regr_avgy_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_avgy() requires two arguments"); - return 1; + strcpy(message,"regr_avgy() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]))) { @@ -257,10 +272,15 @@ extern "C" return 1; } - if (!(data = (struct regr_avgy_data*) malloc(sizeof(struct regr_avgy_data)))) + if (args->arg_type[0] == DECIMAL_RESULT) + { + initid->decimals +=4; + } + + if (!(data = (struct regr_avgy_data*) malloc(sizeof(struct regr_avgy_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->sumy = 0; data->cnt = 0; @@ -315,7 +335,16 @@ extern "C" char* is_null, char* error __attribute__((unused))) { struct regr_avgy_data* data = (struct regr_avgy_data*)initid->ptr; - return data->sumy / data->cnt; + double valOut = 0; + if (data->cnt > 0) + { + valOut = static_cast(data->sumy / data->cnt); + } + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -336,14 +365,14 @@ extern "C" struct regr_count_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_count() requires two arguments"); - return 1; + strcpy(message,"regr_count() requires two arguments"); + return 1; } if (!(data = (struct regr_count_data*) malloc(sizeof(struct regr_count_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; @@ -405,10 +434,10 @@ extern "C" struct regr_slope_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -419,8 +448,8 @@ extern "C" struct regr_slope_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_slope() requires two arguments"); - return 1; + strcpy(message,"regr_slope() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -428,10 +457,12 @@ extern "C" return 1; } + initid->decimals = DECIMAL_NOT_SPECIFIED; + if (!(data = (struct regr_slope_data*) malloc(sizeof(struct regr_slope_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -497,20 +528,23 @@ extern "C" { struct regr_slope_data* data = (struct regr_slope_data*)initid->ptr; double N = data->cnt; + double valOut = 0; + *is_null = 1; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumxy = data->sumxy; - double variance = (N * sumx2) - (sumx * sumx); - if (variance) + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double covar_pop = N * sumxy - sumx * sumy; + long double var_pop = N * sumx2 - sumx * sumx; + if (var_pop != 0) { - return ((N * sumxy) - (sumx * sumy)) / variance; + valOut = static_cast(covar_pop / var_pop); + *is_null = 0; } } - *is_null = 1; - return 0; + return valOut; } //======================================================================= @@ -521,10 +555,10 @@ extern "C" struct regr_intercept_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -535,8 +569,8 @@ extern "C" struct regr_intercept_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_intercept() requires two arguments"); - return 1; + strcpy(message,"regr_intercept() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -544,10 +578,11 @@ extern "C" return 1; } - if (!(data = (struct regr_intercept_data*) malloc(sizeof(struct regr_intercept_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + if (!(data = (struct regr_intercept_data*) malloc(sizeof(struct regr_intercept_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -613,22 +648,23 @@ extern "C" { struct regr_intercept_data* data = (struct regr_intercept_data*)initid->ptr; double N = data->cnt; + double valOut = 0; + *is_null = 1; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumxy = data->sumxy; - double slope = 0; - double variance = (N * sumx2) - (sumx * sumx); - if (variance) + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double numerator = sumy * sumx2 - sumx * sumxy; + long double var_pop = (N * sumx2) - (sumx * sumx); + if (var_pop != 0) { - slope = ((N * sumxy) - (sumx * sumy)) / variance; + valOut = static_cast(numerator / var_pop); + *is_null = 0; } - return (sumy - (slope * sumx)) / N; } - *is_null = 1; - return 0; + return valOut; } //======================================================================= @@ -639,11 +675,11 @@ extern "C" struct regr_r2_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumy2; // sum of (y squared) - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -654,8 +690,8 @@ extern "C" struct regr_r2_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_r2() requires two arguments"); - return 1; + strcpy(message,"regr_r2() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -663,10 +699,12 @@ extern "C" return 1; } - if (!(data = (struct regr_r2_data*) malloc(sizeof(struct regr_r2_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_r2_data*) malloc(sizeof(struct regr_r2_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -735,34 +773,38 @@ extern "C" { struct regr_r2_data* data = (struct regr_r2_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumy2 = data->sumy2; - double sumxy = data->sumxy; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; if (var_popx == 0) { // When var_popx is 0, NULL is the result. *is_null = 1; return 0; } - double var_popy = (sumy2 - (sumy * sumy / N)) / N; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; if (var_popy == 0) { // When var_popy is 0, 1 is the result return 1; } - double std_popx = sqrt(var_popx); - double std_popy = sqrt(var_popy); - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double corr = covar_pop / (std_popy * std_popx); - return corr * corr; + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr * corr); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -773,11 +815,11 @@ extern "C" struct corr_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumy2; // sum of (y squared) - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -788,8 +830,8 @@ extern "C" struct corr_data* data; if (args->arg_count != 2) { - strcpy(message,"corr() requires two arguments"); - return 1; + strcpy(message,"corr() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -797,10 +839,12 @@ extern "C" return 1; } - if (!(data = (struct corr_data*) malloc(sizeof(struct corr_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct corr_data*) malloc(sizeof(struct corr_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -869,34 +913,38 @@ extern "C" { struct corr_data* data = (struct corr_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumy2 = data->sumy2; - double sumxy = data->sumxy; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; if (var_popx == 0) { // When var_popx is 0, NULL is the result. *is_null = 1; return 0; } - double var_popy = (sumy2 - (sumy * sumy / N)) / N; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; if (var_popy == 0) { // When var_popy is 0, 1 is the result return 1; } - double std_popx = sqrt(var_popx); - double std_popy = sqrt(var_popy); - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double corr = covar_pop / (std_popy * std_popx); - return corr; + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + return static_cast(corr); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -907,8 +955,8 @@ extern "C" struct regr_sxx_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) + long double sumx; + long double sumx2; // sum of (x squared) }; #ifdef _MSC_VER @@ -919,8 +967,8 @@ extern "C" struct regr_sxx_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_sxx() requires two arguments"); - return 1; + strcpy(message,"regr_sxx() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -928,10 +976,12 @@ extern "C" return 1; } - if (!(data = (struct regr_sxx_data*) malloc(sizeof(struct regr_sxx_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_sxx_data*) malloc(sizeof(struct regr_sxx_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -990,15 +1040,19 @@ extern "C" { struct regr_sxx_data* data = (struct regr_sxx_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumx2 = data->sumx2; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; - return data->cnt * var_popx; + long double sumx = data->sumx; + long double sumx2 = data->sumx2; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; + valOut = static_cast(N * var_popx); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -1008,8 +1062,8 @@ extern "C" struct regr_syy_data { int64_t cnt; - double sumy; - double sumy2; // sum of (y squared) + long double sumy; + long double sumy2; // sum of (y squared) }; #ifdef _MSC_VER @@ -1020,8 +1074,8 @@ extern "C" struct regr_syy_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_syy() requires two arguments"); - return 1; + strcpy(message,"regr_syy() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]))) { @@ -1029,10 +1083,12 @@ extern "C" return 1; } - if (!(data = (struct regr_syy_data*) malloc(sizeof(struct regr_syy_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_syy_data*) malloc(sizeof(struct regr_syy_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumy = 0.0; @@ -1091,15 +1147,19 @@ extern "C" { struct regr_syy_data* data = (struct regr_syy_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumy = data->sumy; - double sumy2 = data->sumy2; - double var_popy = (sumy2 - (sumy * sumy / N)) / N; - return data->cnt * var_popy; + long double sumy = data->sumy; + long double sumy2 = data->sumy2; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; + valOut = static_cast(N * var_popy); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -1110,9 +1170,9 @@ extern "C" struct regr_sxy_data { int64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -1123,8 +1183,8 @@ extern "C" struct regr_sxy_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_sxy() requires two arguments"); - return 1; + strcpy(message,"regr_sxy() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -1132,10 +1192,12 @@ extern "C" return 1; } - if (!(data = (struct regr_sxy_data*) malloc(sizeof(struct regr_sxy_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_sxy_data*) malloc(sizeof(struct regr_sxy_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -1198,16 +1260,21 @@ extern "C" { struct regr_sxy_data* data = (struct regr_sxy_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - return data->cnt * covar_pop; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double regr_sxy = N * covar_pop; + valOut = static_cast(regr_sxy); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -1218,9 +1285,9 @@ extern "C" struct covar_pop_data { int64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -1231,8 +1298,8 @@ extern "C" struct covar_pop_data* data; if (args->arg_count != 2) { - strcpy(message,"covar_pop() requires two arguments"); - return 1; + strcpy(message,"covar_pop() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -1240,10 +1307,12 @@ extern "C" return 1; } - if (!(data = (struct covar_pop_data*) malloc(sizeof(struct covar_pop_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct covar_pop_data*) malloc(sizeof(struct covar_pop_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -1306,16 +1375,20 @@ extern "C" { struct covar_pop_data* data = (struct covar_pop_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - return covar_pop; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; + valOut = static_cast(covar_pop); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -1325,9 +1398,9 @@ extern "C" struct covar_samp_data { int64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -1338,8 +1411,8 @@ extern "C" struct covar_samp_data* data; if (args->arg_count != 2) { - strcpy(message,"covar_samp() requires two arguments"); - return 1; + strcpy(message,"covar_samp() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -1347,10 +1420,12 @@ extern "C" return 1; } - if (!(data = (struct covar_samp_data*) malloc(sizeof(struct covar_samp_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct covar_samp_data*) malloc(sizeof(struct covar_samp_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -1413,16 +1488,20 @@ extern "C" { struct covar_samp_data* data = (struct covar_samp_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; - double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N-1); - return covar_samp; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); + valOut = static_cast(covar_samp); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } } // vim:ts=4 sw=4: From 2a9c5387c3113b3a7a0138020b8d173d884a33dc Mon Sep 17 00:00:00 2001 From: David Mott Date: Wed, 17 Apr 2019 05:03:28 -0500 Subject: [PATCH 19/40] Enable c++11 --- .gitignore | 1 + CMakeLists.txt | 35 ++++++++++------------------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index aba662bbf..f7b16ba8a 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,4 @@ install_manifest_storage-engine.txt _CPack_Packages columnstoreversion.h .idea/ +.build diff --git a/CMakeLists.txt b/CMakeLists.txt index 94fdde894..bb22d07e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,15 @@ IF(NOT CMAKE_BUILD_TYPE) "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) ENDIF(NOT CMAKE_BUILD_TYPE) -#set( CMAKE_VERBOSE_MAKEFILE on ) +SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") + +SET(CMAKE_CXX_STANDARD 11) +SET(CMAKE_CXX_STANDARD_REQUIRED TRUE) +SET(CMAKE_CXX_EXTENSIONS FALSE) +SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) +SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) +SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) +SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) INCLUDE(columnstore_version.cmake) @@ -83,14 +91,6 @@ INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) -exec_program("git" - ${CMAKE_CURRENT_SOURCE_DIR} - ARGS "describe --match=NeVeRmAtCh --always --dirty" - OUTPUT_VARIABLE GIT_VERSION) - -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${INSTALL_ENGINE} COMPONENT platform) - INCLUDE(bison.cmake) FIND_PROGRAM(LEX_EXECUTABLE flex DOC "path to the flex executable") @@ -112,17 +112,6 @@ if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() -# Jemalloc has issues with SLES 12, so disable for now -IF (EXISTS "/etc/SuSE-release") - SET(JEMALLOC_LIBRARIES "") -ELSE () - INCLUDE (FindJeMalloc.cmake) - if (NOT JEMALLOC_FOUND) - message(FATAL_ERROR "jemalloc not found!") - SET(JEMALLOC_LIBRARIES "") - endif() -ENDIF () - FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") @@ -174,7 +163,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local") SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql") SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${JEMALLOC_LIBRARIES}) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) @@ -183,17 +172,13 @@ SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage SET (ENGINE_COMMON_LDFLAGS "") IF (SERVER_BUILD_INCLUDE_DIR) - IF (NOT IS_ABSOLUTE ${SERVER_BUILD_INCLUDE_DIR}) SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR}) - ENDIF() ELSE() SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include) ENDIF() IF (SERVER_SOURCE_ROOT_DIR) - IF (NOT IS_ABSOLUTE ${SERVER_SOURCE_ROOT_DIR}) SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR}) - ENDIF() ELSE() SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..) ENDIF() From 241d0b0446f5ffc2c2a5f3d5fd65b6a7213f7a31 Mon Sep 17 00:00:00 2001 From: David Mott Date: Wed, 17 Apr 2019 11:36:56 -0500 Subject: [PATCH 20/40] rollback unintended changes --- CMakeLists.txt | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb22d07e5..9fa59a1c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,16 +34,15 @@ IF(NOT CMAKE_BUILD_TYPE) SET(CMAKE_BUILD_TYPE RELWITHDEBINFO CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) ENDIF(NOT CMAKE_BUILD_TYPE) - -SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") - -SET(CMAKE_CXX_STANDARD 11) -SET(CMAKE_CXX_STANDARD_REQUIRED TRUE) -SET(CMAKE_CXX_EXTENSIONS FALSE) -SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) -SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) -SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) -SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) +SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") + +SET(CMAKE_CXX_STANDARD 11) +SET(CMAKE_CXX_STANDARD_REQUIRED TRUE) +SET(CMAKE_CXX_EXTENSIONS FALSE) +SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) +SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) +SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) +SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) INCLUDE(columnstore_version.cmake) @@ -91,6 +90,14 @@ INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) +exec_program("git" + ${CMAKE_CURRENT_SOURCE_DIR} + ARGS "describe --match=NeVeRmAtCh --always --dirty" + OUTPUT_VARIABLE GIT_VERSION) + +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${INSTALL_ENGINE} COMPONENT platform) + INCLUDE(bison.cmake) FIND_PROGRAM(LEX_EXECUTABLE flex DOC "path to the flex executable") @@ -112,6 +119,17 @@ if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() +# Jemalloc has issues with SLES 12, so disable for now +IF (EXISTS "/etc/SuSE-release") + SET(JEMALLOC_LIBRARIES "") +ELSE () + INCLUDE (FindJeMalloc.cmake) + if (NOT JEMALLOC_FOUND) + message(FATAL_ERROR "jemalloc not found!") + SET(JEMALLOC_LIBRARIES "") + endif() +ENDIF () + FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") @@ -163,7 +181,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local") SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql") SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${JEMALLOC_LIBRARIES}) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) @@ -172,13 +190,17 @@ SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage SET (ENGINE_COMMON_LDFLAGS "") IF (SERVER_BUILD_INCLUDE_DIR) + IF (NOT IS_ABSOLUTE ${SERVER_BUILD_INCLUDE_DIR}) SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR}) + ENDIF() ELSE() SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include) ENDIF() IF (SERVER_SOURCE_ROOT_DIR) + IF (NOT IS_ABSOLUTE ${SERVER_SOURCE_ROOT_DIR}) SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR}) + ENDIF() ELSE() SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..) ENDIF() From 67880c2319f8af06a8387740a6bf38191dcd660e Mon Sep 17 00:00:00 2001 From: David Mott Date: Wed, 17 Apr 2019 17:28:47 -0500 Subject: [PATCH 21/40] Fully qualify ambiguous isnan() with std:: --- dbcon/execplan/treenode.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 91ceb2a11..106b40d37 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -608,7 +608,7 @@ inline const std::string& TreeNode::getStrVal() int exponent = (int)floor(log10( fabs(fResult.floatVal))); // This will round down the exponent double base = fResult.floatVal * pow(10, -1.0 * exponent); - if (isnan(exponent) || isnan(base)) + if (isnan(exponent) || std::isnan(base)) { snprintf(tmp, 312, "%f", fResult.floatVal); fResult.strVal = removeTrailing0(tmp, 312); @@ -643,7 +643,7 @@ inline const std::string& TreeNode::getStrVal() int exponent = (int)floor(log10( fabs(fResult.doubleVal))); // This will round down the exponent double base = fResult.doubleVal * pow(10, -1.0 * exponent); - if (isnan(exponent) || isnan(base)) + if (isnan(exponent) || std::isnan(base)) { snprintf(tmp, 312, "%f", fResult.doubleVal); fResult.strVal = removeTrailing0(tmp, 312); From b2810bf35de9421d2aed29e2691b5ae426906879 Mon Sep 17 00:00:00 2001 From: David Mott Date: Thu, 18 Apr 2019 04:43:28 -0500 Subject: [PATCH 22/40] fix ambiguous symbol --- writeengine/bulk/we_bulkloadbuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index f0336d79d..01d6ed575 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -443,7 +443,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { memcpy(&dVal, field, sizeof(dVal)); - if ( isnan(dVal) ) + if ( std::isnan(dVal) ) { if (signbit(dVal)) dVal = column.fMinDblSat; From 06f24df724bf6ed40463cb0279573dfcaf07e8a1 Mon Sep 17 00:00:00 2001 From: Patrice Linel Date: Wed, 17 Apr 2019 22:32:09 -0500 Subject: [PATCH 23/40] change signature array in a std::set ! lookup performance is now log(N). About 10x performance can be expected on cpimport containing varchars. Signed-off-by: Patrice Linel --- writeengine/bulk/we_colbufmgr.cpp | 2 +- writeengine/dictionary/we_dctnry.cpp | 41 +++++++++++------------ writeengine/dictionary/we_dctnry.h | 13 ++++++- writeengine/dictionary/we_dctnrystore.cpp | 2 +- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/writeengine/bulk/we_colbufmgr.cpp b/writeengine/bulk/we_colbufmgr.cpp index 123847260..36ecc218b 100644 --- a/writeengine/bulk/we_colbufmgr.cpp +++ b/writeengine/bulk/we_colbufmgr.cpp @@ -45,7 +45,7 @@ namespace { // Minimum time to wait for a condition, so as to periodically wake up and // check the global job status, to see if the job needs to terminate. -const int COND_WAIT_SECONDS = 3; +const int COND_WAIT_SECONDS = 1; } namespace WriteEngine diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index 4e93ca60a..d477349ad 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -108,7 +109,6 @@ Dctnry::Dctnry() : &m_endHeader, HDR_UNIT_SIZE); m_curFbo = INVALID_NUM; m_curLbid = INVALID_LBID; - memset(m_sigArray, 0, MAX_STRING_CACHE_SIZE * sizeof(Signature)); m_arraySize = 0; clear();//files @@ -130,14 +130,16 @@ Dctnry::~Dctnry() ******************************************************************************/ void Dctnry::freeStringCache( ) { - for (int i = 0; i < m_arraySize; i++) + std::set::iterator it; + for (it=m_sigArray.begin(); it!=m_sigArray.end(); it++) { - delete [] m_sigArray[i].signature; - m_sigArray[i].signature = 0; + Signature sig = *it; + delete [] sig.signature; + sig.signature = 0; } - memset(m_sigArray, 0, MAX_STRING_CACHE_SIZE * sizeof(Signature)); m_arraySize = 0; + m_sigArray.clear(); } /******************************************************************************* @@ -161,7 +163,6 @@ int Dctnry::init() m_curOp = 0; memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); m_curBlock.lbid = INVALID_LBID; - memset(m_sigArray, 0, MAX_STRING_CACHE_SIZE * sizeof(Signature)); m_arraySize = 0; return NO_ERROR; @@ -623,19 +624,17 @@ int Dctnry::openDctnry(const OID& dctnryOID, ******************************************************************************/ bool Dctnry::getTokenFromArray(Signature& sig) { - for (int i = 0; i < (int)m_arraySize ; i++ ) - { - if (sig.size == m_sigArray[i].size) - { - if (!memcmp(sig.signature, m_sigArray[i].signature, sig.size)) - { - sig.token = m_sigArray[i].token; - return true; - }//endif sig compare - }//endif size compare - } + std::set::iterator it; + it = m_sigArray.find(sig); + if ( it == m_sigArray.end()){ + return false; + }else{ + Signature sigfound = *it; + sig.token = sigfound.token; + return true; + } - return false; + return false; } /******************************************************************************* @@ -1329,7 +1328,7 @@ void Dctnry::preLoadStringCache( const DataBlock& fileBlock ) memcpy(aSig.signature, &fileBlock.data[offBeg], len); aSig.token.op = op; aSig.token.fbo = m_curLbid; - m_sigArray[op - 1] = aSig; + m_sigArray.insert(aSig); offEnd = offBeg; hdrOffsetBeg += HDR_UNIT_SIZE; @@ -1368,7 +1367,7 @@ void Dctnry::addToStringCache( const Signature& newSig ) memcpy(asig.signature, newSig.signature, newSig.size ); asig.size = newSig.size; asig.token = newSig.token; - m_sigArray[m_arraySize] = asig; + m_sigArray.insert(asig); m_arraySize++; } @@ -1461,7 +1460,7 @@ int Dctnry::updateDctnry(unsigned char* sigValue, int& sigSize, sig.signature = new unsigned char[sigSize]; memcpy (sig.signature, sigValue, sigSize); sig.token = token; - m_sigArray[m_arraySize] = sig; + m_sigArray.insert(sig); m_arraySize++; } diff --git a/writeengine/dictionary/we_dctnry.h b/writeengine/dictionary/we_dctnry.h index 58d429142..0417a906f 100644 --- a/writeengine/dictionary/we_dctnry.h +++ b/writeengine/dictionary/we_dctnry.h @@ -56,6 +56,17 @@ typedef struct Signature Token token; } Signature; +struct sig_compare { + bool operator() (const Signature& a, const Signature& b) const { + if (a.size == b.size){ + return memcmp(a.signature,b.signature,a.size)<0;} + else if (a.size& oids); virtual int numOfBlocksInFile(); - Signature m_sigArray[MAX_STRING_CACHE_SIZE]; // string cache + std::set m_sigArray; int m_arraySize; // num strings in m_sigArray // m_dctnryHeader used for hdr when readSubBlockEntry is used to read a blk diff --git a/writeengine/dictionary/we_dctnrystore.cpp b/writeengine/dictionary/we_dctnrystore.cpp index 8e2982b57..4a7fefe73 100644 --- a/writeengine/dictionary/we_dctnrystore.cpp +++ b/writeengine/dictionary/we_dctnrystore.cpp @@ -133,7 +133,7 @@ const int DctnryStore::updateDctnryStore(unsigned char* sigValue, sig.signature = new unsigned char[sigSize]; memcpy (sig.signature, sigValue, sigSize); sig.token = token; - m_dctnry.m_sigArray[m_dctnry.m_arraySize] = sig; + m_dctnry.m_sigArray.insert(sig) = sig; m_dctnry.m_arraySize++; } From b38f192e2ea1591a566eaed418a8d42110604795 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 18 Apr 2019 13:31:16 -0500 Subject: [PATCH 24/40] MCOL-1985 Don't change umber of decimals if DECIMAL_NOT_SPECIFIED --- utils/regr/regrmysql.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 5faed3b0b..32cd41209 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -168,7 +168,7 @@ extern "C" return 1; } - if (args->arg_type[1] == DECIMAL_RESULT) + if (initid->decimals != DECIMAL_NOT_SPECIFIED) { initid->decimals +=4; } @@ -272,7 +272,7 @@ extern "C" return 1; } - if (args->arg_type[0] == DECIMAL_RESULT) + if (initid->decimals != DECIMAL_NOT_SPECIFIED) { initid->decimals +=4; } From 138a6c55923cbafc9f589cb4a633cc5f3a396b9b Mon Sep 17 00:00:00 2001 From: David Mott Date: Fri, 19 Apr 2019 11:00:43 -0500 Subject: [PATCH 25/40] move cmake scripts to cmake folder add boost super build project (currently disabled) declare BOOST_NO_CXX11_SCOPED_ENUMS on projects that use boost::filesystem --- .gitignore | 1 + CMakeLists.txt | 50 +-- bison.cmake | 81 ---- .../FindJeMalloc.cmake | 0 FindSnappy.cmake => cmake/FindSnappy.cmake | 0 cmake/boost.CMakeLists.txt.in | 33 ++ cmake/boost.cmake | 34 ++ .../check_compiler_flag.cmake | 0 .../columnstore_version.cmake | 0 .../configureEngine.cmake | 0 .../cpackEngineDEB.cmake | 0 .../cpackEngineRPM.cmake | 0 cmake/superbuild.md | 24 ++ config.h.cmake | 390 ------------------ config.h.in | 266 ++++++------ dbcon/ddlpackage/CMakeLists.txt | 1 - dbcon/mysql/CMakeLists.txt | 1 - utils/configcpp/CMakeLists.txt | 2 + utils/idbdatafile/CMakeLists.txt | 2 + writeengine/redistribute/CMakeLists.txt | 3 + 20 files changed, 252 insertions(+), 636 deletions(-) delete mode 100644 bison.cmake rename FindJeMalloc.cmake => cmake/FindJeMalloc.cmake (100%) rename FindSnappy.cmake => cmake/FindSnappy.cmake (100%) create mode 100644 cmake/boost.CMakeLists.txt.in create mode 100644 cmake/boost.cmake rename check_compiler_flag.cmake => cmake/check_compiler_flag.cmake (100%) rename columnstore_version.cmake => cmake/columnstore_version.cmake (100%) rename configureEngine.cmake => cmake/configureEngine.cmake (100%) rename cpackEngineDEB.cmake => cmake/cpackEngineDEB.cmake (100%) rename cpackEngineRPM.cmake => cmake/cpackEngineRPM.cmake (100%) create mode 100644 cmake/superbuild.md delete mode 100644 config.h.cmake diff --git a/.gitignore b/.gitignore index f7b16ba8a..93be85e28 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,4 @@ _CPack_Packages columnstoreversion.h .idea/ .build +/.vs diff --git a/CMakeLists.txt b/CMakeLists.txt index 9fa59a1c6..dc1b02483 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,17 +34,25 @@ IF(NOT CMAKE_BUILD_TYPE) SET(CMAKE_BUILD_TYPE RELWITHDEBINFO CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) ENDIF(NOT CMAKE_BUILD_TYPE) -SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") - -SET(CMAKE_CXX_STANDARD 11) -SET(CMAKE_CXX_STANDARD_REQUIRED TRUE) -SET(CMAKE_CXX_EXTENSIONS FALSE) -SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) -SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) -SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) -SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) +SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -INCLUDE(columnstore_version.cmake) +INCLUDE(ExternalProject) + +SET(CMAKE_CXX_STANDARD 11) +SET(CMAKE_CXX_STANDARD_REQUIRED TRUE) +SET(CMAKE_CXX_EXTENSIONS FALSE) +SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) +SET(CMAKE_POSITION_INDEPENDENT_CODE TRUE) +SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) +SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) +SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) + +SET_PROPERTY(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external) +LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) + +find_package(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) +find_package(BISON REQUIRED) +INCLUDE(columnstore_version) SET (PACKAGE columnstore) SET (PACKAGE_NAME columnstore) @@ -82,13 +90,13 @@ IF("${isSystemDir}" STREQUAL "-1") SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH};${INSTALL_ENGINE}/mysql/lib") ENDIF("${isSystemDir}" STREQUAL "-1") -INCLUDE (configureEngine.cmake) +INCLUDE (configureEngine) # releasenum is used by external scripts for various tasks. Leave it alone. CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL_ENGINE} COMPONENT platform) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) exec_program("git" ${CMAKE_CURRENT_SOURCE_DIR} @@ -98,8 +106,6 @@ exec_program("git" CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${INSTALL_ENGINE} COMPONENT platform) -INCLUDE(bison.cmake) - FIND_PROGRAM(LEX_EXECUTABLE flex DOC "path to the flex executable") if(NOT LEX_EXECUTABLE) FIND_PROGRAM(LEX_EXECUTABLE lex DOC "path to the lex executable") @@ -108,13 +114,13 @@ if(NOT LEX_EXECUTABLE) endif() endif() -INCLUDE (FindLibXml2) +FIND_PACKAGE(LibXml2) if (NOT LIBXML2_FOUND) MESSAGE(FATAL_ERROR "Could not find a usable libxml2 development environment!") endif() -INCLUDE (FindSnappy.cmake) +INCLUDE (FindSnappy) if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() @@ -123,7 +129,7 @@ endif() IF (EXISTS "/etc/SuSE-release") SET(JEMALLOC_LIBRARIES "") ELSE () - INCLUDE (FindJeMalloc.cmake) + INCLUDE (FindJeMalloc) if (NOT JEMALLOC_FOUND) message(FATAL_ERROR "jemalloc not found!") SET(JEMALLOC_LIBRARIES "") @@ -135,7 +141,7 @@ if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") endif() -INCLUDE(check_compiler_flag.cmake) +INCLUDE(check_compiler_flag) MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) @@ -164,10 +170,6 @@ IF(SECURITY_HARDENED) ENDIF() SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") - - -FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) - SET (ENGINE_LIBDIR "${INSTALL_ENGINE}/lib") SET (ENGINE_BINDIR "${INSTALL_ENGINE}/bin") SET (ENGINE_INCDIR "${INSTALL_ENGINE}/include") @@ -283,6 +285,6 @@ ADD_SUBDIRECTORY(writeengine/server) ADD_SUBDIRECTORY(writeengine/bulk) ADD_SUBDIRECTORY(writeengine/splitter) -INCLUDE(cpackEngineRPM.cmake) -INCLUDE(cpackEngineDEB.cmake) +INCLUDE(cpackEngineRPM) +INCLUDE(cpackEngineDEB) diff --git a/bison.cmake b/bison.cmake deleted file mode 100644 index d5c725fbb..000000000 --- a/bison.cmake +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2009 Sun Microsystems, Inc. -# Use is subject to license terms. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -IF(CMAKE_SYSTEM_NAME MATCHES "SunOS") - # On Solaris, /opt/csw often contains a newer bison - IF(NOT BISON_EXECUTABLE AND EXISTS /opt/csw/bin/bison) - SET(BISON_EXECUTABLE /opt/csw/bin/bison) - ENDIF() -ENDIF() -FIND_PROGRAM(BISON_EXECUTABLE bison DOC "path to the bison executable") -MARK_AS_ADVANCED(BISON_EXECUTABLE "") -IF(NOT BISON_EXECUTABLE) - MESSAGE("Warning: Bison executable not found in PATH") -ELSEIF(BISON_EXECUTABLE AND NOT BISON_USABLE) - # Check version as well - EXEC_PROGRAM(${BISON_EXECUTABLE} ARGS --version OUTPUT_VARIABLE BISON_VERSION_STR) - # Get first line in case it's multiline - STRING(REGEX REPLACE "([^\n]+).*" "\\1" FIRST_LINE "${BISON_VERSION_STR}") - # get version information - STRING(REGEX REPLACE ".* ([0-9]+)\\.([0-9]+)" "\\1" BISON_VERSION_MAJOR "${FIRST_LINE}") - STRING(REGEX REPLACE ".* ([0-9]+)\\.([0-9]+)" "\\2" BISON_VERSION_MINOR "${FIRST_LINE}") - IF (BISON_VERSION_MAJOR LESS 2) - MESSAGE("Warning: bison version is old. please update to version 2") - ELSE() - SET(BISON_USABLE 1 CACHE INTERNAL "Bison version 2 or higher") - ENDIF() -ENDIF() - -# Use bison to generate C++ and header file -MACRO (RUN_BISON input_yy output_cc output_h) - IF(BISON_TOO_OLD) - IF(EXISTS ${output_cc} AND EXISTS ${output_h}) - SET(BISON_USABLE FALSE) - ENDIF() - ENDIF() - IF(BISON_USABLE) - ADD_CUSTOM_COMMAND( - OUTPUT ${output_cc} - ${output_h} - COMMAND ${BISON_EXECUTABLE} -y -p MYSQL - --output=${output_cc} - --defines=${output_h} - ${input_yy} - DEPENDS ${input_yy} - ) - ELSE() - # Bison is missing or not usable, e.g too old - IF(EXISTS ${output_cc} AND EXISTS ${output_h}) - IF(${input_yy} IS_NEWER_THAN ${output_cc} OR ${input_yy} IS_NEWER_THAN ${output_h}) - # Possibly timestamps are messed up in source distribution. - MESSAGE("Warning: no usable bison found, ${input_yy} will not be rebuilt.") - ENDIF() - ELSE() - # Output files are missing, bail out. - SET(ERRMSG - "Bison (GNU parser generator) is required to build MySQL." - "Please install bison." - ) - IF(WIN32) - SET(ERRMSG ${ERRMSG} - "You can download bison from http://gnuwin32.sourceforge.net/packages/bison.htm " - "Choose 'Complete package, except sources' installation. We recommend to " - "install bison into a directory without spaces, e.g C:\\GnuWin32.") - ENDIF() - MESSAGE(FATAL_ERROR ${ERRMSG}) - ENDIF() - ENDIF() -ENDMACRO() diff --git a/FindJeMalloc.cmake b/cmake/FindJeMalloc.cmake similarity index 100% rename from FindJeMalloc.cmake rename to cmake/FindJeMalloc.cmake diff --git a/FindSnappy.cmake b/cmake/FindSnappy.cmake similarity index 100% rename from FindSnappy.cmake rename to cmake/FindSnappy.cmake diff --git a/cmake/boost.CMakeLists.txt.in b/cmake/boost.CMakeLists.txt.in new file mode 100644 index 000000000..5df75c405 --- /dev/null +++ b/cmake/boost.CMakeLists.txt.in @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION @CMAKE_VERSION@) + +include(ExternalProject) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + set(_toolset "gcc") +elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") + set(_toolset "clang") +elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") + set(_toolset "intel-linux") +endif() + +set(_b2args link=shared;threading=multi;variant=release;toolset=${_toolset};--with-system;--with-filesystem;--with-thread;--with-regex;--with-date_time) + +ExternalProject_Add(boost + PREFIX build + URL https://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.zip + URL_HASH SHA256=ae85620e810b87a03e1acf8bbf0d4ad87c0cf7040cf6a4e1d8958488ebe42e7e + DOWNLOAD_NO_PROGRESS TRUE + UPDATE_COMMAND "" + CONFIGURE_COMMAND /bootstrap.sh + --with-toolset=${_toolset} + --prefix=${CMAKE_CURRENT_SOURCE_DIR}/../boost + --with-libraries=system,filesystem,thread,regex,date_time + BUILD_COMMAND /b2 -q ${_b2args} + LOG_BUILD TRUE + BUILD_IN_SOURCE TRUE + INSTALL_COMMAND /b2 -q install ${_b2args} + LOG_INSTALL TRUE +) + +unset(_b2args) +unset(_toolset) \ No newline at end of file diff --git a/cmake/boost.cmake b/cmake/boost.cmake new file mode 100644 index 000000000..ae129287d --- /dev/null +++ b/cmake/boost.cmake @@ -0,0 +1,34 @@ + +# boost super build (see superbuild.md) + + +configure_file(${CMAKE_CURRENT_LIST_DIR}/boost.CMakeLists.txt.in ${CMAKE_CURRENT_BINARY_DIR}/.boost/CMakeLists.txt @ONLY) + +execute_process( + COMMAND ${CMAKE_COMMAND} . + -G "${CMAKE_GENERATOR}" + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/.boost + RESULT_VARIABLE _exec_ret +) + +if(${_exec_ret}) + message(FATAL_ERROR "Error ${_exec_ret} configuring boost dependency.") +endif() + +execute_process( + COMMAND ${CMAKE_COMMAND} --build . + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/.boost + RESULT_VARIABLE _exec_ret +) + +if(${_exec_ret}) + message(FATAL_ERROR "Error ${_exec_ret} building boost dependency: ${_exec_ret}") +endif() + +unset(_exec_ret) + +set(BOOST_ROOT ${CMAKE_CURRENT_BINARY_DIR}/boost) + +find_package(Boost 1.55.0 REQUIRED COMPONENTS system filesystem thread regex date_time) diff --git a/check_compiler_flag.cmake b/cmake/check_compiler_flag.cmake similarity index 100% rename from check_compiler_flag.cmake rename to cmake/check_compiler_flag.cmake diff --git a/columnstore_version.cmake b/cmake/columnstore_version.cmake similarity index 100% rename from columnstore_version.cmake rename to cmake/columnstore_version.cmake diff --git a/configureEngine.cmake b/cmake/configureEngine.cmake similarity index 100% rename from configureEngine.cmake rename to cmake/configureEngine.cmake diff --git a/cpackEngineDEB.cmake b/cmake/cpackEngineDEB.cmake similarity index 100% rename from cpackEngineDEB.cmake rename to cmake/cpackEngineDEB.cmake diff --git a/cpackEngineRPM.cmake b/cmake/cpackEngineRPM.cmake similarity index 100% rename from cpackEngineRPM.cmake rename to cmake/cpackEngineRPM.cmake diff --git a/cmake/superbuild.md b/cmake/superbuild.md new file mode 100644 index 000000000..6bedaeabc --- /dev/null +++ b/cmake/superbuild.md @@ -0,0 +1,24 @@ +CMake Super Build +================= +A super build is a process to download, build and install dependencies with cmake at configure time. This ensure dependencies are available during the initial configure stage. Its accomplished by executing separate cmake configure and build processes inline with the main project cmake which builds and installs the missing dependency. + +Rationale: +---------- +It maybe observed that ExternalProject accomplishes a similar task, however, the target of an ExternalProject is not available until after the build stage. Any scripting logic which requires the dependency during the configure stage will fail. The super build solves this by ensuring the dependency is built independent of the main projects configuration which uses it. + +Example: +-------- +# In the context of the main projects cmake scripts, subshells of cmake are executed to configure and build the dependency +configure_file(some_dependency.CMakeLists.txt.in some_dep_dir\CMakeLists.txt @ONLY) # drop a top-level CMakeLists.txt in a folder for the dependency +execute_process(COMMAND ${CMAKE_COMMAND} . WORKING_DIRECTORY some_dep_dir) # execute configure stage of dependency against newly created CMakeLists.txt from above step +execute_process(COMMAND ${CMAKE_COMMAND} --build . WORKING_DIRECTORY some_dep_dir) # install the dependency +find_package(some_dependency) # the dependency should be installed and can be 'found' or used as appropriate + +NOTES +----- + o The bulk of the work is performed in the generated/copied CMakeLists.txt to download (optional), build and install the dependency. It typically contains the full set of ExternalProject statements and error handling. + o CMake scripts executed in a sub-process with execute_process are independent and share no state whatsoever with the calling process. There are two ways to share state with the sub-shell + - Wrap appropriate @VARIABLE@ decorations in the CMakeLists.in template which get substituted with values when configure_file is executed + - Pass them on the command line of the execute_process statement. e.g.: execute_process(COMMAND ${CMAKE_COMMAND} -DSOME_VAR=${SOME_VAL} -DANOTHER_VAR=${ANOTHER_VAL} ... + +x \ No newline at end of file diff --git a/config.h.cmake b/config.h.cmake deleted file mode 100644 index 74d707b11..000000000 --- a/config.h.cmake +++ /dev/null @@ -1,390 +0,0 @@ -/* config.h.cmake */ -#ifndef TEST_CONFIG_H -#define TEST_CONFIG_H - -/* Define to 1 to let the system come up without using OAM */ -#cmakedefine SKIP_OAM_INIT 1 - -/* Define to 1 if you have the `alarm' function. */ -#cmakedefine HAVE_ALARM 1 - -/* Define to 1 if you have `alloca', as a function or macro. */ -#cmakedefine HAVE_ALLOCA 1 - -/* Define to 1 if you have and it should be used (not on Ultrix). - */ -#cmakedefine HAVE_ALLOCA_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_ARPA_INET_H 1 - -/* Define to 1 if you have the `btowc' function. */ -#cmakedefine HAVE_BTOWC 1 - -/* Define to 1 if you have the declaration of `getenv', and to 0 if you don't. - */ -#cmakedefine HAVE_DECL_GETENV 1 - -/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you - don't. */ -#cmakedefine HAVE_DECL_STRERROR_R 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_DLFCN_H 1 - -/* Define to 1 if you have the `dup2' function. */ -#cmakedefine HAVE_DUP2 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_FCNTL_H 1 - -/* Define to 1 if you have the `floor' function. */ -#cmakedefine HAVE_FLOOR 1 - -/* Define to 1 if you have the `fork' function. */ -#cmakedefine HAVE_FORK 1 - -/* Define to 1 if you have the `ftime' function. */ -#cmakedefine HAVE_FTIME 1 - -/* Define to 1 if you have the `ftruncate' function. */ -#cmakedefine HAVE_FTRUNCATE 1 - -/* Define to 1 if you have the `gethostbyname' function. */ -#cmakedefine HAVE_GETHOSTBYNAME 1 - -/* Define to 1 if you have the `getpagesize' function. */ -#cmakedefine HAVE_GETPAGESIZE 1 - -/* Define to 1 if you have the `gettimeofday' function. */ -#cmakedefine HAVE_GETTIMEOFDAY 1 - -/* Define to 1 if you have the `inet_ntoa' function. */ -#cmakedefine HAVE_INET_NTOA 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the `isascii' function. */ -#cmakedefine HAVE_ISASCII 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_LIMITS_H 1 - -/* Define to 1 if you have the `localtime_r' function. */ -#cmakedefine HAVE_LOCALTIME_R 1 - -/* Define to 1 if your system has a GNU libc compatible `malloc' function, and - to 0 otherwise. */ -#cmakedefine HAVE_MALLOC 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_MALLOC_H 1 - -/* Define to 1 if you have the `mbsrtowcs' function. */ -#cmakedefine HAVE_MBSRTOWCS 1 - -/* Define to 1 if declares mbstate_t. */ -#cmakedefine HAVE_MBSTATE_T 1 - -/* Define to 1 if you have the `memchr' function. */ -#cmakedefine HAVE_MEMCHR 1 - -/* Define to 1 if you have the `memmove' function. */ -#cmakedefine HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_MEMORY_H 1 - -/* Define to 1 if you have the `mempcpy' function. */ -#cmakedefine HAVE_MEMPCPY 1 - -/* Define to 1 if you have the `memset' function. */ -#cmakedefine HAVE_MEMSET 1 - -/* Define to 1 if you have the `mkdir' function. */ -#cmakedefine HAVE_MKDIR 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_NCURSES_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_NETDB_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_NETINET_IN_H 1 - -/* Define to 1 if you have the `pow' function. */ -#cmakedefine HAVE_POW 1 - -/* Define to 1 if the system has the type `ptrdiff_t'. */ -#cmakedefine HAVE_PTRDIFF_T 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_READLINE_READLINE_H 1 - -/* Define to 1 if you have the `regcomp' function. */ -#cmakedefine HAVE_REGCOMP 1 - -/* Define to 1 if you have the `rmdir' function. */ -#cmakedefine HAVE_RMDIR 1 - -/* Define to 1 if you have the `select' function. */ -#cmakedefine HAVE_SELECT 1 - -/* Define to 1 if you have the `setenv' function. */ -#cmakedefine HAVE_SETENV 1 - -/* Define to 1 if you have the `setlocale' function. */ -#cmakedefine HAVE_SETLOCALE 1 - -/* Define to 1 if you have the `socket' function. */ -#cmakedefine HAVE_SOCKET 1 - -/* Define to 1 if `stat' has the bug that it succeeds when given the - zero-length file name argument. */ -#cmakedefine HAVE_STAT_EMPTY_STRING_BUG 1 - -/* Define to 1 if stdbool.h conforms to C99. */ -#cmakedefine HAVE_STDBOOL_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STDDEF_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STDLIB_H 1 - -/* Define to 1 if you have the `strcasecmp' function. */ -#cmakedefine HAVE_STRCASECMP 1 - -/* Define to 1 if you have the `strchr' function. */ -#cmakedefine HAVE_STRCHR 1 - -/* Define to 1 if you have the `strcspn' function. */ -#cmakedefine HAVE_STRCSPN 1 - -/* Define to 1 if you have the `strdup' function. */ -#cmakedefine HAVE_STRDUP 1 - -/* Define to 1 if you have the `strerror' function. */ -#cmakedefine HAVE_STRERROR 1 - -/* Define to 1 if you have the `strerror_r' function. */ -#cmakedefine HAVE_STRERROR_R 1 - -/* Define to 1 if you have the `strftime' function. */ -#cmakedefine HAVE_STRFTIME 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STRING_H 1 - -/* Define to 1 if you have the `strrchr' function. */ -#cmakedefine HAVE_STRRCHR 1 - -/* Define to 1 if you have the `strspn' function. */ -#cmakedefine HAVE_STRSPN 1 - -/* Define to 1 if you have the `strstr' function. */ -#cmakedefine HAVE_STRSTR 1 - -/* Define to 1 if you have the `strtol' function. */ -#cmakedefine HAVE_STRTOL 1 - -/* Define to 1 if you have the `strtoul' function. */ -#cmakedefine HAVE_STRTOUL 1 - -/* Define to 1 if you have the `strtoull' function. */ -#cmakedefine HAVE_STRTOULL 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYSLOG_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_FILE_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_MOUNT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_SELECT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_SOCKET_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_STATFS_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_TIMEB_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_TIME_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_TYPES_H 1 - -/* Define to 1 if you have that is POSIX.1 compatible. */ -#cmakedefine HAVE_SYS_WAIT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_UNISTD_H 1 - -/* Define to 1 if you have the `utime' function. */ -#cmakedefine HAVE_UTIME 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_UTIME_H 1 - -/* Define to 1 if `utime(file, NULL)' sets file's timestamp to the present. */ -#cmakedefine HAVE_UTIME_NULL 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_VALUES_H 1 - -/* Define to 1 if you have the `vfork' function. */ -#cmakedefine HAVE_VFORK 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_VFORK_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_WCHAR_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_WCTYPE_H 1 - -/* Define to 1 if you have the `wmempcpy' function. */ -#cmakedefine HAVE_WMEMPCPY 1 - -/* Define to 1 if `fork' works. */ -#cmakedefine HAVE_WORKING_FORK 1 - -/* Define to 1 if `vfork' works. */ -#cmakedefine HAVE_WORKING_VFORK 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_ZLIB_H 1 - -/* Define to 1 if the system has the type `_Bool'. */ -#cmakedefine HAVE__BOOL 1 - -/* Define to 1 if `lstat' dereferences a symlink specified with a trailing - slash. */ -#cmakedefine LSTAT_FOLLOWS_SLASHED_SYMLINK 1 - -/* Name of package */ -#cmakedefine PACKAGE "${PACKAGE}" - -/* Define to the address where bug reports for this package should be sent. */ -#cmakedefine PACKAGE_BUGREPORT "${PACKAGE_BUGREPORT}" - -/* Define to the full name of this package. */ -#cmakedefine PACKAGE_NAME "${PACKAGE_NAME}" - -/* Define to the full name and version of this package. */ -#cmakedefine PACKAGE_STRING "${PACKAGE_STRING}" - -/* Define to the one symbol short name of this package. */ -#cmakedefine PACKAGE_TARNAME "${PACKAGE_TARNAME}" - -/* Define to the home page for this package. */ -#cmakedefine PACKAGE_URL "${PACKAGE_URL}" - -/* Define to the version of this package. */ -#cmakedefine PACKAGE_VERSION "${PACKAGE_VERSION}" - -/* Define as the return type of signal handlers (`int' or `void'). */ -#cmakedefine RETSIGTYPE ${RETSIGTYPE} - -/* Define to the type of arg 1 for `select'. */ -#cmakedefine SELECT_TYPE_ARG1 ${SELECT_TYPE_ARG1} - -/* Define to the type of args 2, 3 and 4 for `select'. */ -#cmakedefine SELECT_TYPE_ARG234 (${SELECT_TYPE_ARG234}) - -/* Define to the type of arg 5 for `select'. */ -#cmakedefine SELECT_TYPE_ARG5 (${SELECT_TYPE_ARG5}) - -/* Define to 1 if the `S_IS*' macros in do not work properly. */ -#cmakedefine STAT_MACROS_BROKEN 1 - -/* Define to 1 if you have the ANSI C header files. */ -#cmakedefine STDC_HEADERS 1 - -/* Define to 1 if strerror_r returns char *. */ -#cmakedefine STRERROR_R_CHAR_P 1 - -/* Define to 1 if you can safely include both and . */ -#cmakedefine TIME_WITH_SYS_TIME 1 - -/* Define to 1 if your declares `struct tm'. */ -#cmakedefine TM_IN_SYS_TIME 1 - -/* Version number of package */ -#cmakedefine VERSION "${VERSION}" - -/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a - `char[]'. */ -#cmakedefine YYTEXT_POINTER 1 - -/* Define to empty if `const' does not conform to ANSI C. */ -#cmakedefine const - -/* Define to rpl_fnmatch if the replacement function should be used. */ -#cmakedefine fnmatch - -/* Define to `__inline__' or `__inline' if that's what the C compiler - calls it, or to nothing if 'inline' is not supported under any name. */ -#ifndef __cplusplus -#cmakedefine inline ${inline} -#endif - -/* Define to rpl_malloc if the replacement function should be used. */ -#cmakedefine malloc - -/* Define to a type if does not define. */ -#cmakedefine mbstate_t - -/* Define to `int' if does not define. */ -#cmakedefine mode_t ${mode_t} - -/* Define to `long int' if does not define. */ -#cmakedefine off_t ${off_t} - -/* Define to `int' if does not define. */ -#cmakedefine pid_t ${pid_t} - -/* Define to the equivalent of the C99 'restrict' keyword, or to - nothing if this is not supported. Do not define if restrict is - supported directly. */ -#cmakedefine restrict ${restrict} -/* Work around a bug in Sun C++: it does not support _Restrict or - __restrict__, even though the corresponding Sun C compiler ends up with - "#define restrict _Restrict" or "#define restrict __restrict__" in the - previous line. Perhaps some future version of Sun C++ will work with - restrict; if so, hopefully it defines __RESTRICT like Sun C does. */ -#if defined __SUNPRO_CC && !defined __RESTRICT -# define _Restrict -# define __restrict__ -#endif - -/* Define to `unsigned int' if does not define. */ -#cmakedefine size_t ${size_t} - -/* Define as `fork' if `vfork' does not work. */ -#cmakedefine vfork ${VFORK} - -/* Define to empty if the keyword `volatile' does not work. Warning: valid - code using `volatile' can become incorrect without. Disable with care. */ -#cmakedefine volatile - -#endif diff --git a/config.h.in b/config.h.in index 6a451addb..74d707b11 100644 --- a/config.h.in +++ b/config.h.in @@ -1,386 +1,372 @@ -/* config.h.in. Generated from configure.ac by autoheader. */ +/* config.h.cmake */ +#ifndef TEST_CONFIG_H +#define TEST_CONFIG_H -/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP - systems. This function is required for `alloca.c' support on those systems. - */ -#undef CRAY_STACKSEG_END - -/* Define to 1 if using `alloca.c'. */ -#undef C_ALLOCA +/* Define to 1 to let the system come up without using OAM */ +#cmakedefine SKIP_OAM_INIT 1 /* Define to 1 if you have the `alarm' function. */ -#undef HAVE_ALARM +#cmakedefine HAVE_ALARM 1 /* Define to 1 if you have `alloca', as a function or macro. */ -#undef HAVE_ALLOCA +#cmakedefine HAVE_ALLOCA 1 /* Define to 1 if you have and it should be used (not on Ultrix). */ -#undef HAVE_ALLOCA_H +#cmakedefine HAVE_ALLOCA_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_ARPA_INET_H +#cmakedefine HAVE_ARPA_INET_H 1 /* Define to 1 if you have the `btowc' function. */ -#undef HAVE_BTOWC +#cmakedefine HAVE_BTOWC 1 /* Define to 1 if you have the declaration of `getenv', and to 0 if you don't. */ -#undef HAVE_DECL_GETENV +#cmakedefine HAVE_DECL_GETENV 1 /* Define to 1 if you have the declaration of `strerror_r', and to 0 if you don't. */ -#undef HAVE_DECL_STRERROR_R +#cmakedefine HAVE_DECL_STRERROR_R 1 /* Define to 1 if you have the header file. */ -#undef HAVE_DLFCN_H +#cmakedefine HAVE_DLFCN_H 1 /* Define to 1 if you have the `dup2' function. */ -#undef HAVE_DUP2 +#cmakedefine HAVE_DUP2 1 /* Define to 1 if you have the header file. */ -#undef HAVE_FCNTL_H +#cmakedefine HAVE_FCNTL_H 1 /* Define to 1 if you have the `floor' function. */ -#undef HAVE_FLOOR +#cmakedefine HAVE_FLOOR 1 /* Define to 1 if you have the `fork' function. */ -#undef HAVE_FORK +#cmakedefine HAVE_FORK 1 /* Define to 1 if you have the `ftime' function. */ -#undef HAVE_FTIME +#cmakedefine HAVE_FTIME 1 /* Define to 1 if you have the `ftruncate' function. */ -#undef HAVE_FTRUNCATE +#cmakedefine HAVE_FTRUNCATE 1 /* Define to 1 if you have the `gethostbyname' function. */ -#undef HAVE_GETHOSTBYNAME +#cmakedefine HAVE_GETHOSTBYNAME 1 /* Define to 1 if you have the `getpagesize' function. */ -#undef HAVE_GETPAGESIZE +#cmakedefine HAVE_GETPAGESIZE 1 /* Define to 1 if you have the `gettimeofday' function. */ -#undef HAVE_GETTIMEOFDAY +#cmakedefine HAVE_GETTIMEOFDAY 1 /* Define to 1 if you have the `inet_ntoa' function. */ -#undef HAVE_INET_NTOA +#cmakedefine HAVE_INET_NTOA 1 /* Define to 1 if you have the header file. */ -#undef HAVE_INTTYPES_H +#cmakedefine HAVE_INTTYPES_H 1 /* Define to 1 if you have the `isascii' function. */ -#undef HAVE_ISASCII +#cmakedefine HAVE_ISASCII 1 /* Define to 1 if you have the header file. */ -#undef HAVE_LIMITS_H +#cmakedefine HAVE_LIMITS_H 1 /* Define to 1 if you have the `localtime_r' function. */ -#undef HAVE_LOCALTIME_R +#cmakedefine HAVE_LOCALTIME_R 1 /* Define to 1 if your system has a GNU libc compatible `malloc' function, and to 0 otherwise. */ -#undef HAVE_MALLOC +#cmakedefine HAVE_MALLOC 1 /* Define to 1 if you have the header file. */ -#undef HAVE_MALLOC_H +#cmakedefine HAVE_MALLOC_H 1 /* Define to 1 if you have the `mbsrtowcs' function. */ -#undef HAVE_MBSRTOWCS +#cmakedefine HAVE_MBSRTOWCS 1 /* Define to 1 if declares mbstate_t. */ -#undef HAVE_MBSTATE_T +#cmakedefine HAVE_MBSTATE_T 1 /* Define to 1 if you have the `memchr' function. */ -#undef HAVE_MEMCHR +#cmakedefine HAVE_MEMCHR 1 /* Define to 1 if you have the `memmove' function. */ -#undef HAVE_MEMMOVE +#cmakedefine HAVE_MEMMOVE 1 /* Define to 1 if you have the header file. */ -#undef HAVE_MEMORY_H +#cmakedefine HAVE_MEMORY_H 1 /* Define to 1 if you have the `mempcpy' function. */ -#undef HAVE_MEMPCPY +#cmakedefine HAVE_MEMPCPY 1 /* Define to 1 if you have the `memset' function. */ -#undef HAVE_MEMSET +#cmakedefine HAVE_MEMSET 1 /* Define to 1 if you have the `mkdir' function. */ -#undef HAVE_MKDIR +#cmakedefine HAVE_MKDIR 1 /* Define to 1 if you have the header file. */ -#undef HAVE_NCURSES_H +#cmakedefine HAVE_NCURSES_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_NETDB_H +#cmakedefine HAVE_NETDB_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_NETINET_IN_H +#cmakedefine HAVE_NETINET_IN_H 1 /* Define to 1 if you have the `pow' function. */ -#undef HAVE_POW +#cmakedefine HAVE_POW 1 /* Define to 1 if the system has the type `ptrdiff_t'. */ -#undef HAVE_PTRDIFF_T +#cmakedefine HAVE_PTRDIFF_T 1 /* Define to 1 if you have the header file. */ -#undef HAVE_READLINE_READLINE_H +#cmakedefine HAVE_READLINE_READLINE_H 1 /* Define to 1 if you have the `regcomp' function. */ -#undef HAVE_REGCOMP +#cmakedefine HAVE_REGCOMP 1 /* Define to 1 if you have the `rmdir' function. */ -#undef HAVE_RMDIR +#cmakedefine HAVE_RMDIR 1 /* Define to 1 if you have the `select' function. */ -#undef HAVE_SELECT +#cmakedefine HAVE_SELECT 1 /* Define to 1 if you have the `setenv' function. */ -#undef HAVE_SETENV +#cmakedefine HAVE_SETENV 1 /* Define to 1 if you have the `setlocale' function. */ -#undef HAVE_SETLOCALE +#cmakedefine HAVE_SETLOCALE 1 /* Define to 1 if you have the `socket' function. */ -#undef HAVE_SOCKET +#cmakedefine HAVE_SOCKET 1 /* Define to 1 if `stat' has the bug that it succeeds when given the zero-length file name argument. */ -#undef HAVE_STAT_EMPTY_STRING_BUG +#cmakedefine HAVE_STAT_EMPTY_STRING_BUG 1 /* Define to 1 if stdbool.h conforms to C99. */ -#undef HAVE_STDBOOL_H +#cmakedefine HAVE_STDBOOL_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STDDEF_H +#cmakedefine HAVE_STDDEF_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STDINT_H +#cmakedefine HAVE_STDINT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STDLIB_H +#cmakedefine HAVE_STDLIB_H 1 /* Define to 1 if you have the `strcasecmp' function. */ -#undef HAVE_STRCASECMP +#cmakedefine HAVE_STRCASECMP 1 /* Define to 1 if you have the `strchr' function. */ -#undef HAVE_STRCHR +#cmakedefine HAVE_STRCHR 1 /* Define to 1 if you have the `strcspn' function. */ -#undef HAVE_STRCSPN +#cmakedefine HAVE_STRCSPN 1 /* Define to 1 if you have the `strdup' function. */ -#undef HAVE_STRDUP +#cmakedefine HAVE_STRDUP 1 /* Define to 1 if you have the `strerror' function. */ -#undef HAVE_STRERROR +#cmakedefine HAVE_STRERROR 1 /* Define to 1 if you have the `strerror_r' function. */ -#undef HAVE_STRERROR_R +#cmakedefine HAVE_STRERROR_R 1 /* Define to 1 if you have the `strftime' function. */ -#undef HAVE_STRFTIME +#cmakedefine HAVE_STRFTIME 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STRINGS_H +#cmakedefine HAVE_STRINGS_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STRING_H +#cmakedefine HAVE_STRING_H 1 /* Define to 1 if you have the `strrchr' function. */ -#undef HAVE_STRRCHR +#cmakedefine HAVE_STRRCHR 1 /* Define to 1 if you have the `strspn' function. */ -#undef HAVE_STRSPN +#cmakedefine HAVE_STRSPN 1 /* Define to 1 if you have the `strstr' function. */ -#undef HAVE_STRSTR +#cmakedefine HAVE_STRSTR 1 /* Define to 1 if you have the `strtol' function. */ -#undef HAVE_STRTOL +#cmakedefine HAVE_STRTOL 1 /* Define to 1 if you have the `strtoul' function. */ -#undef HAVE_STRTOUL +#cmakedefine HAVE_STRTOUL 1 /* Define to 1 if you have the `strtoull' function. */ -#undef HAVE_STRTOULL +#cmakedefine HAVE_STRTOULL 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYSLOG_H +#cmakedefine HAVE_SYSLOG_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_FILE_H +#cmakedefine HAVE_SYS_FILE_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_MOUNT_H +#cmakedefine HAVE_SYS_MOUNT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_SELECT_H +#cmakedefine HAVE_SYS_SELECT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_SOCKET_H +#cmakedefine HAVE_SYS_SOCKET_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STATFS_H +#cmakedefine HAVE_SYS_STATFS_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STAT_H +#cmakedefine HAVE_SYS_STAT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TIMEB_H +#cmakedefine HAVE_SYS_TIMEB_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TIME_H +#cmakedefine HAVE_SYS_TIME_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TYPES_H +#cmakedefine HAVE_SYS_TYPES_H 1 /* Define to 1 if you have that is POSIX.1 compatible. */ -#undef HAVE_SYS_WAIT_H +#cmakedefine HAVE_SYS_WAIT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H +#cmakedefine HAVE_UNISTD_H 1 /* Define to 1 if you have the `utime' function. */ -#undef HAVE_UTIME +#cmakedefine HAVE_UTIME 1 /* Define to 1 if you have the header file. */ -#undef HAVE_UTIME_H +#cmakedefine HAVE_UTIME_H 1 /* Define to 1 if `utime(file, NULL)' sets file's timestamp to the present. */ -#undef HAVE_UTIME_NULL +#cmakedefine HAVE_UTIME_NULL 1 /* Define to 1 if you have the header file. */ -#undef HAVE_VALUES_H +#cmakedefine HAVE_VALUES_H 1 /* Define to 1 if you have the `vfork' function. */ -#undef HAVE_VFORK +#cmakedefine HAVE_VFORK 1 /* Define to 1 if you have the header file. */ -#undef HAVE_VFORK_H +#cmakedefine HAVE_VFORK_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_WCHAR_H +#cmakedefine HAVE_WCHAR_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_WCTYPE_H +#cmakedefine HAVE_WCTYPE_H 1 /* Define to 1 if you have the `wmempcpy' function. */ -#undef HAVE_WMEMPCPY +#cmakedefine HAVE_WMEMPCPY 1 /* Define to 1 if `fork' works. */ -#undef HAVE_WORKING_FORK +#cmakedefine HAVE_WORKING_FORK 1 /* Define to 1 if `vfork' works. */ -#undef HAVE_WORKING_VFORK +#cmakedefine HAVE_WORKING_VFORK 1 /* Define to 1 if you have the header file. */ -#undef HAVE_ZLIB_H +#cmakedefine HAVE_ZLIB_H 1 /* Define to 1 if the system has the type `_Bool'. */ -#undef HAVE__BOOL +#cmakedefine HAVE__BOOL 1 /* Define to 1 if `lstat' dereferences a symlink specified with a trailing slash. */ -#undef LSTAT_FOLLOWS_SLASHED_SYMLINK - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#undef LT_OBJDIR +#cmakedefine LSTAT_FOLLOWS_SLASHED_SYMLINK 1 /* Name of package */ -#undef PACKAGE +#cmakedefine PACKAGE "${PACKAGE}" /* Define to the address where bug reports for this package should be sent. */ -#undef PACKAGE_BUGREPORT +#cmakedefine PACKAGE_BUGREPORT "${PACKAGE_BUGREPORT}" /* Define to the full name of this package. */ -#undef PACKAGE_NAME +#cmakedefine PACKAGE_NAME "${PACKAGE_NAME}" /* Define to the full name and version of this package. */ -#undef PACKAGE_STRING +#cmakedefine PACKAGE_STRING "${PACKAGE_STRING}" /* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME +#cmakedefine PACKAGE_TARNAME "${PACKAGE_TARNAME}" /* Define to the home page for this package. */ -#undef PACKAGE_URL +#cmakedefine PACKAGE_URL "${PACKAGE_URL}" /* Define to the version of this package. */ -#undef PACKAGE_VERSION +#cmakedefine PACKAGE_VERSION "${PACKAGE_VERSION}" /* Define as the return type of signal handlers (`int' or `void'). */ -#undef RETSIGTYPE +#cmakedefine RETSIGTYPE ${RETSIGTYPE} /* Define to the type of arg 1 for `select'. */ -#undef SELECT_TYPE_ARG1 +#cmakedefine SELECT_TYPE_ARG1 ${SELECT_TYPE_ARG1} /* Define to the type of args 2, 3 and 4 for `select'. */ -#undef SELECT_TYPE_ARG234 +#cmakedefine SELECT_TYPE_ARG234 (${SELECT_TYPE_ARG234}) /* Define to the type of arg 5 for `select'. */ -#undef SELECT_TYPE_ARG5 - -/* If using the C implementation of alloca, define if you know the - direction of stack growth for your system; otherwise it will be - automatically deduced at runtime. - STACK_DIRECTION > 0 => grows toward higher addresses - STACK_DIRECTION < 0 => grows toward lower addresses - STACK_DIRECTION = 0 => direction of growth unknown */ -#undef STACK_DIRECTION +#cmakedefine SELECT_TYPE_ARG5 (${SELECT_TYPE_ARG5}) /* Define to 1 if the `S_IS*' macros in do not work properly. */ -#undef STAT_MACROS_BROKEN +#cmakedefine STAT_MACROS_BROKEN 1 /* Define to 1 if you have the ANSI C header files. */ -#undef STDC_HEADERS +#cmakedefine STDC_HEADERS 1 /* Define to 1 if strerror_r returns char *. */ -#undef STRERROR_R_CHAR_P +#cmakedefine STRERROR_R_CHAR_P 1 /* Define to 1 if you can safely include both and . */ -#undef TIME_WITH_SYS_TIME +#cmakedefine TIME_WITH_SYS_TIME 1 /* Define to 1 if your declares `struct tm'. */ -#undef TM_IN_SYS_TIME +#cmakedefine TM_IN_SYS_TIME 1 /* Version number of package */ -#undef VERSION +#cmakedefine VERSION "${VERSION}" /* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a `char[]'. */ -#undef YYTEXT_POINTER +#cmakedefine YYTEXT_POINTER 1 /* Define to empty if `const' does not conform to ANSI C. */ -#undef const +#cmakedefine const /* Define to rpl_fnmatch if the replacement function should be used. */ -#undef fnmatch +#cmakedefine fnmatch /* Define to `__inline__' or `__inline' if that's what the C compiler calls it, or to nothing if 'inline' is not supported under any name. */ #ifndef __cplusplus -#undef inline +#cmakedefine inline ${inline} #endif /* Define to rpl_malloc if the replacement function should be used. */ -#undef malloc +#cmakedefine malloc /* Define to a type if does not define. */ -#undef mbstate_t +#cmakedefine mbstate_t /* Define to `int' if does not define. */ -#undef mode_t +#cmakedefine mode_t ${mode_t} /* Define to `long int' if does not define. */ -#undef off_t +#cmakedefine off_t ${off_t} /* Define to `int' if does not define. */ -#undef pid_t +#cmakedefine pid_t ${pid_t} /* Define to the equivalent of the C99 'restrict' keyword, or to nothing if this is not supported. Do not define if restrict is supported directly. */ -#undef restrict +#cmakedefine restrict ${restrict} /* Work around a bug in Sun C++: it does not support _Restrict or __restrict__, even though the corresponding Sun C compiler ends up with "#define restrict _Restrict" or "#define restrict __restrict__" in the @@ -392,11 +378,13 @@ #endif /* Define to `unsigned int' if does not define. */ -#undef size_t +#cmakedefine size_t ${size_t} /* Define as `fork' if `vfork' does not work. */ -#undef vfork +#cmakedefine vfork ${VFORK} /* Define to empty if the keyword `volatile' does not work. Warning: valid code using `volatile' can become incorrect without. Disable with care. */ -#undef volatile +#cmakedefine volatile + +#endif diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 30bc97124..3b3e231da 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -11,7 +11,6 @@ ADD_CUSTOM_COMMAND( ADD_CUSTOM_TARGET(ddl-lexer DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ddl-scan.cpp) ADD_CUSTOM_TARGET(ddl-parser DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ddl-gram.cpp) # Parser puts extra info to stderr. -INCLUDE(../../check_compiler_flag.cmake) MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG=1" DEBUG) ########### next target ############### diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index fa2df6eaa..c74e19343 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -29,7 +29,6 @@ add_library(calmysql SHARED ${libcalmysql_SRCS}) target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) -SET_TARGET_PROPERTIES(calmysql PROPERTIES LINK_FLAGS "${calmysql_link_flags} -Wl,-E") set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) SET ( is_columnstore_tables_SRCS diff --git a/utils/configcpp/CMakeLists.txt b/utils/configcpp/CMakeLists.txt index cae83c461..f52f2a84e 100644 --- a/utils/configcpp/CMakeLists.txt +++ b/utils/configcpp/CMakeLists.txt @@ -9,5 +9,7 @@ add_library(configcpp SHARED ${configcpp_LIB_SRCS}) set_target_properties(configcpp PROPERTIES VERSION 1.0.0 SOVERSION 1) +target_compile_definitions(configcpp PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) + install(TARGETS configcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index e06bf1fc8..22190f68f 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -18,6 +18,8 @@ add_library(idbdatafile SHARED ${idbdatafile_LIB_SRCS}) target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES}) +target_compile_definitions(idbdatafile PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) + set_target_properties(idbdatafile PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS idbdatafile DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) diff --git a/writeengine/redistribute/CMakeLists.txt b/writeengine/redistribute/CMakeLists.txt index d62cff1a1..22308ced0 100644 --- a/writeengine/redistribute/CMakeLists.txt +++ b/writeengine/redistribute/CMakeLists.txt @@ -15,5 +15,8 @@ target_link_libraries(writeengineredistribute ${NETSNMP_LIBRARIES}) set_target_properties(writeengineredistribute PROPERTIES VERSION 1.0.0 SOVERSION 1) +target_compile_definitions(writeengineredistribute PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) + + install(TARGETS writeengineredistribute DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) From 1d9f47a55ce8dff2dbf5a258bbe99b9b2bc264d7 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 28 Feb 2018 19:20:16 +0300 Subject: [PATCH 26/40] MCOL-498. Segment files extension uses fallocate() now to optimize load put on SSD disks. --- utils/idbdatafile/BufferedFile.cpp | 17 ++++++++++ utils/idbdatafile/BufferedFile.h | 1 + utils/idbdatafile/IDBDataFile.h | 6 ++++ utils/idbdatafile/UnbufferedFile.cpp | 17 ++++++++++ utils/idbdatafile/UnbufferedFile.h | 1 + utils/idbhdfs/hdfs-shared/HdfsFile.cpp | 6 ++++ utils/idbhdfs/hdfs-shared/HdfsFile.h | 1 + .../hdfs-shared/HdfsRdwrFileBuffer.cpp | 5 +++ .../idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h | 1 + .../idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp | 5 +++ utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h | 1 + writeengine/shared/we_fileop.cpp | 33 +++++++++++++++---- writeengine/shared/we_fileop.h | 3 +- 13 files changed, 90 insertions(+), 7 deletions(-) diff --git a/utils/idbdatafile/BufferedFile.cpp b/utils/idbdatafile/BufferedFile.cpp index ddc3e9c66..15066456f 100644 --- a/utils/idbdatafile/BufferedFile.cpp +++ b/utils/idbdatafile/BufferedFile.cpp @@ -279,4 +279,21 @@ int BufferedFile::close() return ret; } +int BufferedFile::fallocate(int mode, off64_t offset, off64_t length) +{ + int ret = 0; + int savedErrno = 0; + + ret = ::fallocate( fileno(m_fp), mode, offset, length ); + savedErrno = errno; + + if ( ret == -1 && IDBLogger::isEnabled() ) + { + IDBLogger::logNoArg(m_fname, this, "fallocate", errno); + } + + errno = savedErrno; + return ret; +} + } diff --git a/utils/idbdatafile/BufferedFile.h b/utils/idbdatafile/BufferedFile.h index c411ca8de..3e6ae040e 100644 --- a/utils/idbdatafile/BufferedFile.h +++ b/utils/idbdatafile/BufferedFile.h @@ -50,6 +50,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); protected: /* virtual */ diff --git a/utils/idbdatafile/IDBDataFile.h b/utils/idbdatafile/IDBDataFile.h index 6d8046bb8..1ebb4a437 100644 --- a/utils/idbdatafile/IDBDataFile.h +++ b/utils/idbdatafile/IDBDataFile.h @@ -186,6 +186,12 @@ public: */ virtual time_t mtime() = 0; + /** + * The fallocate() method returns the modification time of the file in + * seconds. Returns -1 on error. + */ + virtual int fallocate(int mode, off64_t offset, off64_t length) = 0; + int colWidth() { return m_fColWidth; diff --git a/utils/idbdatafile/UnbufferedFile.cpp b/utils/idbdatafile/UnbufferedFile.cpp index 3b89e13cd..5f7072aa8 100644 --- a/utils/idbdatafile/UnbufferedFile.cpp +++ b/utils/idbdatafile/UnbufferedFile.cpp @@ -329,4 +329,21 @@ int UnbufferedFile::close() return ret; } +int UnbufferedFile::fallocate(int mode, off64_t offset, off64_t length) +{ + int ret = 0; + int savedErrno = 0; + + ret = ::fallocate( m_fd, mode, offset, length ); + savedErrno = errno; + + if ( ret == -1 && IDBLogger::isEnabled() ) + { + IDBLogger::logNoArg(m_fname, this, "fallocate", errno); + } + + errno = savedErrno; + return ret; +} + } diff --git a/utils/idbdatafile/UnbufferedFile.h b/utils/idbdatafile/UnbufferedFile.h index c2a2771e8..6564c59d7 100644 --- a/utils/idbdatafile/UnbufferedFile.h +++ b/utils/idbdatafile/UnbufferedFile.h @@ -47,6 +47,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); protected: /* virtual */ diff --git a/utils/idbhdfs/hdfs-shared/HdfsFile.cpp b/utils/idbhdfs/hdfs-shared/HdfsFile.cpp index 08260c13f..fc1ec6a2b 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsFile.cpp +++ b/utils/idbhdfs/hdfs-shared/HdfsFile.cpp @@ -348,4 +348,10 @@ int HdfsFile::close() return ret; } +int HdfsFile::fallocate(int mode, off64_t offset, off64_t length) +{ + return 0; +} + + } diff --git a/utils/idbhdfs/hdfs-shared/HdfsFile.h b/utils/idbhdfs/hdfs-shared/HdfsFile.h index 64c68d4e7..704643931 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsFile.h +++ b/utils/idbhdfs/hdfs-shared/HdfsFile.h @@ -62,6 +62,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); protected: /* virtual */ diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp index 7d934b3b9..b5795227d 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp @@ -317,4 +317,9 @@ int HdfsRdwrFileBuffer::close() return 0; } +int HdfsRdwrFileBuffer::fallocate(int mode, off64_t offset, off64_t length) +{ + return 0; +} + } diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h index 74fdf7020..8b922f97a 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h @@ -62,6 +62,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual*/ int fallocate(int mode, off64_t offset, off64_t length); protected: /* virtual */ diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp b/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp index 2e6ad3c13..44bea060d 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp @@ -507,5 +507,10 @@ int HdfsRdwrMemBuffer::close() return 0; } +int HdfsRdwrMemBuffer::fallocate(int mode, off64_t offset, off64_t length) +{ + return 0; +} + } diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h b/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h index c998d13a2..6258d6f70 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h @@ -62,6 +62,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); // Returns the total size of all currently allocated HdfsRdwrMemBuffers static size_t getTotalBuff() diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 7a2a2456b..939450980 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -834,7 +834,8 @@ int FileOp::extendFile( width, newFile, // new or existing file false, // don't expand; new extent - false ); // add full (not abbreviated) extent + false, // add full (not abbreviated) extent + true); // try to use fallocate first return rc; } @@ -1016,6 +1017,7 @@ int FileOp::addExtentExactFile( * headers will be included "if" it is a compressed file. * bExpandExtent (in) - Expand existing extent, or initialize a new one * bAbbrevExtent(in) - if creating new extent, is it an abbreviated extent + * bOptExtension(in) - full sized segment file allocation optimization flag * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1028,7 +1030,8 @@ int FileOp::initColumnExtent( int width, bool bNewFile, bool bExpandExtent, - bool bAbbrevExtent ) + bool bAbbrevExtent, + bool bOptExtension) { if ((bNewFile) && (m_compressionType)) { @@ -1071,6 +1074,7 @@ int FileOp::initColumnExtent( int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size int loopCount = 1; int remWriteSize = 0; + off64_t currFileSize = pFile->size(); if (nBlocks > MAX_NBLOCKS) // 64M row extent size { @@ -1099,10 +1103,14 @@ int FileOp::initColumnExtent( Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); #endif - - // Allocate buffer, and store in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of writeBuf. + int savedErrno = 0; + // Try to fallocate the space - fallback to write if fallocate has failed + if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize)) { + // Allocate buffer, store it in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of writeBuf. + // Save errno of the failed fallocate() to log it later. + savedErrno = errno; unsigned char* writeBuf = new unsigned char[writeSize]; boost::scoped_array writeBufPtr( writeBuf ); @@ -1159,6 +1167,18 @@ int FileOp::initColumnExtent( Stats::stopParseEvent(WE_STATS_CREATE_COL_EXTENT); #endif + // Log the fallocate() call result + std::ostringstream oss; + std::string errnoMsg; + Convertor::mapErrnoToString(savedErrno, errnoMsg); + oss << "FileOp::initColumnExtent(): fallocate(" << currFileSize << + ", " << writeSize << "): errno = " << savedErrno << + ": " << errnoMsg; + logging::Message::Args args; + args.add(oss.str()); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_INFO, + logging::M0006); + } return NO_ERROR; @@ -2790,7 +2810,8 @@ int FileOp::expandAbbrevColumnExtent( int rc = FileOp::initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width, false, // existing file true, // expand existing extent - false); // n/a since not adding new extent + false, // n/a since not adding new extent + true); // optimize segment file extension return rc; } diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index f4b7eac4a..8a81e9815 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -507,7 +507,8 @@ private: int width, bool bNewFile, bool bExpandExtent, - bool bAbbrevExtent ); + bool bAbbrevExtent, + bool bOptExtension=false ); static void initDbRootExtentMutexes(); static void removeDbRootExtentMutexes(); From 81fe7fa1a94d101a653e7ed73b9dadad746fe2eb Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 6 Mar 2018 14:20:46 +0300 Subject: [PATCH 27/40] MCOL-498. Add the knob to disable segment|dict file preallocation. Dict files extension uses fallocate() if possible. --- oam/etc/Columnstore.xml | 32 +++ utils/idbdatafile/BufferedFile.cpp | 2 +- utils/idbdatafile/CMakeLists.txt | 2 +- utils/idbdatafile/IDBPolicy.cpp | 20 ++ utils/idbdatafile/IDBPolicy.h | 12 ++ utils/idbdatafile/UnbufferedFile.cpp | 2 +- writeengine/dictionary/we_dctnry.cpp | 1 + writeengine/shared/we_fileop.cpp | 284 +++++++++++++++------------ writeengine/shared/we_fileop.h | 5 +- 9 files changed, 230 insertions(+), 130 deletions(-) diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index b5610bf11..60bc2d9fc 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -102,130 +102,162 @@ 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON 0.0.0.0 8620 + ON C diff --git a/utils/idbdatafile/BufferedFile.cpp b/utils/idbdatafile/BufferedFile.cpp index 15066456f..ae12d0fd4 100644 --- a/utils/idbdatafile/BufferedFile.cpp +++ b/utils/idbdatafile/BufferedFile.cpp @@ -287,7 +287,7 @@ int BufferedFile::fallocate(int mode, off64_t offset, off64_t length) ret = ::fallocate( fileno(m_fp), mode, offset, length ); savedErrno = errno; - if ( ret == -1 && IDBLogger::isEnabled() ) + if ( IDBLogger::isEnabled() ) { IDBLogger::logNoArg(m_fname, this, "fallocate", errno); } diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index e06bf1fc8..fd81d8da8 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -16,7 +16,7 @@ set(idbdatafile_LIB_SRCS add_library(idbdatafile SHARED ${idbdatafile_LIB_SRCS}) -target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES}) +target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS}) set_target_properties(idbdatafile PROPERTIES VERSION 1.0.0 SOVERSION 1) diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 4d0679308..a8c194918 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -18,12 +18,14 @@ #include #include #include +#include #include #include #include // to_upper #include #include "configcpp.h" // for Config +#include "oamcache.h" #include "IDBPolicy.h" #include "PosixFileSystem.h" //#include "HdfsFileSystem.h" @@ -48,6 +50,7 @@ int64_t IDBPolicy::s_hdfsRdwrBufferMaxSize = 0; std::string IDBPolicy::s_hdfsRdwrScratch; bool IDBPolicy::s_configed = false; boost::mutex IDBPolicy::s_mutex; +bool IDBPolicy::s_PreallocSpace = true; void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& hdfsRdwrScratch, int64_t hdfsRdwrBufferMaxSize ) { @@ -224,6 +227,23 @@ void IDBPolicy::configIDBPolicy() string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); string hdfsRdwrScratch = tmpDir + scratch; + // MCOL-498. Set the PMSX.PreallocSpace knob, where X is a PM number, + // to disable file space preallocation. The feature is used in the FileOp code + // and is enabled by default for a backward compatibility. + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + int PMId = oamcache->getLocalPMId(); + char configSectionPref[] = "PMS"; + char configSection[sizeof(configSectionPref)+oam::MAX_MODULE_ID_SIZE]; + ::memset(configSection, 0, sizeof(configSection)); + sprintf(configSection, "%s%d", configSectionPref, PMId); + string PreallocSpace = cf->getConfig(configSection, "PreallocSpace"); + + if ( PreallocSpace.length() != 0 ) + { + boost::to_upper(PreallocSpace); + s_PreallocSpace = ( PreallocSpace != "OFF" ); + } + IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); s_configed = true; diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h index 333b71808..5212f3b11 100644 --- a/utils/idbdatafile/IDBPolicy.h +++ b/utils/idbdatafile/IDBPolicy.h @@ -80,6 +80,11 @@ public: */ static bool useHdfs(); + /** + * Accessor method that returns whether or not HDFS is enabled + */ + static bool PreallocSpace(); + /** * Accessor method that returns whether to use HDFS memory buffers */ @@ -134,6 +139,7 @@ private: static bool isLocalFile( const std::string& path ); static bool s_usehdfs; + static bool s_PreallocSpace; static bool s_bUseRdwrMemBuffer; static std::string s_hdfsRdwrScratch; static int64_t s_hdfsRdwrBufferMaxSize; @@ -153,6 +159,12 @@ bool IDBPolicy::useHdfs() return s_usehdfs; } +inline +bool IDBPolicy::PreallocSpace() +{ + return s_PreallocSpace; +} + inline bool IDBPolicy::useRdwrMemBuffer() { diff --git a/utils/idbdatafile/UnbufferedFile.cpp b/utils/idbdatafile/UnbufferedFile.cpp index 5f7072aa8..200b583b4 100644 --- a/utils/idbdatafile/UnbufferedFile.cpp +++ b/utils/idbdatafile/UnbufferedFile.cpp @@ -337,7 +337,7 @@ int UnbufferedFile::fallocate(int mode, off64_t offset, off64_t length) ret = ::fallocate( m_fd, mode, offset, length ); savedErrno = errno; - if ( ret == -1 && IDBLogger::isEnabled() ) + if ( IDBLogger::isEnabled() ) { IDBLogger::logNoArg(m_fname, this, "fallocate", errno); } diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index 4e93ca60a..c5461d4e5 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -329,6 +329,7 @@ int Dctnry::expandDctnryExtent() blksToAdd, m_dctnryHeader2, m_totalHdrBytes, + true, true ); if (rc != NO_ERROR) diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 939450980..0b94d0d81 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -1017,7 +1017,7 @@ int FileOp::addExtentExactFile( * headers will be included "if" it is a compressed file. * bExpandExtent (in) - Expand existing extent, or initialize a new one * bAbbrevExtent(in) - if creating new extent, is it an abbreviated extent - * bOptExtension(in) - full sized segment file allocation optimization flag + * bOptExtension(in) - use fallocate() to extend the file if it is possible. * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1046,7 +1046,8 @@ int FileOp::initColumnExtent( } // @bug5769 Don't initialize extents or truncate db files on HDFS - if (idbdatafile::IDBPolicy::useHdfs()) + // MCOL-498 We don't need sequential segment files if a PM uses SSD either. + if (idbdatafile::IDBPolicy::useHdfs() || !idbdatafile::IDBPolicy::PreallocSpace()) { //@Bug 3219. update the compression header after the extent is expanded. if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) @@ -1100,85 +1101,91 @@ int FileOp::initColumnExtent( Stats::stopParseEvent(WE_STATS_WAIT_TO_EXPAND_COL_EXTENT); else Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); - - Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); #endif + int savedErrno = 0; - // Try to fallocate the space - fallback to write if fallocate has failed + // MCOL-498 Try to preallocate the space, fallback to write if fallocate has failed if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize)) { + savedErrno = errno; + // Log the failed fallocate() call result + if ( bOptExtension ) + { + std::ostringstream oss; + std::string errnoMsg; + Convertor::mapErrnoToString(savedErrno, errnoMsg); + oss << "FileOp::initColumnExtent(): fallocate(" << currFileSize << + ", " << writeSize << "): errno = " << savedErrno << + ": " << errnoMsg; + logging::Message::Args args; + args.add(oss.str()); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_INFO, + logging::M0006); + } + +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); +#endif // Allocate buffer, store it in scoped_array to insure it's deletion. // Create scope {...} to manage deletion of writeBuf. - // Save errno of the failed fallocate() to log it later. - savedErrno = errno; - unsigned char* writeBuf = new unsigned char[writeSize]; - boost::scoped_array writeBufPtr( writeBuf ); + { - setEmptyBuf( writeBuf, writeSize, emptyVal, width ); + unsigned char* writeBuf = new unsigned char[writeSize]; + boost::scoped_array writeBufPtr( writeBuf ); + + setEmptyBuf( writeBuf, writeSize, emptyVal, width ); + + #ifdef PROFILE + Stats::stopParseEvent(WE_STATS_INIT_COL_EXTENT); + + if (bExpandExtent) + Stats::startParseEvent(WE_STATS_EXPAND_COL_EXTENT); + else + Stats::startParseEvent(WE_STATS_CREATE_COL_EXTENT); + + #endif + + //std::ostringstream oss; + //oss << "initColExtent: width-" << width << + //"; loopCount-" << loopCount << + //"; writeSize-" << writeSize; + //std::cout << oss.str() << std::endl; + if (remWriteSize > 0) + { + if ( pFile->write( writeBuf, remWriteSize ) != remWriteSize ) + { + return ERR_FILE_WRITE; + } + } + + for (int j = 0; j < loopCount; j++) + { + if ( pFile->write( writeBuf, writeSize ) != writeSize ) + { + return ERR_FILE_WRITE; + } + } + } + + //@Bug 3219. update the compression header after the extent is expanded. + if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) + { + updateColumnExtent(pFile, nBlocks); + } + + // @bug 2378. Synchronize here to avoid write buffer pile up too much, + // which could cause controllernode to timeout later when it needs to + // save a snapshot. + pFile->flush(); #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_INIT_COL_EXTENT); - if (bExpandExtent) - Stats::startParseEvent(WE_STATS_EXPAND_COL_EXTENT); + Stats::stopParseEvent(WE_STATS_EXPAND_COL_EXTENT); else - Stats::startParseEvent(WE_STATS_CREATE_COL_EXTENT); - + Stats::stopParseEvent(WE_STATS_CREATE_COL_EXTENT); #endif - //std::ostringstream oss; - //oss << "initColExtent: width-" << width << - //"; loopCount-" << loopCount << - //"; writeSize-" << writeSize; - //std::cout << oss.str() << std::endl; - if (remWriteSize > 0) - { - if ( pFile->write( writeBuf, remWriteSize ) != remWriteSize ) - { - return ERR_FILE_WRITE; - } - } - - for (int j = 0; j < loopCount; j++) - { - if ( pFile->write( writeBuf, writeSize ) != writeSize ) - { - return ERR_FILE_WRITE; - } - } } - - //@Bug 3219. update the compression header after the extent is expanded. - if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) - { - updateColumnExtent(pFile, nBlocks); - } - - // @bug 2378. Synchronize here to avoid write buffer pile up too much, - // which could cause controllernode to timeout later when it needs to - // save a snapshot. - pFile->flush(); - -#ifdef PROFILE - - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_EXPAND_COL_EXTENT); - else - Stats::stopParseEvent(WE_STATS_CREATE_COL_EXTENT); - -#endif - // Log the fallocate() call result - std::ostringstream oss; - std::string errnoMsg; - Convertor::mapErrnoToString(savedErrno, errnoMsg); - oss << "FileOp::initColumnExtent(): fallocate(" << currFileSize << - ", " << writeSize << "): errno = " << savedErrno << - ": " << errnoMsg; - logging::Message::Args args; - args.add(oss.str()); - SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_INFO, - logging::M0006); - } return NO_ERROR; @@ -1794,6 +1801,7 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, * blockHdrInit(in) - data used to initialize each block * blockHdrInitSize(in) - number of bytes in blockHdrInit * bExpandExtent (in) - Expand existing extent, or initialize a new one + * bOptExtension(in) - use fallocate() to extend the file if it is possible. * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1804,10 +1812,13 @@ int FileOp::initDctnryExtent( int nBlocks, unsigned char* blockHdrInit, int blockHdrInitSize, - bool bExpandExtent ) + bool bExpandExtent, + bool bOptExtension ) { + off64_t currFileSize = pFile->size(); // @bug5769 Don't initialize extents or truncate db files on HDFS - if (idbdatafile::IDBPolicy::useHdfs()) + // MCOL-498 We don't need sequential segment files if a PM uses SSD either. + if (idbdatafile::IDBPolicy::useHdfs() || !idbdatafile::IDBPolicy::PreallocSpace()) { if (m_compressionType) updateDctnryExtent(pFile, nBlocks); @@ -1841,86 +1852,107 @@ int FileOp::initDctnryExtent( // Allocate a buffer, initialize it, and use it to create the extent idbassert(dbRoot > 0); -#ifdef PROFILE +#ifdef PROFILE if (bExpandExtent) Stats::startParseEvent(WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT); else Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); - #endif - boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); -#ifdef PROFILE + boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); + +#ifdef PROFILE if (bExpandExtent) Stats::stopParseEvent(WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT); else Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); - - Stats::startParseEvent(WE_STATS_INIT_DCT_EXTENT); #endif - - // Allocate buffer, and store in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of writeBuf. + int savedErrno = 0; + // MCOL-498 Try to preallocate the space, fallback to write if fallocate + // has failed + if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize)) { - unsigned char* writeBuf = new unsigned char[writeSize]; - boost::scoped_array writeBufPtr( writeBuf ); - - memset(writeBuf, 0, writeSize); - - for (int i = 0; i < nBlocks; i++) + // Log the failed fallocate() call result + if ( bOptExtension ) { - memcpy( writeBuf + (i * BYTE_PER_BLOCK), - blockHdrInit, - blockHdrInitSize ); + std::ostringstream oss; + std::string errnoMsg; + Convertor::mapErrnoToString(savedErrno, errnoMsg); + oss << "FileOp::initColumnExtent(): fallocate(" << currFileSize << + ", " << writeSize << "): errno = " << savedErrno << + ": " << errnoMsg; + logging::Message::Args args; + args.add(oss.str()); + SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_INFO, + logging::M0006); } + // Allocate buffer, and store in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of writeBuf. + { +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_INIT_DCT_EXTENT); +#endif + + unsigned char* writeBuf = new unsigned char[writeSize]; + boost::scoped_array writeBufPtr( writeBuf ); + + memset(writeBuf, 0, writeSize); + + for (int i = 0; i < nBlocks; i++) + { + memcpy( writeBuf + (i * BYTE_PER_BLOCK), + blockHdrInit, + blockHdrInitSize ); + } + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_INIT_DCT_EXTENT); + + if (bExpandExtent) + Stats::startParseEvent(WE_STATS_EXPAND_DCT_EXTENT); + else + Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT); +#endif + + //std::ostringstream oss; + //oss << "initDctnryExtent: width-8(assumed)" << + //"; loopCount-" << loopCount << + //"; writeSize-" << writeSize; + //std::cout << oss.str() << std::endl; + if (remWriteSize > 0) + { + if (pFile->write( writeBuf, remWriteSize ) != remWriteSize) + { + return ERR_FILE_WRITE; + } + } + + for (int j = 0; j < loopCount; j++) + { + if (pFile->write( writeBuf, writeSize ) != writeSize) + { + return ERR_FILE_WRITE; + } + } + } + + if (m_compressionType) + updateDctnryExtent(pFile, nBlocks); + + // Synchronize to avoid write buffer pile up too much, which could cause + // controllernode to timeout later when it needs to save a snapshot. + pFile->flush(); #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_INIT_DCT_EXTENT); if (bExpandExtent) - Stats::startParseEvent(WE_STATS_EXPAND_DCT_EXTENT); + Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); else - Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT); - + Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT); #endif - - //std::ostringstream oss; - //oss << "initDctnryExtent: width-8(assumed)" << - //"; loopCount-" << loopCount << - //"; writeSize-" << writeSize; - //std::cout << oss.str() << std::endl; - if (remWriteSize > 0) - { - if (pFile->write( writeBuf, remWriteSize ) != remWriteSize) - { - return ERR_FILE_WRITE; - } - } - - for (int j = 0; j < loopCount; j++) - { - if (pFile->write( writeBuf, writeSize ) != writeSize) - { - return ERR_FILE_WRITE; - } - } } - if (m_compressionType) - updateDctnryExtent(pFile, nBlocks); - - // Synchronize to avoid write buffer pile up too much, which could cause - // controllernode to timeout later when it needs to save a snapshot. - pFile->flush(); -#ifdef PROFILE - - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); - else - Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT); - -#endif } return NO_ERROR; diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 8a81e9815..67d926910 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -324,13 +324,15 @@ public: * @param blockHdrInit(in) - data used to initialize each block header * @param blockHdrInitSize(in) - number of bytes in blockHdrInit * @param bExpandExtent (in) - Expand existing extent, or initialize new one + * @param bOptExtension (in) - use fallocate() to extend the file if it is possible. */ EXPORT int initDctnryExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, unsigned char* blockHdrInit, int blockHdrInitSize, - bool bExpandExtent ); + bool bExpandExtent, + bool bOptExtension = false ); /** * @brief Check whether it is an directory @@ -500,6 +502,7 @@ private: // bNewFile (in) - Adding extent to new file // bExpandExtent (in) - Expand existing extent, or initialize new one // bAbbrevExtent (in) - If adding new extent, is it abbreviated + // bOptExtension(in) - use fallocate() to extend the file if it is possible. int initColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, From 7cf0d55dd0d43c24a0a405c050b7fbbbe77863fe Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 15 Mar 2018 15:50:24 +0300 Subject: [PATCH 28/40] MCOL-498: Fill up the block with NULLs when CS touches for the first time it with INSERT..VALUES. --- writeengine/shared/we_fileop.cpp | 2 +- writeengine/wrapper/we_colop.cpp | 26 +++++++++++++++++++++++++- writeengine/wrapper/we_colop.h | 4 ++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 0b94d0d81..f77817962 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -1879,7 +1879,7 @@ int FileOp::initDctnryExtent( std::ostringstream oss; std::string errnoMsg; Convertor::mapErrnoToString(savedErrno, errnoMsg); - oss << "FileOp::initColumnExtent(): fallocate(" << currFileSize << + oss << "FileOp::initDctnryExtent(): fallocate(" << currFileSize << ", " << writeSize << "): errno = " << savedErrno << ": " << errnoMsg; logging::Message::Args args; diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 4f0e6a0d2..56c1b923a 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1519,6 +1519,7 @@ void ColumnOp::setColParam(Column& column, * rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted * valArray - the array of row values * oldValArray - the array of old value + * bDelete - yet * RETURN: * NO_ERROR if success, other number otherwise ***********************************************************/ @@ -1533,6 +1534,8 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, char charTmpBuf[8]; uint64_t emptyVal; int rc = NO_ERROR; + bool fillUpWEmptyVals = false; + bool fistRowInBlock = false; while (!bExit) { @@ -1551,8 +1554,18 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, return rc; bDataDirty = false; + // MCOL-498 We got into the next block, so the row is first in that block + // - fill the block up with NULLs. + if ( curDataFbo != -1 && !bDelete ) + fillUpWEmptyVals = true; } + // MCOL-498 CS hasn't touched any block yet, + // but the row fill be the first in the block. + fistRowInBlock = ( !(curRowId % (BYTE_PER_BLOCK / curCol.colWidth)) ) ? true : false; + if( fistRowInBlock && !bDelete ) + fillUpWEmptyVals = true; + curDataFbo = dataFbo; rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); @@ -1676,8 +1689,19 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, // take care of the cleanup if (bDataDirty && curDataFbo >= 0) + { + if ( fillUpWEmptyVals ) + { + emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + int writeSize = BYTE_PER_BLOCK - ( dataBio + curCol.colWidth ); + // MCOL-498 Add the check though this is unlikely at the moment of writing. + if ( writeSize ) + setEmptyBuf( dataBuf + dataBio + curCol.colWidth, writeSize, emptyVal, curCol.colWidth ); + fillUpWEmptyVals = false; + fistRowInBlock = false; + } rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); - + } return rc; } diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index 3637ec772..d1947d752 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -267,7 +267,7 @@ public: bool bDelete = false); /** - * @brief Write row(s) for update and delete @Bug 1886,2870 + * @brief Write row(s) for delete @Bug 1886,2870 */ EXPORT virtual int writeRows(Column& curCol, uint64_t totalRow, @@ -277,7 +277,7 @@ public: bool bDelete = false); /** - * @brief Write row(s) for update and delete @Bug 1886,2870 + * @brief Write row(s) for update @Bug 1886,2870 */ EXPORT virtual int writeRowsValues(Column& curCol, uint64_t totalRow, From 6db8b1f4322f83e5b8c059fa10ba9e204edacd93 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 20 Mar 2018 17:39:26 +0300 Subject: [PATCH 29/40] MCOL-498: Fill up the last used block with empty values, whilst doing bulk insertion with uncompressed files. --- writeengine/bulk/we_bulkload.h | 2 +- writeengine/bulk/we_colbuf.cpp | 25 +++++++++++++++++++++-- writeengine/bulk/we_colbuf.h | 5 ++++- writeengine/bulk/we_colbufcompressed.cpp | 3 ++- writeengine/bulk/we_colbufcompressed.h | 5 ++++- writeengine/bulk/we_colbufmgr.cpp | 26 +++++++++++++++++------- writeengine/bulk/we_colbufmgr.h | 5 ++++- 7 files changed, 57 insertions(+), 14 deletions(-) diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index c4aecdbc2..4c1adab09 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -64,7 +64,7 @@ class BulkLoad : public FileOp public: /** - * @brief BulkLoad onstructor + * @brief BulkLoad constructor */ EXPORT BulkLoad(); diff --git a/writeengine/bulk/we_colbuf.cpp b/writeengine/bulk/we_colbuf.cpp index 1d164858c..4ade584ed 100644 --- a/writeengine/bulk/we_colbuf.cpp +++ b/writeengine/bulk/we_colbuf.cpp @@ -29,6 +29,7 @@ #include "we_columninfo.h" #include "we_log.h" #include "we_stats.h" +#include "we_blockop.h" #include #include "IDBDataFile.h" using namespace idbdatafile; @@ -101,15 +102,33 @@ void ColumnBuffer::resizeAndCopy(int newSize, int startOffset, int endOffset) //------------------------------------------------------------------------------ // Write data stored up in the output buffer to the segment column file. //------------------------------------------------------------------------------ -int ColumnBuffer::writeToFile(int startOffset, int writeSize) +int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWNulls) { if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; + unsigned char *newBuf = NULL; + + if ( fillUpWNulls ) + { + BlockOp blockOp; + //TO DO Use scoped_ptr here + newBuf = new unsigned char[BYTE_PER_BLOCK]; + uint64_t EmptyValue = blockOp.getEmptyRowValue(fColInfo->column.dataType, + fColInfo->column.width); + ::memcpy(static_cast(newBuf), + static_cast(fBuffer + startOffset), writeSize); + blockOp.setEmptyBuf(newBuf + writeSize, BYTE_PER_BLOCK - writeSize, + EmptyValue, fColInfo->column.width); + } #ifdef PROFILE Stats::startParseEvent(WE_STATS_WRITE_COL); #endif - size_t nitems = fFile->write(fBuffer + startOffset, writeSize) / writeSize; + size_t nitems; + if ( fillUpWNulls ) + nitems = fFile->write(newBuf, BYTE_PER_BLOCK) / BYTE_PER_BLOCK; + else + nitems = fFile->write(fBuffer + startOffset, writeSize) / writeSize; if (nitems != 1) return ERR_FILE_WRITE; @@ -118,6 +137,8 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize) Stats::stopParseEvent(WE_STATS_WRITE_COL); #endif + //TO DO Use scoped_ptr here + delete newBuf; return NO_ERROR; } diff --git a/writeengine/bulk/we_colbuf.h b/writeengine/bulk/we_colbuf.h index d5f3af570..baf11b6bb 100644 --- a/writeengine/bulk/we_colbuf.h +++ b/writeengine/bulk/we_colbuf.h @@ -107,8 +107,11 @@ public: * * @param startOffset The buffer offset from where the write should begin * @param writeSize The number of bytes to be written to the file + * @param fillUpWNulls The flag to fill the buffer with NULLs up to + * the block boundary. */ - virtual int writeToFile(int startOffset, int writeSize); + virtual int writeToFile(int startOffset, int writeSize, + bool fillUpWNulls = false); protected: diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 9abc0038a..019fe6e46 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -167,7 +167,8 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf( // file, and instead buffer up the data to be compressed in 4M chunks before // writing it out. //------------------------------------------------------------------------------ -int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) +int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, + bool fillUpWNulls) { if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; diff --git a/writeengine/bulk/we_colbufcompressed.h b/writeengine/bulk/we_colbufcompressed.h index 6ea70339c..335a8f2ba 100644 --- a/writeengine/bulk/we_colbufcompressed.h +++ b/writeengine/bulk/we_colbufcompressed.h @@ -83,8 +83,11 @@ public: * * @param startOffset The buffer offset from where the write should begin * @param writeSize The number of bytes to be written to the file + * @param fillUpWNulls The flag to fill the buffer with NULLs up to + * the block boundary. */ - virtual int writeToFile(int startOffset, int writeSize); + virtual int writeToFile(int startOffset, int writeSize, + bool fillUpWNulls = false); private: diff --git a/writeengine/bulk/we_colbufmgr.cpp b/writeengine/bulk/we_colbufmgr.cpp index 123847260..14ba19402 100644 --- a/writeengine/bulk/we_colbufmgr.cpp +++ b/writeengine/bulk/we_colbufmgr.cpp @@ -529,7 +529,7 @@ int ColumnBufferManager::writeToFile(int endOffset) // internal buffer, or if an abbreviated extent is expanded. //------------------------------------------------------------------------------ int ColumnBufferManager::writeToFileExtentCheck( - uint32_t startOffset, uint32_t writeSize) + uint32_t startOffset, uint32_t writeSize, bool fillUpWNulls) { if (fLog->isDebug( DEBUG_3 )) @@ -571,7 +571,7 @@ int ColumnBufferManager::writeToFileExtentCheck( if (availableFileSize >= writeSize) { - int rc = fCBuf->writeToFile(startOffset, writeSize); + int rc = fCBuf->writeToFile(startOffset, writeSize, fillUpWNulls); if (rc != NO_ERROR) { @@ -583,6 +583,10 @@ int ColumnBufferManager::writeToFileExtentCheck( return rc; } + // MCOL-498 Fill it up to the block size boundary. + if ( fillUpWNulls ) + writeSize = BLOCK_SIZE; + fColInfo->updateBytesWrittenCounts( writeSize ); } else @@ -624,7 +628,7 @@ int ColumnBufferManager::writeToFileExtentCheck( } int writeSize2 = writeSize - writeSize1; - fCBuf->writeToFile(startOffset + writeSize1, writeSize2); + rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2, fillUpWNulls); if (rc != NO_ERROR) { @@ -636,6 +640,10 @@ int ColumnBufferManager::writeToFileExtentCheck( return rc; } + // MCOL-498 Fill it up to the block size boundary. + if ( fillUpWNulls ) + writeSize2 = BLOCK_SIZE; + fColInfo->updateBytesWrittenCounts( writeSize2 ); } @@ -668,17 +676,21 @@ int ColumnBufferManager::flush( ) int bufferSize = fCBuf->getSize(); + // MCOL-498 There are less the BLOCK_SIZE bytes in the buffer left, so // Account for circular buffer by making 2 calls to write the data, // if we are wrapping around at the end of the buffer. if (fBufFreeOffset < fBufWriteOffset) { - RETURN_ON_ERROR( writeToFileExtentCheck( - fBufWriteOffset, bufferSize - fBufWriteOffset) ); + // The check could be redundant. + bool fillUpWEmpty = ( static_cast(bufferSize - fBufWriteOffset) >= BLOCK_SIZE ) + ? false : true; + RETURN_ON_ERROR( writeToFileExtentCheck( fBufWriteOffset, + bufferSize - fBufWriteOffset, fillUpWEmpty) ); fBufWriteOffset = 0; } - + // fill the buffer up with NULLs. RETURN_ON_ERROR( writeToFileExtentCheck( - fBufWriteOffset, fBufFreeOffset - fBufWriteOffset) ); + fBufWriteOffset, fBufFreeOffset - fBufWriteOffset, true) ); fBufWriteOffset = fBufFreeOffset; return NO_ERROR; diff --git a/writeengine/bulk/we_colbufmgr.h b/writeengine/bulk/we_colbufmgr.h index 8ec86fabe..35e837ec1 100644 --- a/writeengine/bulk/we_colbufmgr.h +++ b/writeengine/bulk/we_colbufmgr.h @@ -193,9 +193,12 @@ protected: * write out the buffer. * @param startOffset The buffer offset where the write should begin * @param writeSize The number of bytes to be written to the file + * @param fillUpWNulls The flag to fill the buffer with NULLs up to + * the block boundary. * @return success or fail status */ - virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize); + virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize, + bool fillUpWNulls = false); //------------------------------------------------------------------------- // Protected Data Members From 8037af5161244320d418127962c7451fb5acf198 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sun, 25 Mar 2018 22:03:08 +0300 Subject: [PATCH 30/40] MCOL-498 Fill up next block with empty values if insert values up to the block boundary. --- writeengine/server/we_dmlcommandproc.cpp | 4 +- writeengine/shared/.we_fileop.h.swo | Bin 0 -> 16384 bytes writeengine/shared/we_fileop.cpp | 1 - writeengine/shared/we_fileop.h | 2 + writeengine/wrapper/we_colop.cpp | 57 +++++++++++++++++++++-- 5 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 writeengine/shared/.we_fileop.h.swo diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 5da10433e..a6c5b025c 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -102,13 +102,13 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: bs >> tmp32; uint32_t dbroot = tmp32; - //cout << "processSingleInsert received bytestream length " << bs.length() << endl; + cout << "processSingleInsert received bytestream length " << bs.length() << endl; messageqcpp::ByteStream::byte packageType; bs >> packageType; insertPkg.read( bs); uint32_t sessionId = insertPkg.get_SessionID(); - //cout << " processSingleInsert for session " << sessionId << endl; + cout << " processSingleInsert for session " << sessionId << endl; DMLTable* tablePtr = insertPkg.get_Table(); RowList rows = tablePtr->get_RowList(); diff --git a/writeengine/shared/.we_fileop.h.swo b/writeengine/shared/.we_fileop.h.swo new file mode 100644 index 0000000000000000000000000000000000000000..46e47f43e9e4b891f218888636cb9d5f0e6c00e1 GIT binary patch literal 16384 zcmeI3ON<;x8OJMzVEiHo2tp_lg>9_e)$Go`Vw>5;%IwUnJ+Lp$%Bo|kb;QBBM6CbN|XbFa7ZK$vGd^nRo6Us zXBJQnNbZ(??%AsDuj>2P*VSELw@afd*?E3?WRzk1IAbfHefHuHPZ?}tin06ZuI>3$ z(SNHL)j%4<*!4f|a%eEVX3Mzc`?9`0Ts3^#uu8+G=hUliAU$6U2g2VFe!nAxyJov0 zzUkXi=-)xb@P!rM6veXb2(LC$>ED;AHd5fGQlKu&!>5ldPmGVLOa0i<&+|`yV)>?$ zAaXWRAW|SwAW|SwAW|SwAW|Sw;AU4q*7mapAi_Q^#1*~2rECAb-oC8MztL6yp01zM zit3OP{Y11j&_j(kphtd zkphtdkphtdkphtdkphtdkphtdH>CoG#n`Rz{0p>R#{2&(T7|#f#@H+1d*C9t0~`j2 zz>6Pa>_LzLw}VfC-yLD>5s(9?!6|U#Fk{!jKfxQ|&)^z(5_}h20Uo#)q(KTyfX{&= z;Km`w{sI01{s^vt7r^u22jJVF4vJtFWWW*dw}Xtm23`g)f!~6w;2H2JxB?yk_kuYv z0%G7exC871*AFoE8}K5y27Uw{16RP;!4|j&EQ3#j+rbg=#zz@@9XtwF!3tOaw}JiO zmHmu84(Io&rAx4}%9m8l=D!cngE`cW@Ov1AYoja5p#%4uOMUA9(u1jQs?d z;8t)QNs+g~o8af*DImc#cn{pm>-8YU8vG@w-aCEy|5usv^vF zzF85nB79C?U2w5w2QuKE&s|UQ(l)OLw!0SA*;b6FXL6n=V~otjhlZH?kvYG#m@Dv- z=Q&iaw$RW3H$8k4!cBbJF{G?47*)Xs!{RBX4jqc?{qFrTcIlZqq1nE&(&wML?aHx} zMacuPRu!)7YQN8!R7Gb`jN|N@;Y(ZEo*TNay+&=zfF&Fu#VnK@;Ir8@dJfH+&0D3G zczlP~X>~3)cu*6jUA9eb8PX^jfoS%dhG>LQb4}lszFNpuKl=O%2PBYIBC#P%e6BM+ zlbcT@mKx4w-LiM)M9bNL4>B^>+ui1=L1VbvtuJQNFb5%mP_~|qTO~En0b;}VMNsox zi$YI&4MBtzMuKf|!?jvHRNYg8?9ovgmtFg)M_Bi>D?EA)6fPRxyU$ds2eQ@gR!<>l zA4NL?=>MfV|I3x!jA@G8PVY&9ej8miCns^)iE2%5uNV&h5+99my4*HxOIGmb7^L~n zLSJkNv*s9XB9Ss2M3&5N2U1j1xWF854W*uw#_z&*O(6W5zjeg+>n7^DukW5S#vXRa zw-elO9M8<#UlqM)8PgX=_bkJd@g1?6Qiozvp#gw!(2ag?HDWSAL+WZ$^|qzVG$$7l zyL$^8PuqUuVh`tb%l3Cp?Yqqx{m+l_s%Hs|I_>s~{HAkgTuo)$o!!^XZ+Mt78k^~O zfu5kV=@}Yyy4Z$zjdpuBNGEGCcW=tZH<1X{%Vce_HZa)h7MB)*^y@VJ?rKMD)g0Tj zCD(S;t=0096dMS9oGN;nPSG8=%XY0^g1rR0pP7yk1llZ5@UViEIX%j}pX%J@7~&yXl}0?}2-i$zPA4o$HGmg7J$&zFO+ zTjt@()1u*)vI|ADXsd%nPT~w}ima3?_t4xEW?S{7g*VnV39zk(V}YXWe!K&*jI}lk0qvkB*Ox zjSi1ZjE=?lay~htA5Uhzx{Jpzdf4Mx*BpuQJIDB}Z@BA@?ee_zg^)2^MrBN>vySKa zF+Ky~^s^yuDtK~Hv#qSy0k8Sqnr~F6IHD8J)iJKVd!|PldQ6ywF`oCzauXjw({ymb ztMj1Z)g6mhj19ptxe0p%Q?`MdQ*E1jWp&=!g=KzLxWdPTv!pI?K4+W4#kE3KdROx@ zp6{#pP#af;G;AkO&y@3-TVm7W0`A&hzLyoS+6vb;(z=g>#F`1)f~Ez*m#G9PC|~Qc8HJ6&u1;UAL>4-{25D zhNhBSA`68!u{dH3_ut=h_F&$R1MA~x@c5$doLmlX$ zp5ogWXwz^pz%5UyEj-@F>9hSfW@8MvZ?9D(A4m;SSWhAzF*(}lnJ_XLt^Y@{E?vUf zmDc}__vf!-o&Rg_BzOWm1h#+$Ob`ceVcq{G_!(FS_kl^k!24ML{{lP@o&#k-xq({% z`d(H1EXDG(_TDG(_TDG({JmjdA$lw)nt zwe~nhkAUUIqLf|2Ay}@`gJ5$7jO;r-$Krub*0E-;79}2UgRDi%WZT8kLGZ$si^Ooa zu1_S^ge-)I4CthImkH!9#_3>MN)I4&gM2bAf9OyAGO#d?Aq%PU%B_rOteG)Jmf(KX zBkEPJt(m~;%8#>NCp60G#NATkjr!g8&BhwpaIFjFn%Y!p`P|H-cfLf3{Wn5U!elR< z81=TLli;W}+mP?d3Z42>sU%QU#j`AI5Ae(G6 zrE+(Tu;{i$?I7n9j}L#K8>JPp;l*06yCx+8f)BnCb}5XiFsdnFIOdB|y<8UlRQRX< z7u^YqX`PETGbM2YAvaJsFsRnNoKh@!VWg5bi9{6(=B*ABWV*VJ5uK;1uwj@*UP2;q z%6&@SVeBpSdW)9YDkkXlOPDnB@Y*owXiB}}K|*L6gAnUzc<2loORFgUb*>WB&%GBhE|! literal 0 HcmV?d00001 diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index f77817962..e08bea3fb 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -63,7 +63,6 @@ namespace WriteEngine /*static*/ boost::mutex FileOp::m_createDbRootMutexes; /*static*/ boost::mutex FileOp::m_mkdirMutex; /*static*/ std::map FileOp::m_DbRootAddExtentMutexes; -const int MAX_NBLOCKS = 8192; // max number of blocks written to an extent // in 1 call to fwrite(), during initialization //StopWatch timer; diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 67d926910..175b3fb0d 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -50,6 +50,8 @@ #define EXPORT #endif +#define MAX_NBLOCKS 8192 + #include "brmtypes.h" /** Namespace WriteEngine */ diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 56c1b923a..067453b87 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -35,6 +35,7 @@ using namespace std; #include "idbcompress.h" #include "writeengine.h" #include "cacheutils.h" +#include "we_fileop.h" using namespace execplan; @@ -471,6 +472,13 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, if ( rc != NO_ERROR) return rc; + // MCOL-498 Fill up the first block with empty values. + { + uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, column.colWidth); + } + + for (j = 0; j < totalRowPerBlock; j++) { if (isEmptyRow(buf, j, column)) @@ -1536,12 +1544,14 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, int rc = NO_ERROR; bool fillUpWEmptyVals = false; bool fistRowInBlock = false; + bool lastRowInBlock = false; + uint16_t rowsInBlock = BYTE_PER_BLOCK / curCol.colWidth; while (!bExit) { curRowId = rowIdArray[i]; - calculateRowId(curRowId, BYTE_PER_BLOCK / curCol.colWidth, curCol.colWidth, dataFbo, dataBio); + calculateRowId(curRowId, rowsInBlock, curCol.colWidth, dataFbo, dataBio); // load another data block if necessary if (curDataFbo != dataFbo) @@ -1562,7 +1572,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, // MCOL-498 CS hasn't touched any block yet, // but the row fill be the first in the block. - fistRowInBlock = ( !(curRowId % (BYTE_PER_BLOCK / curCol.colWidth)) ) ? true : false; + fistRowInBlock = ( !(curRowId % (rowsInBlock)) ) ? true : false; if( fistRowInBlock && !bDelete ) fillUpWEmptyVals = true; @@ -1696,11 +1706,48 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, int writeSize = BYTE_PER_BLOCK - ( dataBio + curCol.colWidth ); // MCOL-498 Add the check though this is unlikely at the moment of writing. if ( writeSize ) - setEmptyBuf( dataBuf + dataBio + curCol.colWidth, writeSize, emptyVal, curCol.colWidth ); - fillUpWEmptyVals = false; - fistRowInBlock = false; + setEmptyBuf( dataBuf + dataBio + curCol.colWidth, writeSize, + emptyVal, curCol.colWidth ); + //fillUpWEmptyVals = false; + //fistRowInBlock = false; } + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + + if ( rc != NO_ERROR) + return rc; + + // MCOL-498 If it was the last row in a block fill the next block with + // empty vals, otherwise next ColumnOp::allocRowId() + // will fail on the next block. + lastRowInBlock = ( rowsInBlock - ( curRowId % rowsInBlock ) == 1 ) ? true : false; + if ( lastRowInBlock ) + { + if( !fillUpWEmptyVals ) + emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + // MCOL-498 Skip if this is the last block in an extent. + if ( curDataFbo != MAX_NBLOCKS - 1) + { + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + if ( rc != NO_ERROR) + return rc; + + curDataFbo += 1; + rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + if ( rc != NO_ERROR) + return rc; + + unsigned char zeroSubBlock[BYTE_PER_SUBBLOCK]; + std::memset(zeroSubBlock, 0, BYTE_PER_SUBBLOCK); + // The first subblock is made of 0 - fill the block with empty vals. + if ( !std::memcmp(dataBuf, zeroSubBlock, BYTE_PER_SUBBLOCK) ) + { + setEmptyBuf(dataBuf, BYTE_PER_BLOCK, emptyVal, curCol.colWidth); + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + } + } + } + } return rc; } From 46a46aa6b1b977392370b789608b1a2f331f5049 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 20 Aug 2018 16:01:30 +0300 Subject: [PATCH 31/40] MCOL-498 Support for non dict compressed columns. --- writeengine/bulk/we_colbufcompressed.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 019fe6e46..6aa172158 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -173,6 +173,10 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; + int fillUpWNullsWriteSize = 0; + if (fillUpWNulls) + fillUpWNullsWriteSize = BYTE_PER_BLOCK - writeSize % BYTE_PER_BLOCK; + // If we are starting a new file, we need to reinit the buffer and // find out what our file offset should be set to. if (!fToBeCompressedCapacity) @@ -220,7 +224,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, // Expand the compression buffer size if working with an abbrev extent, and // the bytes we are about to add will overflow the abbreviated extent. if ((fToBeCompressedCapacity < IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) && - ((fNumBytes + writeSize) > fToBeCompressedCapacity) ) + ((fNumBytes + writeSize + fillUpWNullsWriteSize) > fToBeCompressedCapacity) ) { std::ostringstream oss; oss << "Expanding abbrev to-be-compressed buffer for: OID-" << @@ -232,7 +236,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN; } - if ((fNumBytes + writeSize) <= fToBeCompressedCapacity) + if ((fNumBytes + writeSize + fillUpWNullsWriteSize) <= fToBeCompressedCapacity) { if (fLog->isDebug( DEBUG_2 )) { @@ -243,12 +247,14 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; addBytes-" << writeSize << + "; extraBytes-" << fillUpWNullsWriteSize << "; totBytes-" << (fNumBytes + writeSize); fLog->logMsg( oss.str(), MSGLVL_INFO2 ); } memcpy(bufOffset, (fBuffer + startOffset), writeSize); fNumBytes += writeSize; + fNumBytes += fillUpWNullsWriteSize; } else // Not enough room to add all the data to the to-be-compressed buffer { @@ -339,6 +345,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); fNumBytes += writeSizeOut; + fNumBytes += fillUpWNullsWriteSize; } startOffsetX += writeSizeOut; From 29becc2971c39c81b944eedbf68ead233dc8ac56 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 14 Dec 2018 14:42:06 +0300 Subject: [PATCH 32/40] MCOL-498 Passed test100. --- writeengine/shared/.we_fileop.h.swo | Bin 16384 -> 0 bytes writeengine/shared/we_fileop.cpp | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 writeengine/shared/.we_fileop.h.swo diff --git a/writeengine/shared/.we_fileop.h.swo b/writeengine/shared/.we_fileop.h.swo deleted file mode 100644 index 46e47f43e9e4b891f218888636cb9d5f0e6c00e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI3ON<;x8OJMzVEiHo2tp_lg>9_e)$Go`Vw>5;%IwUnJ+Lp$%Bo|kb;QBBM6CbN|XbFa7ZK$vGd^nRo6Us zXBJQnNbZ(??%AsDuj>2P*VSELw@afd*?E3?WRzk1IAbfHefHuHPZ?}tin06ZuI>3$ z(SNHL)j%4<*!4f|a%eEVX3Mzc`?9`0Ts3^#uu8+G=hUliAU$6U2g2VFe!nAxyJov0 zzUkXi=-)xb@P!rM6veXb2(LC$>ED;AHd5fGQlKu&!>5ldPmGVLOa0i<&+|`yV)>?$ zAaXWRAW|SwAW|SwAW|SwAW|Sw;AU4q*7mapAi_Q^#1*~2rECAb-oC8MztL6yp01zM zit3OP{Y11j&_j(kphtd zkphtdkphtdkphtdkphtdkphtdH>CoG#n`Rz{0p>R#{2&(T7|#f#@H+1d*C9t0~`j2 zz>6Pa>_LzLw}VfC-yLD>5s(9?!6|U#Fk{!jKfxQ|&)^z(5_}h20Uo#)q(KTyfX{&= z;Km`w{sI01{s^vt7r^u22jJVF4vJtFWWW*dw}Xtm23`g)f!~6w;2H2JxB?yk_kuYv z0%G7exC871*AFoE8}K5y27Uw{16RP;!4|j&EQ3#j+rbg=#zz@@9XtwF!3tOaw}JiO zmHmu84(Io&rAx4}%9m8l=D!cngE`cW@Ov1AYoja5p#%4uOMUA9(u1jQs?d z;8t)QNs+g~o8af*DImc#cn{pm>-8YU8vG@w-aCEy|5usv^vF zzF85nB79C?U2w5w2QuKE&s|UQ(l)OLw!0SA*;b6FXL6n=V~otjhlZH?kvYG#m@Dv- z=Q&iaw$RW3H$8k4!cBbJF{G?47*)Xs!{RBX4jqc?{qFrTcIlZqq1nE&(&wML?aHx} zMacuPRu!)7YQN8!R7Gb`jN|N@;Y(ZEo*TNay+&=zfF&Fu#VnK@;Ir8@dJfH+&0D3G zczlP~X>~3)cu*6jUA9eb8PX^jfoS%dhG>LQb4}lszFNpuKl=O%2PBYIBC#P%e6BM+ zlbcT@mKx4w-LiM)M9bNL4>B^>+ui1=L1VbvtuJQNFb5%mP_~|qTO~En0b;}VMNsox zi$YI&4MBtzMuKf|!?jvHRNYg8?9ovgmtFg)M_Bi>D?EA)6fPRxyU$ds2eQ@gR!<>l zA4NL?=>MfV|I3x!jA@G8PVY&9ej8miCns^)iE2%5uNV&h5+99my4*HxOIGmb7^L~n zLSJkNv*s9XB9Ss2M3&5N2U1j1xWF854W*uw#_z&*O(6W5zjeg+>n7^DukW5S#vXRa zw-elO9M8<#UlqM)8PgX=_bkJd@g1?6Qiozvp#gw!(2ag?HDWSAL+WZ$^|qzVG$$7l zyL$^8PuqUuVh`tb%l3Cp?Yqqx{m+l_s%Hs|I_>s~{HAkgTuo)$o!!^XZ+Mt78k^~O zfu5kV=@}Yyy4Z$zjdpuBNGEGCcW=tZH<1X{%Vce_HZa)h7MB)*^y@VJ?rKMD)g0Tj zCD(S;t=0096dMS9oGN;nPSG8=%XY0^g1rR0pP7yk1llZ5@UViEIX%j}pX%J@7~&yXl}0?}2-i$zPA4o$HGmg7J$&zFO+ zTjt@()1u*)vI|ADXsd%nPT~w}ima3?_t4xEW?S{7g*VnV39zk(V}YXWe!K&*jI}lk0qvkB*Ox zjSi1ZjE=?lay~htA5Uhzx{Jpzdf4Mx*BpuQJIDB}Z@BA@?ee_zg^)2^MrBN>vySKa zF+Ky~^s^yuDtK~Hv#qSy0k8Sqnr~F6IHD8J)iJKVd!|PldQ6ywF`oCzauXjw({ymb ztMj1Z)g6mhj19ptxe0p%Q?`MdQ*E1jWp&=!g=KzLxWdPTv!pI?K4+W4#kE3KdROx@ zp6{#pP#af;G;AkO&y@3-TVm7W0`A&hzLyoS+6vb;(z=g>#F`1)f~Ez*m#G9PC|~Qc8HJ6&u1;UAL>4-{25D zhNhBSA`68!u{dH3_ut=h_F&$R1MA~x@c5$doLmlX$ zp5ogWXwz^pz%5UyEj-@F>9hSfW@8MvZ?9D(A4m;SSWhAzF*(}lnJ_XLt^Y@{E?vUf zmDc}__vf!-o&Rg_BzOWm1h#+$Ob`ceVcq{G_!(FS_kl^k!24ML{{lP@o&#k-xq({% z`d(H1EXDG(_TDG(_TDG({JmjdA$lw)nt zwe~nhkAUUIqLf|2Ay}@`gJ5$7jO;r-$Krub*0E-;79}2UgRDi%WZT8kLGZ$si^Ooa zu1_S^ge-)I4CthImkH!9#_3>MN)I4&gM2bAf9OyAGO#d?Aq%PU%B_rOteG)Jmf(KX zBkEPJt(m~;%8#>NCp60G#NATkjr!g8&BhwpaIFjFn%Y!p`P|H-cfLf3{Wn5U!elR< z81=TLli;W}+mP?d3Z42>sU%QU#j`AI5Ae(G6 zrE+(Tu;{i$?I7n9j}L#K8>JPp;l*06yCx+8f)BnCb}5XiFsdnFIOdB|y<8UlRQRX< z7u^YqX`PETGbM2YAvaJsFsRnNoKh@!VWg5bi9{6(=B*ABWV*VJ5uK;1uwj@*UP2;q z%6&@SVeBpSdW)9YDkkXlOPDnB@Y*owXiB}}K|*L6gAnUzc<2loORFgUb*>WB&%GBhE|! diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index e08bea3fb..e028705f1 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -1104,7 +1104,7 @@ int FileOp::initColumnExtent( int savedErrno = 0; // MCOL-498 Try to preallocate the space, fallback to write if fallocate has failed - if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize)) + if ( !bOptExtension || ( nBlocks < 300 && pFile->fallocate(0, currFileSize, writeSize) )) { savedErrno = errno; // Log the failed fallocate() call result @@ -1870,7 +1870,7 @@ int FileOp::initDctnryExtent( int savedErrno = 0; // MCOL-498 Try to preallocate the space, fallback to write if fallocate // has failed - if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize)) + //if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize)) { // Log the failed fallocate() call result if ( bOptExtension ) From cbdcdb9f10146451d89b38dfd18e29171903b830 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 20 Dec 2018 06:52:05 +0300 Subject: [PATCH 33/40] MCOL-498 Add DBRootX.PreallocSpace setting in the XML. Dict files extents now contain a correct number of blocks available. --- oam/etc/Columnstore.xml | 32 -------- utils/idbdatafile/IDBPolicy.cpp | 94 ++++++++++++++++++------ utils/idbdatafile/IDBPolicy.h | 14 ++-- writeengine/dictionary/we_dctnry.cpp | 4 +- writeengine/server/we_dmlcommandproc.cpp | 4 +- writeengine/shared/we_define.h | 3 +- writeengine/shared/we_fileop.cpp | 65 ++++++++++------ writeengine/wrapper/we_colop.cpp | 8 +- 8 files changed, 131 insertions(+), 93 deletions(-) diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 60bc2d9fc..b5610bf11 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -102,162 +102,130 @@ 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON 0.0.0.0 8620 - ON C diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index a8c194918..6d06b9b30 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -18,14 +18,13 @@ #include #include #include -#include #include #include -#include // to_upper #include #include "configcpp.h" // for Config #include "oamcache.h" +#include "liboamcpp.h" #include "IDBPolicy.h" #include "PosixFileSystem.h" //#include "HdfsFileSystem.h" @@ -50,7 +49,7 @@ int64_t IDBPolicy::s_hdfsRdwrBufferMaxSize = 0; std::string IDBPolicy::s_hdfsRdwrScratch; bool IDBPolicy::s_configed = false; boost::mutex IDBPolicy::s_mutex; -bool IDBPolicy::s_PreallocSpace = true; +std::vector IDBPolicy::s_PreallocSpace; void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& hdfsRdwrScratch, int64_t hdfsRdwrBufferMaxSize ) { @@ -184,10 +183,12 @@ void IDBPolicy::configIDBPolicy() bool idblog = false; string idblogstr = cf->getConfig("SystemConfig", "DataFileLog"); - if ( idblogstr.length() != 0 ) + // Must be faster. + if ( idblogstr.size() == 2 + && ( idblogstr[0] == 'O' || idblogstr[0] == 'o' ) + && ( idblogstr[1] == 'N' || idblogstr[1] == 'n' )) { - boost::to_upper(idblogstr); - idblog = ( idblogstr == "ON" ); + idblog = true; } //-------------------------------------------------------------------------- @@ -227,26 +228,73 @@ void IDBPolicy::configIDBPolicy() string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); string hdfsRdwrScratch = tmpDir + scratch; - // MCOL-498. Set the PMSX.PreallocSpace knob, where X is a PM number, - // to disable file space preallocation. The feature is used in the FileOp code - // and is enabled by default for a backward compatibility. - oam::OamCache* oamcache = oam::OamCache::makeOamCache(); - int PMId = oamcache->getLocalPMId(); - char configSectionPref[] = "PMS"; - char configSection[sizeof(configSectionPref)+oam::MAX_MODULE_ID_SIZE]; - ::memset(configSection, 0, sizeof(configSection)); - sprintf(configSection, "%s%d", configSectionPref, PMId); - string PreallocSpace = cf->getConfig(configSection, "PreallocSpace"); - - if ( PreallocSpace.length() != 0 ) - { - boost::to_upper(PreallocSpace); - s_PreallocSpace = ( PreallocSpace != "OFF" ); - } + // MCOL-498. Use DBRootX.PreallocSpace to disable + // dbroots file space preallocation. + // The feature is used in the FileOp code and enabled by default. + char configSectionPref[] = "DBRoot"; + int confSectionLen = sizeof(configSectionPref)+oam::MAX_MODULE_ID_SIZE; + char configSection[confSectionLen]; IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); - s_configed = true; + + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + int PMId = oamcache->getLocalPMId(); + + oam::OamCache::PMDbrootsMap_t pmDbrootsMap; + pmDbrootsMap.reset(new oam::OamCache::PMDbrootsMap_t::element_type()); + oam::systemStorageInfo_t t; + oam::Oam oamInst; + t = oamInst.getStorageConfig(); + + oam::DeviceDBRootList moduledbrootlist = boost::get<2>(t); + oam::DeviceDBRootList::iterator pt = moduledbrootlist.begin(); + + while ( pt != moduledbrootlist.end() && (*pt).DeviceID != PMId) + { + pt++; + continue; + } + + // CS could return here b/c it initialised this singleton and + // there is no DBRootX sections in XML. + if (pt == moduledbrootlist.end()) + { + return; + } + + oam::DBRootConfigList &dbRootVec = (*pt).dbrootConfigList; + s_PreallocSpace.reserve(dbRootVec.size()>>1); + { + int rc; + oam::DBRootConfigList::iterator dbRootIter = dbRootVec.begin(); + for(; dbRootIter != dbRootVec.end(); dbRootIter++) + { + ::memset(configSection + sizeof(configSectionPref), 0, oam::MAX_MODULE_ID_SIZE); + rc = snprintf(configSection, confSectionLen, "%s%d", configSectionPref, *dbRootIter); + // gcc 8.2 warnings + if ( rc < 0 || rc >= confSectionLen) + { + ostringstream oss; + oss << "IDBPolicy::configIDBPolicy: failed to parse DBRootX section."; + throw runtime_error(oss.str()); + } + string setting = cf->getConfig(configSection, "PreallocSpace"); + + if ( setting.length() != 0 ) + { + if ( setting.size() == 3 + && ( setting[0] == 'O' || setting[0] == 'o' ) + && ( setting[1] == 'F' || setting[1] == 'f' ) + && ( setting[2] == 'F' || setting[2] == 'f' ) + ) + { + // int into uint16_t implicit conversion + s_PreallocSpace.push_back(*dbRootIter); + } + } + } + } } } diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h index 5212f3b11..0e28d5004 100644 --- a/utils/idbdatafile/IDBPolicy.h +++ b/utils/idbdatafile/IDBPolicy.h @@ -19,6 +19,7 @@ #define IDBPOLICY_H_ #include +#include #include #include @@ -81,9 +82,9 @@ public: static bool useHdfs(); /** - * Accessor method that returns whether or not HDFS is enabled + * Checks for disk space preallocation feature status for a dbroot */ - static bool PreallocSpace(); + static bool PreallocSpace(uint16_t dbRoot); /** * Accessor method that returns whether to use HDFS memory buffers @@ -139,7 +140,7 @@ private: static bool isLocalFile( const std::string& path ); static bool s_usehdfs; - static bool s_PreallocSpace; + static std::vector s_PreallocSpace; static bool s_bUseRdwrMemBuffer; static std::string s_hdfsRdwrScratch; static int64_t s_hdfsRdwrBufferMaxSize; @@ -159,10 +160,13 @@ bool IDBPolicy::useHdfs() return s_usehdfs; } +// MCOL-498 Looking for dbRoot in the List set in configIDBPolicy. inline -bool IDBPolicy::PreallocSpace() +bool IDBPolicy::PreallocSpace(uint16_t dbRoot) { - return s_PreallocSpace; + std::vector::iterator dbRootIter = + find(s_PreallocSpace.begin(), s_PreallocSpace.end(), dbRoot); + return dbRootIter != s_PreallocSpace.end(); } inline diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index c5461d4e5..c7cfea1d5 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -259,12 +259,14 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, if ( m_dFile != NULL ) { + bool optimizePrealloc = ( flag ) ? false : true; rc = FileOp::initDctnryExtent( m_dFile, m_dbRoot, totalSize, m_dctnryHeader2, m_totalHdrBytes, - false ); + false, + optimizePrealloc ); if (rc != NO_ERROR) { diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index a6c5b025c..5da10433e 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -102,13 +102,13 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: bs >> tmp32; uint32_t dbroot = tmp32; - cout << "processSingleInsert received bytestream length " << bs.length() << endl; + //cout << "processSingleInsert received bytestream length " << bs.length() << endl; messageqcpp::ByteStream::byte packageType; bs >> packageType; insertPkg.read( bs); uint32_t sessionId = insertPkg.get_SessionID(); - cout << " processSingleInsert for session " << sessionId << endl; + //cout << " processSingleInsert for session " << sessionId << endl; DMLTable* tablePtr = insertPkg.get_Table(); RowList rows = tablePtr->get_RowList(); diff --git a/writeengine/shared/we_define.h b/writeengine/shared/we_define.h index c26b926fd..8c3e85fe9 100644 --- a/writeengine/shared/we_define.h +++ b/writeengine/shared/we_define.h @@ -45,7 +45,8 @@ const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file const int BYTE_PER_BLOCK = 8192; // Num bytes per data block const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block -const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; +const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; // Used for initial number of blocks calculation +const int MAX_INITIAL_EXTENT_BLOCKS_TO_DISK = 256; // Number of blocks in abbrev extent for 8byte col. // Num rows reserved to disk for 'initial' extent const int FILE_NAME_SIZE = 200; // Max size of file name const long long MAX_ALLOW_ERROR_COUNT = 100000; //Max allowable error count diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index e028705f1..d079700dd 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -1046,7 +1046,7 @@ int FileOp::initColumnExtent( // @bug5769 Don't initialize extents or truncate db files on HDFS // MCOL-498 We don't need sequential segment files if a PM uses SSD either. - if (idbdatafile::IDBPolicy::useHdfs() || !idbdatafile::IDBPolicy::PreallocSpace()) + if (idbdatafile::IDBPolicy::useHdfs()) { //@Bug 3219. update the compression header after the extent is expanded. if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) @@ -1101,10 +1101,19 @@ int FileOp::initColumnExtent( else Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); #endif - + // MCOL-498 Skip the huge preallocations if the option is set + // for the dbroot + if ( bOptExtension ) + { + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) + ? bOptExtension : false; + } int savedErrno = 0; - // MCOL-498 Try to preallocate the space, fallback to write if fallocate has failed - if ( !bOptExtension || ( nBlocks < 300 && pFile->fallocate(0, currFileSize, writeSize) )) + // MCOL-498 fallocate the abbreviated extent, + // fallback to sequential write if fallocate failed + if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK + && pFile->fallocate(0, currFileSize, writeSize) ) + ) { savedErrno = errno; // Log the failed fallocate() call result @@ -1817,7 +1826,7 @@ int FileOp::initDctnryExtent( off64_t currFileSize = pFile->size(); // @bug5769 Don't initialize extents or truncate db files on HDFS // MCOL-498 We don't need sequential segment files if a PM uses SSD either. - if (idbdatafile::IDBPolicy::useHdfs() || !idbdatafile::IDBPolicy::PreallocSpace()) + if (idbdatafile::IDBPolicy::useHdfs()) { if (m_compressionType) updateDctnryExtent(pFile, nBlocks); @@ -1867,12 +1876,21 @@ int FileOp::initDctnryExtent( else Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); #endif - int savedErrno = 0; - // MCOL-498 Try to preallocate the space, fallback to write if fallocate - // has failed - //if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize)) + // MCOL-498 Skip the huge preallocations if the option is set + // for the dbroot + if ( bOptExtension ) { - // Log the failed fallocate() call result + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) + ? bOptExtension : false; + } + int savedErrno = 0; + // MCOL-498 fallocate the abbreviated extent, + // fallback to sequential write if fallocate failed + if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK + && pFile->fallocate(0, currFileSize, writeSize) ) + ) + { + // MCOL-498 Log the failed fallocate() call result if ( bOptExtension ) { std::ostringstream oss; @@ -1935,23 +1953,22 @@ int FileOp::initDctnryExtent( return ERR_FILE_WRITE; } } - } - - if (m_compressionType) - updateDctnryExtent(pFile, nBlocks); - - // Synchronize to avoid write buffer pile up too much, which could cause - // controllernode to timeout later when it needs to save a snapshot. - pFile->flush(); + // CS doesn't account flush timings. #ifdef PROFILE - - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); - else - Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT); + if (bExpandExtent) + Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); + else + Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT); #endif - } + } + } // preallocation fallback end + // MCOL-498 CS has to set a number of blocs in the chunk header + if ( m_compressionType ) + { + updateDctnryExtent(pFile, nBlocks); + } + pFile->flush(); } return NO_ERROR; diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 067453b87..48d9983a0 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1527,7 +1527,7 @@ void ColumnOp::setColParam(Column& column, * rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted * valArray - the array of row values * oldValArray - the array of old value - * bDelete - yet + * bDelete - yet. The flag must be useless. * RETURN: * NO_ERROR if success, other number otherwise ***********************************************************/ @@ -1571,7 +1571,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, } // MCOL-498 CS hasn't touched any block yet, - // but the row fill be the first in the block. + // but the row filled will be the first in the block. fistRowInBlock = ( !(curRowId % (rowsInBlock)) ) ? true : false; if( fistRowInBlock && !bDelete ) fillUpWEmptyVals = true; @@ -1708,8 +1708,6 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if ( writeSize ) setEmptyBuf( dataBuf + dataBio + curCol.colWidth, writeSize, emptyVal, curCol.colWidth ); - //fillUpWEmptyVals = false; - //fistRowInBlock = false; } rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); @@ -1726,7 +1724,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if( !fillUpWEmptyVals ) emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); // MCOL-498 Skip if this is the last block in an extent. - if ( curDataFbo != MAX_NBLOCKS - 1) + if ( curDataFbo % MAX_NBLOCKS != MAX_NBLOCKS - 1 ) { rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); if ( rc != NO_ERROR) From abf7ef80c234857b8fa9266b001d739a9d57ddae Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sun, 20 Jan 2019 21:12:23 +0300 Subject: [PATCH 34/40] MCOL-498 Changes made according with review suggestions. Add more comments. Changed return value for HDFS'es fallocate. Removed unnecessary code in ColumnBufferCompressed::writeToFile Replaced Nulls with Empties in variable names. --- utils/idbdatafile/BufferedFile.cpp | 6 ++++ utils/idbdatafile/IDBDataFile.h | 7 ++-- utils/idbdatafile/UnbufferedFile.cpp | 6 ++++ .../hdfs-shared/HdfsRdwrFileBuffer.cpp | 10 +++++- writeengine/bulk/we_colbuf.cpp | 13 ++++--- writeengine/bulk/we_colbuf.h | 6 ++-- writeengine/bulk/we_colbufcompressed.cpp | 14 ++------ writeengine/bulk/we_colbufmgr.cpp | 36 +++++++++++-------- writeengine/bulk/we_colbufmgr.h | 4 +-- writeengine/dictionary/we_dctnry.cpp | 2 ++ writeengine/shared/we_fileop.cpp | 27 +++++++++----- writeengine/shared/we_fileop.h | 4 +-- writeengine/wrapper/we_colop.cpp | 16 +++++---- 13 files changed, 96 insertions(+), 55 deletions(-) diff --git a/utils/idbdatafile/BufferedFile.cpp b/utils/idbdatafile/BufferedFile.cpp index ae12d0fd4..05973ed23 100644 --- a/utils/idbdatafile/BufferedFile.cpp +++ b/utils/idbdatafile/BufferedFile.cpp @@ -279,6 +279,12 @@ int BufferedFile::close() return ret; } +/** + @brief + The wrapper for fallocate function. + @see + This one is used in shared/we_fileop.cpp to skip expensive file preallocation. +*/ int BufferedFile::fallocate(int mode, off64_t offset, off64_t length) { int ret = 0; diff --git a/utils/idbdatafile/IDBDataFile.h b/utils/idbdatafile/IDBDataFile.h index 1ebb4a437..e0d1f18ac 100644 --- a/utils/idbdatafile/IDBDataFile.h +++ b/utils/idbdatafile/IDBDataFile.h @@ -187,8 +187,11 @@ public: virtual time_t mtime() = 0; /** - * The fallocate() method returns the modification time of the file in - * seconds. Returns -1 on error. + * The fallocate() method preallocates disk space cheaper then + * sequential write. fallocate() is supported by a limited number + * of FSes.This method is implemented for Un-/BufferedFile classes + * only. + * Returns -1 on error. */ virtual int fallocate(int mode, off64_t offset, off64_t length) = 0; diff --git a/utils/idbdatafile/UnbufferedFile.cpp b/utils/idbdatafile/UnbufferedFile.cpp index 200b583b4..5286c69b4 100644 --- a/utils/idbdatafile/UnbufferedFile.cpp +++ b/utils/idbdatafile/UnbufferedFile.cpp @@ -329,6 +329,12 @@ int UnbufferedFile::close() return ret; } +/** + @brief + The wrapper for fallocate function. + @see + This one is used in shared/we_fileop.cpp to skip expensive file preallocation. +*/ int UnbufferedFile::fallocate(int mode, off64_t offset, off64_t length) { int ret = 0; diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp index b5795227d..cab6ba545 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp @@ -317,9 +317,17 @@ int HdfsRdwrFileBuffer::close() return 0; } +/** + @brief + The dummy wrapper for fallocate function. + This is an open question which code must this method return. + fallocate fails for HDFS b/c it doesn't use it. + @see + This one is used in shared/we_fileop.cpp to skip expensive file preallocation. +*/ int HdfsRdwrFileBuffer::fallocate(int mode, off64_t offset, off64_t length) { - return 0; + return -1; } } diff --git a/writeengine/bulk/we_colbuf.cpp b/writeengine/bulk/we_colbuf.cpp index 4ade584ed..da25704af 100644 --- a/writeengine/bulk/we_colbuf.cpp +++ b/writeengine/bulk/we_colbuf.cpp @@ -101,18 +101,19 @@ void ColumnBuffer::resizeAndCopy(int newSize, int startOffset, int endOffset) //------------------------------------------------------------------------------ // Write data stored up in the output buffer to the segment column file. +// fillUpWEmpties is set when CS finishes with writing to add extra empty +// magics to fill up the block to its boundary. //------------------------------------------------------------------------------ -int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWNulls) +int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWEmpties) { if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; unsigned char *newBuf = NULL; - if ( fillUpWNulls ) + if ( fillUpWEmpties ) { BlockOp blockOp; - //TO DO Use scoped_ptr here newBuf = new unsigned char[BYTE_PER_BLOCK]; uint64_t EmptyValue = blockOp.getEmptyRowValue(fColInfo->column.dataType, fColInfo->column.width); @@ -125,19 +126,21 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWNulls) Stats::startParseEvent(WE_STATS_WRITE_COL); #endif size_t nitems; - if ( fillUpWNulls ) + if ( fillUpWEmpties ) nitems = fFile->write(newBuf, BYTE_PER_BLOCK) / BYTE_PER_BLOCK; else nitems = fFile->write(fBuffer + startOffset, writeSize) / writeSize; if (nitems != 1) + { + delete newBuf; return ERR_FILE_WRITE; + } #ifdef PROFILE Stats::stopParseEvent(WE_STATS_WRITE_COL); #endif - //TO DO Use scoped_ptr here delete newBuf; return NO_ERROR; } diff --git a/writeengine/bulk/we_colbuf.h b/writeengine/bulk/we_colbuf.h index baf11b6bb..5416b4373 100644 --- a/writeengine/bulk/we_colbuf.h +++ b/writeengine/bulk/we_colbuf.h @@ -107,11 +107,11 @@ public: * * @param startOffset The buffer offset from where the write should begin * @param writeSize The number of bytes to be written to the file - * @param fillUpWNulls The flag to fill the buffer with NULLs up to - * the block boundary. + * @param fillUpWEmpties The flag to fill the buffer with empty magic values + * up to the block boundary. */ virtual int writeToFile(int startOffset, int writeSize, - bool fillUpWNulls = false); + bool fillUpWEmpties = false); protected: diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 6aa172158..9abc0038a 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -167,16 +167,11 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf( // file, and instead buffer up the data to be compressed in 4M chunks before // writing it out. //------------------------------------------------------------------------------ -int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, - bool fillUpWNulls) +int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) { if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; - int fillUpWNullsWriteSize = 0; - if (fillUpWNulls) - fillUpWNullsWriteSize = BYTE_PER_BLOCK - writeSize % BYTE_PER_BLOCK; - // If we are starting a new file, we need to reinit the buffer and // find out what our file offset should be set to. if (!fToBeCompressedCapacity) @@ -224,7 +219,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, // Expand the compression buffer size if working with an abbrev extent, and // the bytes we are about to add will overflow the abbreviated extent. if ((fToBeCompressedCapacity < IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) && - ((fNumBytes + writeSize + fillUpWNullsWriteSize) > fToBeCompressedCapacity) ) + ((fNumBytes + writeSize) > fToBeCompressedCapacity) ) { std::ostringstream oss; oss << "Expanding abbrev to-be-compressed buffer for: OID-" << @@ -236,7 +231,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN; } - if ((fNumBytes + writeSize + fillUpWNullsWriteSize) <= fToBeCompressedCapacity) + if ((fNumBytes + writeSize) <= fToBeCompressedCapacity) { if (fLog->isDebug( DEBUG_2 )) { @@ -247,14 +242,12 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; addBytes-" << writeSize << - "; extraBytes-" << fillUpWNullsWriteSize << "; totBytes-" << (fNumBytes + writeSize); fLog->logMsg( oss.str(), MSGLVL_INFO2 ); } memcpy(bufOffset, (fBuffer + startOffset), writeSize); fNumBytes += writeSize; - fNumBytes += fillUpWNullsWriteSize; } else // Not enough room to add all the data to the to-be-compressed buffer { @@ -345,7 +338,6 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); fNumBytes += writeSizeOut; - fNumBytes += fillUpWNullsWriteSize; } startOffsetX += writeSizeOut; diff --git a/writeengine/bulk/we_colbufmgr.cpp b/writeengine/bulk/we_colbufmgr.cpp index 14ba19402..0ae34b6c9 100644 --- a/writeengine/bulk/we_colbufmgr.cpp +++ b/writeengine/bulk/we_colbufmgr.cpp @@ -521,7 +521,9 @@ int ColumnBufferManager::writeToFile(int endOffset) // and the remaining buffer data will be written to the next segment file in // the DBRoot, partition, segement number sequence. // This function also catches and handles the case where an abbreviated -// extent needs to be expanded to a full extent on disk. +// extent needs to be expanded to a full extent on disk. When fillUpWEmpties is +// set then CS finishes with writing and has to fill with magics this block +// up to its boundary. // // WARNING: This means this function may change the information in the // ColumnInfo struct that owns this ColumnBufferManager, if a @@ -529,7 +531,7 @@ int ColumnBufferManager::writeToFile(int endOffset) // internal buffer, or if an abbreviated extent is expanded. //------------------------------------------------------------------------------ int ColumnBufferManager::writeToFileExtentCheck( - uint32_t startOffset, uint32_t writeSize, bool fillUpWNulls) + uint32_t startOffset, uint32_t writeSize, bool fillUpWEmpties) { if (fLog->isDebug( DEBUG_3 )) @@ -571,7 +573,7 @@ int ColumnBufferManager::writeToFileExtentCheck( if (availableFileSize >= writeSize) { - int rc = fCBuf->writeToFile(startOffset, writeSize, fillUpWNulls); + int rc = fCBuf->writeToFile(startOffset, writeSize); if (rc != NO_ERROR) { @@ -583,9 +585,11 @@ int ColumnBufferManager::writeToFileExtentCheck( return rc; } - // MCOL-498 Fill it up to the block size boundary. - if ( fillUpWNulls ) + // MCOL-498 Fill this block up to its boundary. + if ( fillUpWEmpties ) + { writeSize = BLOCK_SIZE; + } fColInfo->updateBytesWrittenCounts( writeSize ); } @@ -628,7 +632,7 @@ int ColumnBufferManager::writeToFileExtentCheck( } int writeSize2 = writeSize - writeSize1; - rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2, fillUpWNulls); + rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2); if (rc != NO_ERROR) { @@ -640,9 +644,11 @@ int ColumnBufferManager::writeToFileExtentCheck( return rc; } - // MCOL-498 Fill it up to the block size boundary. - if ( fillUpWNulls ) + // MCOL-498 Fill this block up to its boundary. + if ( fillUpWEmpties ) + { writeSize2 = BLOCK_SIZE; + } fColInfo->updateBytesWrittenCounts( writeSize2 ); } @@ -651,7 +657,8 @@ int ColumnBufferManager::writeToFileExtentCheck( } //------------------------------------------------------------------------------ -// Flush the contents of internal fCBuf (column buffer) to disk. +// Flush the contents of internal fCBuf (column buffer) to disk. If CS flushes +// less then BLOCK_SIZE bytes then it propagates this event down the stack. //------------------------------------------------------------------------------ int ColumnBufferManager::flush( ) { @@ -676,19 +683,20 @@ int ColumnBufferManager::flush( ) int bufferSize = fCBuf->getSize(); - // MCOL-498 There are less the BLOCK_SIZE bytes in the buffer left, so + // MCOL-498 There are less the BLOCK_SIZE bytes in the buffer left + // so propagate this info down the stack to fill the buffer up + // with empty magics. // Account for circular buffer by making 2 calls to write the data, // if we are wrapping around at the end of the buffer. if (fBufFreeOffset < fBufWriteOffset) { - // The check could be redundant. - bool fillUpWEmpty = ( static_cast(bufferSize - fBufWriteOffset) >= BLOCK_SIZE ) + bool fillUpWEmpties = ( static_cast(bufferSize - fBufWriteOffset) >= BLOCK_SIZE ) ? false : true; RETURN_ON_ERROR( writeToFileExtentCheck( fBufWriteOffset, - bufferSize - fBufWriteOffset, fillUpWEmpty) ); + bufferSize - fBufWriteOffset, fillUpWEmpties) ); fBufWriteOffset = 0; } - // fill the buffer up with NULLs. + // MCOL-498 fill the buffer up with empty magics. RETURN_ON_ERROR( writeToFileExtentCheck( fBufWriteOffset, fBufFreeOffset - fBufWriteOffset, true) ); fBufWriteOffset = fBufFreeOffset; diff --git a/writeengine/bulk/we_colbufmgr.h b/writeengine/bulk/we_colbufmgr.h index 35e837ec1..574c8facc 100644 --- a/writeengine/bulk/we_colbufmgr.h +++ b/writeengine/bulk/we_colbufmgr.h @@ -193,12 +193,12 @@ protected: * write out the buffer. * @param startOffset The buffer offset where the write should begin * @param writeSize The number of bytes to be written to the file - * @param fillUpWNulls The flag to fill the buffer with NULLs up to + * @param fillUpWEmpties The flag to fill the buffer with NULLs up to * the block boundary. * @return success or fail status */ virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize, - bool fillUpWNulls = false); + bool fillUpWEmpties = false); //------------------------------------------------------------------------- // Protected Data Members diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index c7cfea1d5..7e63bc952 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -259,6 +259,8 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, if ( m_dFile != NULL ) { + // MCOL-498 CS doesn't optimize abbreviated extent + // creation. bool optimizePrealloc = ( flag ) ? false : true; rc = FileOp::initDctnryExtent( m_dFile, m_dbRoot, diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index d079700dd..2dfd315a7 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -540,7 +540,9 @@ bool FileOp::existsOIDDir( FID fid ) const * the applicable column segment file does not exist, it is created. * If this is the very first file for the specified DBRoot, then the * partition and segment number must be specified, else the selected - * partition and segment numbers are returned. + * partition and segment numbers are returned. This method tries to + * optimize full extents creation either skiping disk space + * preallocation(if activated) or via fallocate. * PARAMETERS: * oid - OID of the column to be extended * emptyVal - Empty value to be used for oid @@ -826,6 +828,7 @@ int FileOp::extendFile( return rc; // Initialize the contents of the extent. + // MCOL-498 optimize full extent creation. rc = initColumnExtent( pFile, dbRoot, allocSize, @@ -834,7 +837,7 @@ int FileOp::extendFile( newFile, // new or existing file false, // don't expand; new extent false, // add full (not abbreviated) extent - true); // try to use fallocate first + true); // try to optimize extent creation return rc; } @@ -1006,6 +1009,10 @@ int FileOp::addExtentExactFile( * This function can be used to initialize an entirely new extent, or * to finish initializing an extent that has already been started. * nBlocks controls how many 8192-byte blocks are to be written out. + * If bOptExtension is set then method first checks config for + * DBRootX.Prealloc. If it is disabled then it skips disk space + * preallocation. If not it tries to go with fallocate first then + * fallbacks to sequential write. * PARAMETERS: * pFile (in) - IDBDataFile* of column segment file to be written to * dbRoot (in) - DBRoot of pFile @@ -1016,7 +1023,7 @@ int FileOp::addExtentExactFile( * headers will be included "if" it is a compressed file. * bExpandExtent (in) - Expand existing extent, or initialize a new one * bAbbrevExtent(in) - if creating new extent, is it an abbreviated extent - * bOptExtension(in) - use fallocate() to extend the file if it is possible. + * bOptExtension(in) - skip or optimize full extent preallocation. * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1045,7 +1052,6 @@ int FileOp::initColumnExtent( } // @bug5769 Don't initialize extents or truncate db files on HDFS - // MCOL-498 We don't need sequential segment files if a PM uses SSD either. if (idbdatafile::IDBPolicy::useHdfs()) { //@Bug 3219. update the compression header after the extent is expanded. @@ -1102,7 +1108,8 @@ int FileOp::initColumnExtent( Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); #endif // MCOL-498 Skip the huge preallocations if the option is set - // for the dbroot + // for the dbroot. This check is skiped for abbreviated extent. + // IMO it is better to check bool then to call a function. if ( bOptExtension ) { bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) @@ -1802,6 +1809,10 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, * This function can be used to initialize an entirely new extent, or * to finish initializing an extent that has already been started. * nBlocks controls how many 8192-byte blocks are to be written out. + * If bOptExtension is set then method first checks config for + * DBRootX.Prealloc. If it is disabled then it skips disk space + * preallocation. If not it tries to go with fallocate first then + * fallbacks to sequential write. * PARAMETERS: * pFile (in) - IDBDataFile* of column segment file to be written to * dbRoot (in) - DBRoot of pFile @@ -1809,7 +1820,7 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, * blockHdrInit(in) - data used to initialize each block * blockHdrInitSize(in) - number of bytes in blockHdrInit * bExpandExtent (in) - Expand existing extent, or initialize a new one - * bOptExtension(in) - use fallocate() to extend the file if it is possible. + * bOptExtension(in) - skip or optimize full extent preallocation. * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1825,7 +1836,6 @@ int FileOp::initDctnryExtent( { off64_t currFileSize = pFile->size(); // @bug5769 Don't initialize extents or truncate db files on HDFS - // MCOL-498 We don't need sequential segment files if a PM uses SSD either. if (idbdatafile::IDBPolicy::useHdfs()) { if (m_compressionType) @@ -1877,7 +1887,8 @@ int FileOp::initDctnryExtent( Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); #endif // MCOL-498 Skip the huge preallocations if the option is set - // for the dbroot + // for the dbroot. This check is skiped for abbreviated extent. + // IMO it is better to check bool then to call a function. if ( bOptExtension ) { bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 175b3fb0d..2fdb279e6 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -326,7 +326,7 @@ public: * @param blockHdrInit(in) - data used to initialize each block header * @param blockHdrInitSize(in) - number of bytes in blockHdrInit * @param bExpandExtent (in) - Expand existing extent, or initialize new one - * @param bOptExtension (in) - use fallocate() to extend the file if it is possible. + * @param bOptExtension (in) - skip or optimize full extent preallocation */ EXPORT int initDctnryExtent( IDBDataFile* pFile, uint16_t dbRoot, @@ -504,7 +504,7 @@ private: // bNewFile (in) - Adding extent to new file // bExpandExtent (in) - Expand existing extent, or initialize new one // bAbbrevExtent (in) - If adding new extent, is it abbreviated - // bOptExtension(in) - use fallocate() to extend the file if it is possible. + // bOptExtension(in) - skip or optimize full extent preallocation int initColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 48d9983a0..6ec3bb445 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -472,7 +472,9 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, if ( rc != NO_ERROR) return rc; - // MCOL-498 Fill up the first block with empty values. + // MCOL-498 This must be a first block in a new extent so + // fill the block up to its boundary with empties. Otherwise + // there could be fantom values. { uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, column.colWidth); @@ -1543,7 +1545,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, uint64_t emptyVal; int rc = NO_ERROR; bool fillUpWEmptyVals = false; - bool fistRowInBlock = false; + bool firstRowInBlock = false; bool lastRowInBlock = false; uint16_t rowsInBlock = BYTE_PER_BLOCK / curCol.colWidth; @@ -1565,15 +1567,15 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, bDataDirty = false; // MCOL-498 We got into the next block, so the row is first in that block - // - fill the block up with NULLs. + // - fill the block up with empty magics. if ( curDataFbo != -1 && !bDelete ) fillUpWEmptyVals = true; } // MCOL-498 CS hasn't touched any block yet, // but the row filled will be the first in the block. - fistRowInBlock = ( !(curRowId % (rowsInBlock)) ) ? true : false; - if( fistRowInBlock && !bDelete ) + firstRowInBlock = ( !(curRowId % (rowsInBlock)) ) ? true : false; + if( firstRowInBlock && !bDelete ) fillUpWEmptyVals = true; curDataFbo = dataFbo; @@ -1585,7 +1587,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, bDataDirty = true; } - // This is a awkward way to convert void* and get ith element, I just don't have a good solution for that + // This is a awkward way to convert void* and get its element, I just don't have a good solution for that // How about pVal = valArray + i*curCol.colWidth? switch (curCol.colType) { @@ -1715,7 +1717,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if ( rc != NO_ERROR) return rc; - // MCOL-498 If it was the last row in a block fill the next block with + // MCOL-498 If it was the last row in a block fill the next block with // empty vals, otherwise next ColumnOp::allocRowId() // will fail on the next block. lastRowInBlock = ( rowsInBlock - ( curRowId % rowsInBlock ) == 1 ) ? true : false; From ecbf6b76062cab956552d3f2981b4f8a0182b561 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 21 Jan 2019 11:39:15 +0300 Subject: [PATCH 35/40] MCOL-498 Returned changes in we_colbufcompressed.* b/c they are neccesary to track compressed data size. --- writeengine/bulk/we_colbufcompressed.cpp | 15 +++++++++++---- writeengine/bulk/we_colbufcompressed.h | 6 +++--- writeengine/bulk/we_colbufmgr.cpp | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 9abc0038a..43534dc75 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -167,11 +167,16 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf( // file, and instead buffer up the data to be compressed in 4M chunks before // writing it out. //------------------------------------------------------------------------------ -int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) +int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, + bool fillUpWEmpties) { if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; + int fillUpWEmptiesWriteSize = 0; + if (fillUpWEmpties) + fillUpWEmptiesWriteSize = BYTE_PER_BLOCK - writeSize % BYTE_PER_BLOCK; + // If we are starting a new file, we need to reinit the buffer and // find out what our file offset should be set to. if (!fToBeCompressedCapacity) @@ -219,7 +224,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) // Expand the compression buffer size if working with an abbrev extent, and // the bytes we are about to add will overflow the abbreviated extent. if ((fToBeCompressedCapacity < IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) && - ((fNumBytes + writeSize) > fToBeCompressedCapacity) ) + ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) > fToBeCompressedCapacity) ) { std::ostringstream oss; oss << "Expanding abbrev to-be-compressed buffer for: OID-" << @@ -231,7 +236,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN; } - if ((fNumBytes + writeSize) <= fToBeCompressedCapacity) + if ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) <= fToBeCompressedCapacity) { if (fLog->isDebug( DEBUG_2 )) { @@ -242,12 +247,14 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; addBytes-" << writeSize << + "; extraBytes-" << fillUpWEmptiesWriteSize << "; totBytes-" << (fNumBytes + writeSize); fLog->logMsg( oss.str(), MSGLVL_INFO2 ); } memcpy(bufOffset, (fBuffer + startOffset), writeSize); fNumBytes += writeSize; + fNumBytes += fillUpWEmptiesWriteSize; } else // Not enough room to add all the data to the to-be-compressed buffer { @@ -338,6 +345,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); fNumBytes += writeSizeOut; + fNumBytes += fillUpWEmptiesWriteSize; } startOffsetX += writeSizeOut; @@ -347,7 +355,6 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) return NO_ERROR; } - //------------------------------------------------------------------------------ // Compress and write out the data in the to-be-compressed buffer. // Also may write out the compression header. diff --git a/writeengine/bulk/we_colbufcompressed.h b/writeengine/bulk/we_colbufcompressed.h index 335a8f2ba..5c4cccffc 100644 --- a/writeengine/bulk/we_colbufcompressed.h +++ b/writeengine/bulk/we_colbufcompressed.h @@ -83,11 +83,11 @@ public: * * @param startOffset The buffer offset from where the write should begin * @param writeSize The number of bytes to be written to the file - * @param fillUpWNulls The flag to fill the buffer with NULLs up to - * the block boundary. + * @param fillUpWEmpties The flag to fill the buffer with empty magic + * values up to the block boundary. */ virtual int writeToFile(int startOffset, int writeSize, - bool fillUpWNulls = false); + bool fillUpWEmpties = false); private: diff --git a/writeengine/bulk/we_colbufmgr.cpp b/writeengine/bulk/we_colbufmgr.cpp index 0ae34b6c9..9e6e78dc7 100644 --- a/writeengine/bulk/we_colbufmgr.cpp +++ b/writeengine/bulk/we_colbufmgr.cpp @@ -573,7 +573,7 @@ int ColumnBufferManager::writeToFileExtentCheck( if (availableFileSize >= writeSize) { - int rc = fCBuf->writeToFile(startOffset, writeSize); + int rc = fCBuf->writeToFile(startOffset, writeSize, fillUpWEmpties); if (rc != NO_ERROR) { @@ -632,7 +632,7 @@ int ColumnBufferManager::writeToFileExtentCheck( } int writeSize2 = writeSize - writeSize1; - rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2); + rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2, fillUpWEmpties); if (rc != NO_ERROR) { From bc3c780e358ec6dfce469fa22817e52ea8e1a65b Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 20 Feb 2019 19:56:28 +0300 Subject: [PATCH 36/40] MCOL-498 Revived unit tests for writeengine/shared and add new tests for extent extention. Added a getter, moved some methods from protected into public to use with unit tests, e.g createFile, setPreallocSpace. Added code stub in FileOp::oid2FileName to use with UT. --- CMakeLists.txt | 10 + utils/idbdatafile/IDBPolicy.cpp | 5 + utils/idbdatafile/IDBPolicy.h | 4 + writeengine/dictionary/CMakeLists.txt | 41 -- writeengine/dictionary/we_dctnry.h | 15 + writeengine/shared/CMakeLists.txt | 43 +- ...shared.cpp => shared_components_tests.cpp} | 493 ++++++++++++------ writeengine/shared/we_fileop.cpp | 20 +- writeengine/shared/we_fileop.h | 60 ++- writeengine/wrapper/CMakeLists.txt | 1 - writeengine/wrapper/we_colop.cpp | 3 +- 11 files changed, 417 insertions(+), 278 deletions(-) rename writeengine/shared/{tshared.cpp => shared_components_tests.cpp} (77%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 94fdde894..0039adc96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -276,6 +276,16 @@ ADD_SUBDIRECTORY(writeengine/server) ADD_SUBDIRECTORY(writeengine/bulk) ADD_SUBDIRECTORY(writeengine/splitter) +# WriteEngine component tests +IF( WITH_SHARED_COMP_TESTS ) + # search for cppunit + INCLUDE (findcppunit.cmake) + if (NOT CPPUNIT_FOUND) + MESSAGE(FATAL_ERROR "CPPUnit not found please install cppunit-devel for CentOS/RedHat or libcppunit-dev for Ubuntu/Debian") + endif() + ADD_SUBDIRECTORY(writeengine/shared) +ENDIF( WITH_SHARED_COMP_TESTS ) + INCLUDE(cpackEngineRPM.cmake) INCLUDE(cpackEngineDEB.cmake) diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 6d06b9b30..58b1d4601 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -297,4 +297,9 @@ void IDBPolicy::configIDBPolicy() } } +void IDBPolicy::setPreallocSpace(uint16_t dbRoot) +{ + s_PreallocSpace.push_back(dbRoot); +} + } diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h index 0e28d5004..33ec1f17e 100644 --- a/utils/idbdatafile/IDBPolicy.h +++ b/utils/idbdatafile/IDBPolicy.h @@ -129,6 +129,10 @@ public: static int listDirectory(const char* pathname, std::list& contents); static bool isDir(const char* pathname); static int copyFile(const char* srcPath, const char* destPath); + /** + * This is used in WE shared components Unit Tests + */ + static void setPreallocSpace(uint16_t dbRoot); private: /** diff --git a/writeengine/dictionary/CMakeLists.txt b/writeengine/dictionary/CMakeLists.txt index fe934813b..37317d2f1 100644 --- a/writeengine/dictionary/CMakeLists.txt +++ b/writeengine/dictionary/CMakeLists.txt @@ -1,44 +1,3 @@ - -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES} ) - - ########### install files ############### install(FILES we_dctnry.h DESTINATION include) - - - -#original Makefile.am contents follow: - -## Copyright (C) 2014 InfiniDB, Inc. -## -## This program is free software; you can redistribute it and/or -## modify it under the terms of the GNU General Public License -## as published by the Free Software Foundation; version 2 of -## the License. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, -## MA 02110-1301, USA. -# -## $Id: Makefile.am 864 2009-04-02 19:22:49Z rdempsey $ -### Process this file with automake to produce Makefile.in -# -#include_HEADERS = we_dctnry.h -# -#test: -# -#coverage: -# -#leakcheck: -# -#docs: -# -#bootstrap: install-data-am -# diff --git a/writeengine/dictionary/we_dctnry.h b/writeengine/dictionary/we_dctnry.h index 58d429142..2b7686696 100644 --- a/writeengine/dictionary/we_dctnry.h +++ b/writeengine/dictionary/we_dctnry.h @@ -132,6 +132,10 @@ public: { return m_curLbid; } + const unsigned char* getDctnryHeader2() const + { + return m_dctnryHeader2; + } /** * @brief Insert a signature value to a file block and return token/pointer. @@ -222,6 +226,17 @@ public: { return NO_ERROR; } + /** + * @brief Use this only in Unit Tests and not in prod + */ + virtual IDBDataFile* createDctnryFileUnit(const char* name, + int width, + const char* mode, + int ioBuffSize) + { + return createDctnryFile(name, width, mode, ioBuffSize); + } + //------------------------------------------------------------------------------ // Protected members diff --git a/writeengine/shared/CMakeLists.txt b/writeengine/shared/CMakeLists.txt index f49b76e9e..898da28a0 100644 --- a/writeengine/shared/CMakeLists.txt +++ b/writeengine/shared/CMakeLists.txt @@ -1,44 +1,11 @@ -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES} ) +include_directories(${ENGINE_COMMON_INCLUDES} ../dictionary) +add_executable(we_shared_components_tests ./shared_components_tests.cpp) +target_link_libraries(we_shared_components_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES}) + +install(TARGETS we_shared_components_tests DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### install files ############### install(FILES we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h DESTINATION include) - - - -#original Makefile.am contents follow: - -## Copyright (C) 2014 InfiniDB, Inc. -## -## This program is free software; you can redistribute it and/or -## modify it under the terms of the GNU General Public License -## as published by the Free Software Foundation; version 2 of -## the License. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, -## MA 02110-1301, USA. -# -## $Id: Makefile.am 3720 2012-04-04 18:18:49Z rdempsey $ -### Process this file with automake to produce Makefile.in -# -#include_HEADERS = we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h -# -#test: -# -#coverage: -# -#leakcheck: -# -#docs: -# -#bootstrap: install-data-am -# diff --git a/writeengine/shared/tshared.cpp b/writeengine/shared/shared_components_tests.cpp similarity index 77% rename from writeengine/shared/tshared.cpp rename to writeengine/shared/shared_components_tests.cpp index 22fa0723f..e569ecee6 100644 --- a/writeengine/shared/tshared.cpp +++ b/writeengine/shared/shared_components_tests.cpp @@ -29,18 +29,28 @@ using namespace boost; #include -#include -#include -#include -#include -#include -#include -#include +#include "we_dbfileop.h" +#include "we_type.h" +#include "we_log.h" +#include "we_convertor.h" +#include "we_brm.h" +#include "we_cache.h" +#include "we_colop.h" +#include "IDBDataFile.h" +#include "BufferedFile.h" +#include "IDBPolicy.h" +#include "IDBFileSystem.h" +#include "idbcompress.h" +#include "calpontsystemcatalog.h" +#include "we_colopcompress.h" +#include "we_dctnrycompress.h" +using namespace compress; +using namespace idbdatafile; using namespace WriteEngine; using namespace BRM; -int compare (const void* a, const void* b) +/*int compare (const void* a, const void* b) { return ( *(uint32_t*)a - * (uint32_t*)b ); } @@ -49,51 +59,55 @@ int compare1(const void* a, const void* b) { return ( (*(SortTuple*)a).key - (*(SortTuple*)b).key ); } - +*/ class SharedTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE( SharedTest ); + CPPUNIT_TEST_SUITE( SharedTest ); -//CPPUNIT_TEST(setUp); +CPPUNIT_TEST(setUp); //CPPUNIT_TEST( test1 ); // File operation testing - CPPUNIT_TEST( testFileNameOp ); - CPPUNIT_TEST( testFileHandleOp ); +// CPPUNIT_TEST( testFileNameOp ); +// CPPUNIT_TEST( testFileHandleOp ); CPPUNIT_TEST( testDirBasic ); + CPPUNIT_TEST( testCreateDeleteFile ); // Data block related testing - CPPUNIT_TEST( testCalculateRowIdBitmap ); - CPPUNIT_TEST( testBlockBuffer ); - CPPUNIT_TEST( testBitBasic ); - CPPUNIT_TEST( testBufferBit ); - CPPUNIT_TEST( testBitShift ); - CPPUNIT_TEST( testEmptyRowValue ); - CPPUNIT_TEST( testCorrectRowWidth ); +// CPPUNIT_TEST( testCalculateRowIdBitmap ); +// CPPUNIT_TEST( testBlockBuffer ); +// CPPUNIT_TEST( testBitBasic ); +// CPPUNIT_TEST( testBufferBit ); +// CPPUNIT_TEST( testBitShift ); +// CPPUNIT_TEST( testEmptyRowValue ); +// CPPUNIT_TEST( testCorrectRowWidth ); // DB File Block related testing - CPPUNIT_TEST( testDbBlock ); +// CPPUNIT_TEST( testDbBlock ); - CPPUNIT_TEST( testCopyDbFile ); +// CPPUNIT_TEST( testCopyDbFile ); +// Extent & dict related testing + CPPUNIT_TEST( testExtensionWOPrealloc ); + CPPUNIT_TEST( testDictExtensionWOPrealloc ); // Semaphore related testing - CPPUNIT_TEST( testSem ); +// CPPUNIT_TEST( testSem ); // Log related testing CPPUNIT_TEST( testLog ); // Version Buffer related testing - CPPUNIT_TEST( testHWM ); - CPPUNIT_TEST( testVB ); +// CPPUNIT_TEST( testHWM ); +// CPPUNIT_TEST( testVB ); // Disk manager related testing - CPPUNIT_TEST( testDM ); - CPPUNIT_TEST( tearDown ); +// CPPUNIT_TEST( testDM ); +// CPPUNIT_TEST( tearDown ); // Cache related testing - CPPUNIT_TEST( testCacheBasic ); - CPPUNIT_TEST( testCacheReadWrite ); +// CPPUNIT_TEST( testCacheBasic ); +// CPPUNIT_TEST( testCacheReadWrite ); CPPUNIT_TEST( testCleanup ); // NEVER COMMENT OUT THIS LINE CPPUNIT_TEST_SUITE_END(); @@ -113,8 +127,6 @@ public: void test1() { - //m_wrapper.test(); -// int numOfBlock = 10240; int numOfBlock = 1024; FILE* pFile; unsigned char writeBuf[BYTE_PER_BLOCK * 10]; @@ -139,7 +151,7 @@ public: } } } - +/* void testFileNameOp() { FileOp fileOp; @@ -342,23 +354,74 @@ public: } +*/ void testDirBasic() { FileOp fileOp; char dirName[30]; int rc; - strcpy( dirName, "testdir" ); - fileOp.removeDir( dirName ); + printf("\nRunning testDirBasic \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + IDBFileSystem& fs = IDBPolicy::getFs( "/tmp" ); + strcpy( dirName, "/tmp/testdir42" ); + fs.remove( dirName ); CPPUNIT_ASSERT( fileOp.isDir( dirName ) == false ); rc = fileOp.createDir( dirName ); CPPUNIT_ASSERT( rc == NO_ERROR ); CPPUNIT_ASSERT( fileOp.isDir( dirName ) == true ); - fileOp.removeDir( dirName ); + fs.remove( dirName ); } + void testCreateDeleteFile() + { + IDBDataFile* pFile = NULL; + FileOp fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ]; + + printf("\nRunning testCreateDeleteFile \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy( fileName, "versionbuffer" ); + fileOp.compressionType(1); + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + + int width = blockOp.getCorrectRowWidth( execplan::CalpontSystemCatalog::BIGINT, 8 ); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + uint64_t emptyVal = blockOp.getEmptyRowValue( execplan::CalpontSystemCatalog::BIGINT, 8 ); + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile( fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + 1 ); // dbroot + CPPUNIT_ASSERT( rc == NO_ERROR ); + + fileOp.closeFile(pFile); + + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", 1); + + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); + CPPUNIT_ASSERT( rc == NO_ERROR ); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + } +/* void testCalculateRowIdBitmap() { BlockOp blockOp; @@ -374,23 +437,23 @@ public: CPPUNIT_ASSERT( bio == 16 ); // Assuming 2048 per data block, 4 byte width - /* rowId = 2049; - CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 2048, 4, fbo, bio ) == true ); - CPPUNIT_ASSERT( fbo == 1 ); - CPPUNIT_ASSERT( bio == 16 ); - - // Assuming 4096 per data block, 2 byte width - rowId = 2049; - CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 4096, 2, fbo, bio ) == true ); - CPPUNIT_ASSERT( fbo == 1 ); - CPPUNIT_ASSERT( bio == 16 ); - - // Assuming 8192 per data block, 1 byte width - rowId = 2049; - CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 8192, 1, fbo, bio ) == true ); - CPPUNIT_ASSERT( fbo == 1 ); - CPPUNIT_ASSERT( bio == 16 ); - */ +// rowId = 2049; +// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 2048, 4, fbo, bio ) == true ); +// CPPUNIT_ASSERT( fbo == 1 ); +// CPPUNIT_ASSERT( bio == 16 ); +// +// // Assuming 4096 per data block, 2 byte width +// rowId = 2049; +// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 4096, 2, fbo, bio ) == true ); +// CPPUNIT_ASSERT( fbo == 1 ); +// CPPUNIT_ASSERT( bio == 16 ); +// +// // Assuming 8192 per data block, 1 byte width +// rowId = 2049; +// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 8192, 1, fbo, bio ) == true ); +// CPPUNIT_ASSERT( fbo == 1 ); +// CPPUNIT_ASSERT( bio == 16 ); +// rowId = 65546; CPPUNIT_ASSERT( blockOp.calculateRowBitmap( rowId, BYTE_PER_BLOCK * 8, fbo, bio, bbo ) == true ); CPPUNIT_ASSERT( fbo == 1 ); @@ -550,19 +613,19 @@ public: curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 8 ); CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - /* - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 9 ); - CPPUNIT_ASSERT( curVal == 0x80000001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 10 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 12 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 19 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - */ + +// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 9 ); +// CPPUNIT_ASSERT( curVal == 0x80000001 ); +// +// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 10 ); +// CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); +// +// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 12 ); +// CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); +// +// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 19 ); +// CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); +// curVal = blockOp.getEmptyRowValue( WriteEngine::DATE, 4 ); CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); @@ -645,18 +708,18 @@ public: curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 8 ); CPPUNIT_ASSERT( curVal == 8 ); - /* curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 9 ); - CPPUNIT_ASSERT( curVal == 4 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 10 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 12 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 19 ); - CPPUNIT_ASSERT( curVal == 8 ); - */ +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 9 ); +// CPPUNIT_ASSERT( curVal == 4 ); +// +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 10 ); +// CPPUNIT_ASSERT( curVal == 8 ); +// +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 12 ); +// CPPUNIT_ASSERT( curVal == 8 ); +// +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 19 ); +// CPPUNIT_ASSERT( curVal == 8 ); +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DATE, 8 ); CPPUNIT_ASSERT( curVal == 4 ); @@ -892,126 +955,229 @@ public: dbFileOp.closeFile( pTargetFile ); } +*/ - void testSem() + void testExtensionWOPrealloc() { - SemOp semOp; + IDBDataFile* pFile = NULL; FileOp fileOp; + BlockOp blockOp; + char fileName[20]; int rc; - bool bSuccess; - key_t key; - int sid, totalNum = 5; - char fileName[100]; + char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ]; + int dbRoot = 1; - semOp.setMaxSemVal( 3 ); + printf("\nRunning testExtensionWOPrealloc \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy( fileName, "versionbuffer" ); + fileOp.compressionType(1); - bSuccess = semOp.getKey( NULL, key ); - CPPUNIT_ASSERT( bSuccess == false ); + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - rc = fileOp.getFileName( 9991, fileName ); + int width = blockOp.getCorrectRowWidth( execplan::CalpontSystemCatalog::BIGINT, 8 ); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + uint64_t emptyVal = blockOp.getEmptyRowValue( execplan::CalpontSystemCatalog::BIGINT, 8 ); + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile( fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + dbRoot ); // dbroot CPPUNIT_ASSERT( rc == NO_ERROR ); - bSuccess = semOp.getKey( fileName, key ); - CPPUNIT_ASSERT( bSuccess == false ); + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); - rc = fileOp.getFileName( 999, fileName ); + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); CPPUNIT_ASSERT( rc == NO_ERROR ); - bSuccess = semOp.getKey( fileName, key ); - printf( "\nkey=%d", key ); - CPPUNIT_ASSERT( bSuccess == true ); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); - if ( semOp.existSem( sid, key ) ) - semOp.deleteSem( sid ); + // Extend the extent up to 64MB + // first run w preallocation + idbdatafile::BufferedFile* bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); + pFile = dynamic_cast(bFile); + + rc = fileOp.initColumnExtent(pFile, + dbRoot, + BYTE_PER_BLOCK, // number of blocks + emptyVal, + width, + false, // use existing file + true, // expand the extent + false, // add full (not abbreviated) extent + false); // don't optimize extention - rc = semOp.createSem( sid, key, 1000 ); - CPPUNIT_ASSERT( rc == ERR_MAX_SEM ); - rc = semOp.createSem( sid, key, totalNum ); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(bFile->size() == 67108864); + fileOp.closeFile(pFile); + // file has been extended delete the file before + // the second run + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); + + // second run with disabled preallocation + rc = fileOp.createFile( fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + dbRoot ); // dbroot CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = semOp.createSem( sid, key, totalNum ); - CPPUNIT_ASSERT( rc == ERR_SEM_EXIST ); + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); - rc = semOp.openSem( sid, key ); + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); CPPUNIT_ASSERT( rc == NO_ERROR ); - semOp.printAllVal( sid ); + fileOp.closeFile(pFile); - // lock - printf( "\nlock one in 2" ); - rc = semOp.lockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 2 ); - semOp.printAllVal( sid ); - - printf( "\nlock one in 2" ); - rc = semOp.lockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 1 ); - semOp.printAllVal( sid ); - - printf( "\nlock one in 2" ); - rc = semOp.lockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 0 ); - semOp.printAllVal( sid ); - - rc = semOp.lockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == ERR_NO_SEM_RESOURCE ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 0 ); - - rc = semOp.lockSem( sid, -2 ); - CPPUNIT_ASSERT( rc == ERR_VALUE_OUTOFRANGE ); - - rc = semOp.lockSem( sid + 1, 1 ); - CPPUNIT_ASSERT( rc == ERR_LOCK_FAIL ); - - // unlock - rc = semOp.unlockSem( sid, -2 ); - CPPUNIT_ASSERT( rc == ERR_VALUE_OUTOFRANGE ); - - rc = semOp.unlockSem( sid, 1 ); - CPPUNIT_ASSERT( rc == ERR_NO_SEM_LOCK ); - - rc = semOp.unlockSem( sid + 1, 2 ); - CPPUNIT_ASSERT( rc == ERR_UNLOCK_FAIL ); - - printf( "\nunlock one in 2" ); - rc = semOp.unlockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 1 ); - semOp.printAllVal( sid ); - - semOp.deleteSem( sid ); - CPPUNIT_ASSERT( semOp.existSem( sid, key ) == false ); - - CPPUNIT_ASSERT( semOp.getSemCount( sid + 1 ) == 0 ); + bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); + pFile = dynamic_cast(bFile); + + // disable disk space preallocation and extend + idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + rc = fileOp.initColumnExtent(pFile, + dbRoot, + BYTE_PER_BLOCK, // number of blocks + emptyVal, + width, + false, // use existing file + true, // expand the extent + false, // add full (not abbreviated) extent + true); // optimize extention + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(bFile->size() == 2105344); + fileOp.closeFile(pFile); + // file has been extended + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); } + // Create a dict file. Extend it w and w/o preallocation. + // Check the file sizes. + void testDictExtensionWOPrealloc() + { + FileOp fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + int dbRoot = 1; + int colWidth = 65535; + + DctnryCompress1 m_Dctnry; + // This is the magic for the stub in FileOp::oid2FileName + int oId = 42; + + printf("\nRunning testDictExtensionWOPrealloc "); + printf("There could be InetStreamSocket::connect errors \n"); + m_Dctnry.setDebugLevel( DEBUG_3 ); + + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy( fileName, "versionbuffer" ); + + rc = m_Dctnry.dropDctnry(oId); + // FileOp::oid2FileName is called under the hood + // Dctnry::createDctnry could be used with running CS + // createDctnryFile also uses DBRM under the hood it works though. + IDBDataFile* m_dFile = m_Dctnry.createDctnryFileUnit(fileName, + colWidth, + "w+b", + DEFAULT_BUFSIZ); + + idbdatafile::BufferedFile* bFile = (idbdatafile::BufferedFile*)m_dFile; + CPPUNIT_ASSERT(m_dFile != NULL); + + const int m_totalHdrBytes = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE; + + m_Dctnry.compressionType(1); + rc = m_Dctnry.initDctnryExtent( m_dFile, + dbRoot, + BYTE_PER_BLOCK, // 8192 + const_cast(m_Dctnry.getDctnryHeader2()), + m_totalHdrBytes, + false, + false ); //enable preallocation + // Check the file size and remove the file + CPPUNIT_ASSERT(bFile->size() == 67379200); + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); + + // Create a Dictionary for the second time + m_dFile = m_Dctnry.createDctnryFileUnit(fileName, + colWidth, + "w+b", + DEFAULT_BUFSIZ); + + // Get the file size later + bFile = (idbdatafile::BufferedFile*)m_dFile; + CPPUNIT_ASSERT(m_dFile != NULL); + + // disable preallocation and create a Dictionary + idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + m_Dctnry.compressionType(1); + rc = m_Dctnry.initDctnryExtent( m_dFile, + dbRoot, + BYTE_PER_BLOCK, + const_cast(m_Dctnry.getDctnryHeader2()), + m_totalHdrBytes, + false, + true ); //skip preallocation + + // Check the size and remove the file. + CPPUNIT_ASSERT(bFile->size() == 483328); + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); + } + + + + + void testLog() { Log log; + FileOp fileOp; string msg; int iVal = 3; - float fVal = 2.0; + char logFile[] = "test1.log"; + char logErrFile[] = "test1err.log"; - log.setLogFileName( "test1.log", "test1err.log" ); + log.setLogFileName( logFile, logErrFile ); msg = Convertor::int2Str( iVal ); - log.logMsg( msg + " this is a info message", INFO ); + log.logMsg( msg + " this is a info message", MSGLVL_INFO1 ); msg = Convertor::getTimeStr(); - log.logMsg( Convertor::float2Str( fVal ) + " this is a warning message", WARNING ); - log.logMsg( "this is an error message ", 1011, ERROR ); - log.logMsg( "this is a critical message", 1211, CRITICAL ); - - //...Test formatting an unsigned 64 bit integer. - uint64_t i64Value(UINT64_MAX); - msg = Convertor::i64ToStr( i64Value ); - CPPUNIT_ASSERT( (msg == "18446744073709551615") ); - log.logMsg( msg + " this is an info message with the max uint64_t integer value", INFO ); + log.logMsg( " this is a warning message", MSGLVL_WARNING ); + log.logMsg( "this is an error message ", 1011, MSGLVL_ERROR ); + log.logMsg( "this is a critical message", 1211, MSGLVL_CRITICAL ); + CPPUNIT_ASSERT( fileOp.exists( logFile ) == true ); + CPPUNIT_ASSERT( fileOp.exists( logErrFile ) == true ); + fileOp.deleteFile( logFile ); + fileOp.deleteFile( logErrFile ); + CPPUNIT_ASSERT( fileOp.exists( logFile ) == false ); + CPPUNIT_ASSERT( fileOp.exists( logErrFile ) == false ); } +/* + void testHWM() { int rc ; @@ -1375,6 +1541,7 @@ public: } } +*/ void testCleanup() { diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 2dfd315a7..bc176da6f 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -985,6 +985,8 @@ int FileOp::addExtentExactFile( return rc; // Initialize the contents of the extent. + // CS doesn't optimize file operations to have a valid + // segment files with empty magics rc = initColumnExtent( pFile, dbRoot, allocSize, @@ -1118,6 +1120,8 @@ int FileOp::initColumnExtent( int savedErrno = 0; // MCOL-498 fallocate the abbreviated extent, // fallback to sequential write if fallocate failed + // Couldn't use fallocate for full extents, e.g. ADD COLUMN DDL + // b/c CS has to fill the file with empty magics. if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK && pFile->fallocate(0, currFileSize, writeSize) ) ) @@ -1944,12 +1948,7 @@ int FileOp::initDctnryExtent( Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT); #endif - //std::ostringstream oss; - //oss << "initDctnryExtent: width-8(assumed)" << - //"; loopCount-" << loopCount << - //"; writeSize-" << writeSize; - //std::cout << oss.str() << std::endl; - if (remWriteSize > 0) + if (remWriteSize > 0) { if (pFile->write( writeBuf, remWriteSize ) != remWriteSize) { @@ -2302,6 +2301,15 @@ int FileOp::oid2FileName( FID fid, #endif +// Need this stub to use ColumnOp::writeRow in the unit tests +#ifdef WITH_UNIT_TESTS + if (fid == 42) + { + sprintf(fullFileName, "./versionbuffer"); + return NO_ERROR; + } +#endif + /* If is a version buffer file, the format is different. */ if (fid < 1000) { diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 2fdb279e6..e7f2fa576 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -92,6 +92,15 @@ public: execplan::CalpontSystemCatalog::ColDataType colDataType, uint64_t emptyVal = 0, int width = 1 ) ; + + /** + * @brief Create a file with a fixed file size by its name. + * Changed to public for UT. + */ + int createFile( const char* fileName, int fileSize, + uint64_t emptyVal, int width, + uint16_t dbRoot ); + /** * @brief Delete a file */ @@ -467,34 +476,6 @@ public: int compressionType() const; EXPORT virtual int flushFile(int rc, std::map& oids); - -protected: - EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks); - EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks); - - int m_compressionType; // compresssion type - -private: - //not copyable - FileOp(const FileOp& rhs); - FileOp& operator=(const FileOp& rhs); - - int createFile( const char* fileName, int fileSize, - uint64_t emptyVal, int width, - uint16_t dbRoot ); - - int expandAbbrevColumnChunk( IDBDataFile* pFile, - uint64_t emptyVal, - int colWidth, - const compress::CompChunkPtr& chunkInPtr, - compress::CompChunkPtr& chunkOutPt); - - int initAbbrevCompColumnExtent( IDBDataFile* pFile, - uint16_t dbRoot, - int nBlocks, - uint64_t emptyVal, - int width); - // Initialize an extent in a column segment file // pFile (in) IDBDataFile* of column segment file to be written to // dbRoot (in) - DBRoot of pFile @@ -515,6 +496,29 @@ private: bool bAbbrevExtent, bool bOptExtension=false ); +protected: + EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks); + EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks); + + int m_compressionType; // compresssion type + +private: + //not copyable + FileOp(const FileOp& rhs); + FileOp& operator=(const FileOp& rhs); + + int expandAbbrevColumnChunk( IDBDataFile* pFile, + uint64_t emptyVal, + int colWidth, + const compress::CompChunkPtr& chunkInPtr, + compress::CompChunkPtr& chunkOutPt); + + int initAbbrevCompColumnExtent( IDBDataFile* pFile, + uint16_t dbRoot, + int nBlocks, + uint64_t emptyVal, + int width); + static void initDbRootExtentMutexes(); static void removeDbRootExtentMutexes(); diff --git a/writeengine/wrapper/CMakeLists.txt b/writeengine/wrapper/CMakeLists.txt index 8bed5c065..205e3cd77 100644 --- a/writeengine/wrapper/CMakeLists.txt +++ b/writeengine/wrapper/CMakeLists.txt @@ -44,4 +44,3 @@ target_link_libraries(writeengine ${NETSNMP_LIBRARIES}) set_target_properties(writeengine PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS writeengine DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) - diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 6ec3bb445..1359570e5 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1529,7 +1529,8 @@ void ColumnOp::setColParam(Column& column, * rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted * valArray - the array of row values * oldValArray - the array of old value - * bDelete - yet. The flag must be useless. + * bDelete - yet. The flag must be useless b/c writeRows + * is used for deletion. * RETURN: * NO_ERROR if success, other number otherwise ***********************************************************/ From 22c0c98e61d52a582666489ee68bc55ecb5f4c73 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 3 Apr 2019 10:26:57 +0300 Subject: [PATCH 37/40] MCOL-498 Reduced number of blocks created for abbreviated extents thus reduced IO load when creating a table. Uncompressed abbreviated segment and dicts aren't affected by this b/c CS'es system catalog uses uncompressed dict files. CS now doesn't work with empty dicts files. --- writeengine/dictionary/we_dctnry.cpp | 7 +- writeengine/shared/we_fileop.cpp | 142 +++++++++++---------------- 2 files changed, 61 insertions(+), 88 deletions(-) diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index 7e63bc952..dfc5622b2 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -259,16 +259,15 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, if ( m_dFile != NULL ) { - // MCOL-498 CS doesn't optimize abbreviated extent + // MCOL-498 CS optimizes abbreviated extent // creation. - bool optimizePrealloc = ( flag ) ? false : true; rc = FileOp::initDctnryExtent( m_dFile, m_dbRoot, totalSize, m_dctnryHeader2, m_totalHdrBytes, false, - optimizePrealloc ); + true ); // explicitly optimize if (rc != NO_ERROR) { @@ -334,7 +333,7 @@ int Dctnry::expandDctnryExtent() m_dctnryHeader2, m_totalHdrBytes, true, - true ); + true ); // explicitly optimize if (rc != NO_ERROR) return rc; diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index bc176da6f..784f8b4d9 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -18,7 +18,6 @@ // $Id: we_fileop.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ #include "config.h" - #include #include #include @@ -541,8 +540,8 @@ bool FileOp::existsOIDDir( FID fid ) const * If this is the very first file for the specified DBRoot, then the * partition and segment number must be specified, else the selected * partition and segment numbers are returned. This method tries to - * optimize full extents creation either skiping disk space - * preallocation(if activated) or via fallocate. + * optimize full extents creation skiping disk space + * preallocation(if activated). * PARAMETERS: * oid - OID of the column to be extended * emptyVal - Empty value to be used for oid @@ -1013,8 +1012,7 @@ int FileOp::addExtentExactFile( * nBlocks controls how many 8192-byte blocks are to be written out. * If bOptExtension is set then method first checks config for * DBRootX.Prealloc. If it is disabled then it skips disk space - * preallocation. If not it tries to go with fallocate first then - * fallbacks to sequential write. + * preallocation. * PARAMETERS: * pFile (in) - IDBDataFile* of column segment file to be written to * dbRoot (in) - DBRoot of pFile @@ -1025,7 +1023,7 @@ int FileOp::addExtentExactFile( * headers will be included "if" it is a compressed file. * bExpandExtent (in) - Expand existing extent, or initialize a new one * bAbbrevExtent(in) - if creating new extent, is it an abbreviated extent - * bOptExtension(in) - skip or optimize full extent preallocation. + * bOptExtension(in) - skip full extent preallocation. * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1072,6 +1070,19 @@ int FileOp::initColumnExtent( // Create vector of mutexes used to serialize extent access per DBRoot initDbRootExtentMutexes( ); + // MCOL-498 Skip the huge preallocations if the option is set + // for the dbroot. This check is skiped for abbreviated extent. + // IMO it is better to check bool then to call a function. + if ( bOptExtension ) + { + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) + ? bOptExtension : false; + } + // Reduce number of blocks allocated for abbreviated extents thus + // CS writes less when creates a new table. This couldn't be zero + // b/c Snappy compressed file format doesn't tolerate empty files. + int realNBlocks = ( bOptExtension && nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK ) ? 3 : nBlocks; + // Determine the number of blocks in each call to fwrite(), and the // number of fwrite() calls to make, based on this. In other words, // we put a cap on the "writeSize" so that we don't allocate and write @@ -1079,16 +1090,15 @@ int FileOp::initColumnExtent( // expanding an abbreviated 64M extent, we may not have an even // multiple of MAX_NBLOCKS to write; remWriteSize is the number of // blocks above and beyond loopCount*MAX_NBLOCKS. - int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size + int writeSize = realNBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size int loopCount = 1; int remWriteSize = 0; - off64_t currFileSize = pFile->size(); - if (nBlocks > MAX_NBLOCKS) // 64M row extent size + if (realNBlocks > MAX_NBLOCKS) // 64M row extent size { writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; - loopCount = nBlocks / MAX_NBLOCKS; - remWriteSize = nBlocks - (loopCount * MAX_NBLOCKS); + loopCount = realNBlocks / MAX_NBLOCKS; + remWriteSize = realNBlocks - (loopCount * MAX_NBLOCKS); } // Allocate a buffer, initialize it, and use it to create the extent @@ -1109,39 +1119,13 @@ int FileOp::initColumnExtent( else Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); #endif - // MCOL-498 Skip the huge preallocations if the option is set - // for the dbroot. This check is skiped for abbreviated extent. - // IMO it is better to check bool then to call a function. - if ( bOptExtension ) + // Skip space preallocation if configured so + // fallback to sequential write otherwise. + // Couldn't avoid preallocation for full extents, + // e.g. ADD COLUMN DDL b/c CS has to fill the file + // with empty magics. + if ( !bOptExtension ) { - bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) - ? bOptExtension : false; - } - int savedErrno = 0; - // MCOL-498 fallocate the abbreviated extent, - // fallback to sequential write if fallocate failed - // Couldn't use fallocate for full extents, e.g. ADD COLUMN DDL - // b/c CS has to fill the file with empty magics. - if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK - && pFile->fallocate(0, currFileSize, writeSize) ) - ) - { - savedErrno = errno; - // Log the failed fallocate() call result - if ( bOptExtension ) - { - std::ostringstream oss; - std::string errnoMsg; - Convertor::mapErrnoToString(savedErrno, errnoMsg); - oss << "FileOp::initColumnExtent(): fallocate(" << currFileSize << - ", " << writeSize << "): errno = " << savedErrno << - ": " << errnoMsg; - logging::Message::Args args; - args.add(oss.str()); - SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_INFO, - logging::M0006); - } - #ifdef PROFILE Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); #endif @@ -1231,7 +1215,7 @@ int FileOp::initAbbrevCompColumnExtent( uint64_t emptyVal, int width) { - // Reserve disk space for full abbreviated extent + // Reserve disk space for optimized abbreviated extent int rc = initColumnExtent( pFile, dbRoot, nBlocks, @@ -1239,8 +1223,8 @@ int FileOp::initAbbrevCompColumnExtent( width, true, // new file false, // don't expand; add new extent - true ); // add abbreviated extent - + true, // add abbreviated extent + true); // optimize the initial extent if (rc != NO_ERROR) { return rc; @@ -1815,8 +1799,7 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, * nBlocks controls how many 8192-byte blocks are to be written out. * If bOptExtension is set then method first checks config for * DBRootX.Prealloc. If it is disabled then it skips disk space - * preallocation. If not it tries to go with fallocate first then - * fallbacks to sequential write. + * preallocation. * PARAMETERS: * pFile (in) - IDBDataFile* of column segment file to be written to * dbRoot (in) - DBRoot of pFile @@ -1824,7 +1807,7 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, * blockHdrInit(in) - data used to initialize each block * blockHdrInitSize(in) - number of bytes in blockHdrInit * bExpandExtent (in) - Expand existing extent, or initialize a new one - * bOptExtension(in) - skip or optimize full extent preallocation. + * bOptExtension(in) - skip full extent preallocation. * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1838,7 +1821,6 @@ int FileOp::initDctnryExtent( bool bExpandExtent, bool bOptExtension ) { - off64_t currFileSize = pFile->size(); // @bug5769 Don't initialize extents or truncate db files on HDFS if (idbdatafile::IDBPolicy::useHdfs()) { @@ -1854,6 +1836,21 @@ int FileOp::initDctnryExtent( // Create vector of mutexes used to serialize extent access per DBRoot initDbRootExtentMutexes( ); + // MCOL-498 Skip the huge preallocations if the option is set + // for the dbroot. This check is skiped for abbreviated extent. + // IMO it is better to check bool then to call a function. + // CS uses non-compressed dict files for its system catalog so + // CS doesn't optimize non-compressed dict creation. + if ( bOptExtension ) + { + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot) + && m_compressionType) ? bOptExtension : false; + } + // Reduce number of blocks allocated for abbreviated extents thus + // CS writes less when creates a new table. This couldn't be zero + // b/c Snappy compressed file format doesn't tolerate empty files. + int realNBlocks = ( bOptExtension && nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK ) ? 1 : nBlocks; + // Determine the number of blocks in each call to fwrite(), and the // number of fwrite() calls to make, based on this. In other words, // we put a cap on the "writeSize" so that we don't allocate and write @@ -1861,15 +1858,15 @@ int FileOp::initDctnryExtent( // expanding an abbreviated 64M extent, we may not have an even // multiple of MAX_NBLOCKS to write; remWriteSize is the number of // blocks above and beyond loopCount*MAX_NBLOCKS. - int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size + int writeSize = realNBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size int loopCount = 1; int remWriteSize = 0; - if (nBlocks > MAX_NBLOCKS) // 64M row extent size + if (realNBlocks > MAX_NBLOCKS) // 64M row extent size { writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; - loopCount = nBlocks / MAX_NBLOCKS; - remWriteSize = nBlocks - (loopCount * MAX_NBLOCKS); + loopCount = realNBlocks / MAX_NBLOCKS; + remWriteSize = realNBlocks - (loopCount * MAX_NBLOCKS); } // Allocate a buffer, initialize it, and use it to create the extent @@ -1890,36 +1887,13 @@ int FileOp::initDctnryExtent( else Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); #endif - // MCOL-498 Skip the huge preallocations if the option is set - // for the dbroot. This check is skiped for abbreviated extent. - // IMO it is better to check bool then to call a function. - if ( bOptExtension ) + // Skip space preallocation if configured so + // fallback to sequential write otherwise. + // Couldn't avoid preallocation for full extents, + // e.g. ADD COLUMN DDL b/c CS has to fill the file + // with empty magics. + if ( !bOptExtension ) { - bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) - ? bOptExtension : false; - } - int savedErrno = 0; - // MCOL-498 fallocate the abbreviated extent, - // fallback to sequential write if fallocate failed - if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK - && pFile->fallocate(0, currFileSize, writeSize) ) - ) - { - // MCOL-498 Log the failed fallocate() call result - if ( bOptExtension ) - { - std::ostringstream oss; - std::string errnoMsg; - Convertor::mapErrnoToString(savedErrno, errnoMsg); - oss << "FileOp::initDctnryExtent(): fallocate(" << currFileSize << - ", " << writeSize << "): errno = " << savedErrno << - ": " << errnoMsg; - logging::Message::Args args; - args.add(oss.str()); - SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_INFO, - logging::M0006); - } - // Allocate buffer, and store in scoped_array to insure it's deletion. // Create scope {...} to manage deletion of writeBuf. { @@ -1932,7 +1906,7 @@ int FileOp::initDctnryExtent( memset(writeBuf, 0, writeSize); - for (int i = 0; i < nBlocks; i++) + for (int i = 0; i < realNBlocks; i++) { memcpy( writeBuf + (i * BYTE_PER_BLOCK), blockHdrInit, From f4f053dd8cb5e09a94f6eba58b5da2e71f6d88dd Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 23 Apr 2019 14:34:49 +0300 Subject: [PATCH 38/40] MCOL-3267 CS now executes sorting inside UNION ALL sub-selects. --- dbcon/mysql/ha_calpont_execplan.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 1094ffc37..7b33f4318 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -7138,7 +7138,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // for subquery, order+limit by will be supported in infinidb. build order by columns // @todo union order by and limit support - if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + if (gwi.hasWindowFunc + || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT + || ( isUnion && ordercol )) { for (; ordercol; ordercol = ordercol->next) { From 2b9f54d682f13e93e5d91c7ec159945dca671a8c Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 23 Apr 2019 15:41:20 -0500 Subject: [PATCH 39/40] MCOL-1985 Server set decimal count for UDF based on both parameters. This doesn't work for regr_avgx and regr_avgy, which only care about one of them. Do our best to handle it reasonably. Still gives unlimited decimals for InnoDB when the unused parameter is not numeric. --- utils/regr/corr.h | 3 +-- utils/regr/covar_pop.h | 3 +-- utils/regr/covar_samp.h | 3 +-- utils/regr/regr_avgx.cpp | 7 ------- utils/regr/regr_avgx.h | 3 +-- utils/regr/regr_avgy.cpp | 9 +-------- utils/regr/regr_avgy.h | 3 +-- utils/regr/regr_count.h | 3 +-- utils/regr/regr_intercept.h | 3 +-- utils/regr/regr_r2.h | 3 +-- utils/regr/regr_slope.h | 3 +-- utils/regr/regr_sxx.h | 3 +-- utils/regr/regr_sxy.h | 3 +-- utils/regr/regr_syy.h | 3 +-- utils/regr/regrmysql.cpp | 17 ++++++++++++----- 15 files changed, 25 insertions(+), 44 deletions(-) diff --git a/utils/regr/corr.h b/utils/regr/corr.h index eba7597eb..d1b5f55ac 100644 --- a/utils/regr/corr.h +++ b/utils/regr/corr.h @@ -25,8 +25,7 @@ * Columnstore interface for for the corr function * * - * CREATE AGGREGATE FUNCTION corr returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION corr returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_corr diff --git a/utils/regr/covar_pop.h b/utils/regr/covar_pop.h index fc47d4497..dda396fb9 100644 --- a/utils/regr/covar_pop.h +++ b/utils/regr/covar_pop.h @@ -25,8 +25,7 @@ * Columnstore interface for for the covar_pop function * * - * CREATE AGGREGATE FUNCTION covar_pop returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION covar_pop returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_covar_pop diff --git a/utils/regr/covar_samp.h b/utils/regr/covar_samp.h index 6aba65054..a65625520 100644 --- a/utils/regr/covar_samp.h +++ b/utils/regr/covar_samp.h @@ -25,8 +25,7 @@ * Columnstore interface for for the covar_samp function * * - * CREATE AGGREGATE FUNCTION covar_samp returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION covar_samp returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_covar_samp diff --git a/utils/regr/regr_avgx.cpp b/utils/regr/regr_avgx.cpp index bf010e648..8e4314d01 100644 --- a/utils/regr/regr_avgx.cpp +++ b/utils/regr/regr_avgx.cpp @@ -63,13 +63,6 @@ mcsv1_UDAF::ReturnCode regr_avgx::init(mcsv1Context* context, context->setErrorMessage("regr_avgx() with a non-numeric x argument"); return mcsv1_UDAF::ERROR; } - if (!(isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgx() with a non-numeric independant (second) argument"); - return mcsv1_UDAF::ERROR; - } context->setUserDataSize(sizeof(regr_avgx_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_avgx.h b/utils/regr/regr_avgx.h index 75791f769..960a6a892 100644 --- a/utils/regr/regr_avgx.h +++ b/utils/regr/regr_avgx.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_avgx function * * - * CREATE AGGREGATE FUNCTION regr_avgx returns REAL soname - * 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_avgx returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_avgx diff --git a/utils/regr/regr_avgy.cpp b/utils/regr/regr_avgy.cpp index 7325d991f..3d49e96b4 100644 --- a/utils/regr/regr_avgy.cpp +++ b/utils/regr/regr_avgy.cpp @@ -60,14 +60,7 @@ mcsv1_UDAF::ReturnCode regr_avgy::init(mcsv1Context* context, { // The error message will be prepended with // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgy() with a non-numeric x argument"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgy() with a non-numeric dependant (first) argument"); + context->setErrorMessage("regr_avgy() with a non-numeric y argument"); return mcsv1_UDAF::ERROR; } diff --git a/utils/regr/regr_avgy.h b/utils/regr/regr_avgy.h index c99021f9f..c2a3020da 100644 --- a/utils/regr/regr_avgy.h +++ b/utils/regr/regr_avgy.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_avgy function * * - * CREATE AGGREGATE FUNCTION regr_avgy returns REAL soname - * 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_avgy returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_avgy diff --git a/utils/regr/regr_count.h b/utils/regr/regr_count.h index 4f4fc558e..25cde7898 100644 --- a/utils/regr/regr_count.h +++ b/utils/regr/regr_count.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_count function * * - * CREATE AGGREGATE FUNCTION regr_count returns INTEGER - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_count returns INTEGER soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_count diff --git a/utils/regr/regr_intercept.h b/utils/regr/regr_intercept.h index ed82477cd..ef8dc6de5 100644 --- a/utils/regr/regr_intercept.h +++ b/utils/regr/regr_intercept.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_intercept function * * - * CREATE AGGREGATE FUNCTION regr_intercept returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_intercept returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_intercept diff --git a/utils/regr/regr_r2.h b/utils/regr/regr_r2.h index d440ad5a1..968814067 100644 --- a/utils/regr/regr_r2.h +++ b/utils/regr/regr_r2.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_r2 function * * - * CREATE AGGREGATE FUNCTION regr_r2 returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_r2 returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_r2 diff --git a/utils/regr/regr_slope.h b/utils/regr/regr_slope.h index 9c148d895..8a20494c1 100644 --- a/utils/regr/regr_slope.h +++ b/utils/regr/regr_slope.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_slope function * * - * CREATE AGGREGATE FUNCTION regr_slope returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_slope returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_slope diff --git a/utils/regr/regr_sxx.h b/utils/regr/regr_sxx.h index 14d82bd55..53c771b6f 100644 --- a/utils/regr/regr_sxx.h +++ b/utils/regr/regr_sxx.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_sxx function * * - * CREATE AGGREGATE FUNCTION regr_sxx returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_sxx returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_sxx diff --git a/utils/regr/regr_sxy.h b/utils/regr/regr_sxy.h index 25aa34145..6371c6fed 100644 --- a/utils/regr/regr_sxy.h +++ b/utils/regr/regr_sxy.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_sxy function * * - * CREATE AGGREGATE FUNCTION regr_sxy returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_sxy returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_sxy diff --git a/utils/regr/regr_syy.h b/utils/regr/regr_syy.h index a837fab13..d1a582f4d 100644 --- a/utils/regr/regr_syy.h +++ b/utils/regr/regr_syy.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_syy function * * - * CREATE AGGREGATE FUNCTION regr_syy returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_syy returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_syy diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 32cd41209..2570163f1 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -167,10 +167,13 @@ extern "C" strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); return 1; } - - if (initid->decimals != DECIMAL_NOT_SPECIFIED) + if (args->arg_type[1] == DECIMAL_RESULT && initid->decimals != DECIMAL_NOT_SPECIFIED) { - initid->decimals +=4; + initid->decimals += 4; + } + else + { + initid->decimals = DECIMAL_NOT_SPECIFIED; } if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) @@ -272,9 +275,13 @@ extern "C" return 1; } - if (initid->decimals != DECIMAL_NOT_SPECIFIED) + if (args->arg_type[0] == DECIMAL_RESULT && initid->decimals != DECIMAL_NOT_SPECIFIED) { - initid->decimals +=4; + initid->decimals += 4; + } + else + { + initid->decimals = DECIMAL_NOT_SPECIFIED; } if (!(data = (struct regr_avgy_data*) malloc(sizeof(struct regr_avgy_data)))) From 291fbac506d4f3724ba152b77cd979391bea38e3 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 25 Apr 2019 10:51:02 +0100 Subject: [PATCH 40/40] Fix merge issue --- dbcon/mysql/ha_calpont_impl.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index b20354726..83442b3d3 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -2135,7 +2135,7 @@ int ha_calpont_impl_rnd_init(TABLE* table) // prevent "create table as select" from running on slave thd->infinidb_vtable.hasInfiniDBTable = true; - cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); if (thd->slave_thread && !ci->replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || @@ -2190,7 +2190,6 @@ int ha_calpont_impl_rnd_init(TABLE* table) if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); idbassert(ci != 0); // MySQL sometimes calls rnd_init multiple times, plan should only be @@ -2674,7 +2673,7 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) { THD* thd = current_thd; - cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); if (thd->slave_thread && !ci->replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || @@ -2715,7 +2714,6 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); // @bug 3078 if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) {