From cad6736d641392e4e73bc20fcb2ffa3267edeab3 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Fri, 11 Feb 2022 15:18:52 +0300 Subject: [PATCH 01/55] enum for SIMD out of ifdef --- utils/common/simd_sse.h | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/utils/common/simd_sse.h b/utils/common/simd_sse.h index 9815029c7..9f7e8ffb9 100644 --- a/utils/common/simd_sse.h +++ b/utils/common/simd_sse.h @@ -17,6 +17,17 @@ #pragma once +// Column filtering is dispatched 4-way based on the column type, +// which defines implementation of comparison operations for the column values +enum ENUM_KIND +{ + KIND_DEFAULT, // compared as signed integers + KIND_UNSIGNED, // compared as unsigned integers + KIND_FLOAT, // compared as floating-point numbers + KIND_TEXT +}; // whitespace-trimmed and then compared as signed integers + + #if defined(__x86_64__) #include @@ -36,16 +47,6 @@ #include -// Column filtering is dispatched 4-way based on the column type, -// which defines implementation of comparison operations for the column values -enum ENUM_KIND -{ - KIND_DEFAULT, // compared as signed integers - KIND_UNSIGNED, // compared as unsigned integers - KIND_FLOAT, // compared as floating-point numbers - KIND_TEXT -}; // whitespace-trimmed and then compared as signed integers - namespace simd { using vi128_t = __m128i; From d8751adf2a179f8f20a377712a6e31930a005089 Mon Sep 17 00:00:00 2001 From: mariadb-RomanNavrotskiy Date: Sat, 12 Feb 2022 21:33:30 +0200 Subject: [PATCH 02/55] deb dependency 10.8 --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 9df70ecb8..20c09024a 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Depends: binutils, libjemalloc1 | libjemalloc2, libsnappy1 | libsnappy1v5, liblz4-1, - mariadb-server-10.7 (= ${server:Version}), + mariadb-server-10.8 (= ${server:Version}), net-tools, python3, ${misc:Depends}, From 973e5024d8690d7cd17e237b59f8d15bc143668c Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Wed, 9 Feb 2022 19:03:00 -0500 Subject: [PATCH 03/55] MCOL-4957 Fix performance slowdown for processing TIMESTAMP columns. Part 1: As part of MCOL-3776 to address synchronization issue while accessing the fTimeZone member of the Func class, mutex locks were added to the accessor and mutator methods. However, this slows down processing of TIMESTAMP columns in PrimProc significantly as all threads across all concurrently running queries would serialize on the mutex. This is because PrimProc only has a single global object for the functor class (class derived from Func in utils/funcexp/functor.h) for a given function name. To fix this problem: (1) We remove the fTimeZone as a member of the Func derived classes (hence removing the mutexes) and instead use the fOperationType member of the FunctionColumn class to propagate the timezone values down to the individual functor processing functions such as FunctionColumn::getStrVal(), FunctionColumn::getIntVal(), etc. (2) To achieve (1), a timezone member is added to the execplan::CalpontSystemCatalog::ColType class. Part 2: Several functors in the Funcexp code call dataconvert::gmtSecToMySQLTime() and dataconvert::mySQLTimeToGmtSec() functions for conversion between seconds since unix epoch and broken-down representation. These functions in turn call the C library function localtime_r() which currently has a known bug of holding a global lock via a call to __tz_convert. This significantly reduces performance in multi-threaded applications where multiple threads concurrently call localtime_r(). More details on the bug: https://sourceware.org/bugzilla/show_bug.cgi?id=16145 This bug in localtime_r() caused processing of the Functors in PrimProc to slowdown significantly since a query execution causes Functors code to be processed in a multi-threaded manner. As a fix, we remove the calls to localtime_r() from gmtSecToMySQLTime() and mySQLTimeToGmtSec() by performing the timezone-to-offset conversion (done in dataconvert::timeZoneToOffset()) during the execution plan creation in the plugin. Note that localtime_r() is only called when the time_zone system variable is set to "SYSTEM". This fix also required changing the timezone type from a std::string to a long across the system. --- datatypes/mcs_datatype.cpp | 8 +- datatypes/mcs_datatype.h | 35 ++- dbcon/ddlpackage/ddlpkg.h | 13 +- dbcon/ddlpackage/serialize.cpp | 7 +- dbcon/ddlpackageproc/altertableprocessor.cpp | 3 +- dbcon/ddlpackageproc/altertableprocessor.h | 2 +- dbcon/dmlpackage/calpontdmlpackage.h | 6 +- dbcon/dmlpackage/commanddmlpackage.cpp | 7 +- dbcon/dmlpackage/deletedmlpackage.cpp | 7 +- dbcon/dmlpackage/insertdmlpackage.cpp | 11 +- dbcon/dmlpackage/updatedmlpackage.cpp | 7 +- dbcon/execplan/aggregatecolumn.cpp | 7 +- dbcon/execplan/aggregatecolumn.h | 6 +- dbcon/execplan/arithmeticoperator.cpp | 7 +- dbcon/execplan/arithmeticoperator.h | 6 +- dbcon/execplan/calpontselectexecutionplan.cpp | 7 +- dbcon/execplan/calpontselectexecutionplan.h | 6 +- dbcon/execplan/calpontsystemcatalog.cpp | 4 +- dbcon/execplan/calpontsystemcatalog.h | 17 +- dbcon/execplan/constantcolumn.cpp | 7 +- dbcon/execplan/constantcolumn.h | 6 +- dbcon/execplan/functioncolumn.cpp | 8 +- dbcon/execplan/functioncolumn.h | 25 +- .../execplan/mcsanalyzetableexecutionplan.cpp | 7 +- dbcon/execplan/mcsanalyzetableexecutionplan.h | 6 +- dbcon/execplan/simplecolumn.cpp | 7 +- dbcon/execplan/simplecolumn.h | 8 +- dbcon/execplan/simplefilter.cpp | 9 +- dbcon/execplan/simplefilter.h | 8 +- dbcon/execplan/treenode.h | 19 +- dbcon/execplan/windowfunctioncolumn.cpp | 7 +- dbcon/execplan/windowfunctioncolumn.h | 6 +- dbcon/joblist/groupconcat.h | 2 +- dbcon/joblist/jlf_common.h | 2 +- dbcon/joblist/jlf_execplantojoblist.cpp | 4 +- dbcon/joblist/jlf_subquery.cpp | 2 +- dbcon/joblist/jobstep.h | 6 +- dbcon/joblist/tupleunion.h | 2 +- dbcon/mysql/ha_exists_sub.cpp | 2 +- dbcon/mysql/ha_from_sub.cpp | 28 +- dbcon/mysql/ha_in_sub.cpp | 2 +- dbcon/mysql/ha_mcs.cpp | 6 +- dbcon/mysql/ha_mcs.h | 1 + dbcon/mysql/ha_mcs_datatype.h | 15 +- dbcon/mysql/ha_mcs_ddl.cpp | 29 +- dbcon/mysql/ha_mcs_dml.cpp | 10 +- dbcon/mysql/ha_mcs_execplan.cpp | 207 +++++++------ dbcon/mysql/ha_mcs_impl.cpp | 79 +++-- dbcon/mysql/ha_mcs_impl.h | 11 +- dbcon/mysql/ha_mcs_impl_if.h | 22 +- dbcon/mysql/ha_mcs_partition.cpp | 10 +- dbcon/mysql/ha_mcs_pushdown.cpp | 12 +- dbcon/mysql/ha_mcs_pushdown.h | 5 + dbcon/mysql/ha_pseudocolumn.cpp | 4 +- dbcon/mysql/ha_scalar_sub.cpp | 9 +- dbcon/mysql/ha_select_sub.cpp | 2 +- dbcon/mysql/ha_view.cpp | 2 +- dbcon/mysql/ha_window_function.cpp | 42 ++- ddlproc/ddlprocessor.cpp | 2 +- .../columnstore/basic/r/type_timestamp.result | 141 +++++++++ .../columnstore/basic/t/type_timestamp.test | 131 ++++++++ utils/dataconvert/dataconvert.cpp | 15 +- utils/dataconvert/dataconvert.h | 288 ++++++------------ utils/funcexp/func_add_time.cpp | 2 +- utils/funcexp/func_between.cpp | 2 +- utils/funcexp/func_bitwise.cpp | 26 +- utils/funcexp/func_cast.cpp | 20 +- utils/funcexp/func_ceil.cpp | 2 +- utils/funcexp/func_char_length.cpp | 2 +- utils/funcexp/func_convert_tz.cpp | 8 +- utils/funcexp/func_date.cpp | 4 +- utils/funcexp/func_date_add.cpp | 2 +- utils/funcexp/func_date_format.cpp | 6 +- utils/funcexp/func_day.cpp | 2 +- utils/funcexp/func_dayname.cpp | 2 +- utils/funcexp/func_dayofweek.cpp | 2 +- utils/funcexp/func_dayofyear.cpp | 2 +- utils/funcexp/func_extract.cpp | 2 +- utils/funcexp/func_floor.cpp | 4 +- utils/funcexp/func_hour.cpp | 2 +- utils/funcexp/func_if.cpp | 40 +-- utils/funcexp/func_last_day.cpp | 2 +- utils/funcexp/func_math.cpp | 2 +- utils/funcexp/func_minute.cpp | 2 +- utils/funcexp/func_month.cpp | 2 +- utils/funcexp/func_monthname.cpp | 2 +- utils/funcexp/func_nullif.cpp | 3 +- utils/funcexp/func_quarter.cpp | 2 +- utils/funcexp/func_regexp.cpp | 4 +- utils/funcexp/func_round.cpp | 8 +- utils/funcexp/func_second.cpp | 2 +- utils/funcexp/func_str_to_date.cpp | 16 +- utils/funcexp/func_time.cpp | 4 +- utils/funcexp/func_time_format.cpp | 4 +- utils/funcexp/func_time_to_sec.cpp | 2 +- utils/funcexp/func_timediff.cpp | 11 +- utils/funcexp/func_timestampdiff.cpp | 4 +- utils/funcexp/func_to_days.cpp | 2 +- utils/funcexp/func_truncate.cpp | 8 +- utils/funcexp/func_week.cpp | 2 +- utils/funcexp/func_weekday.cpp | 2 +- utils/funcexp/func_year.cpp | 2 +- utils/funcexp/func_yearweek.cpp | 2 +- utils/funcexp/functor.cpp | 4 +- utils/funcexp/functor.h | 28 +- utils/funcexp/functor_real.h | 6 + utils/funcexp/functor_str.h | 2 +- utils/rowgroup/rowaggregation.cpp | 9 +- utils/rowgroup/rowaggregation.h | 8 +- writeengine/bulk/cpimport.cpp | 4 +- writeengine/bulk/we_bulkload.cpp | 2 +- writeengine/bulk/we_bulkload.h | 12 +- writeengine/bulk/we_bulkloadbuffer.cpp | 2 +- writeengine/bulk/we_bulkloadbuffer.h | 6 +- writeengine/bulk/we_tableinfo.cpp | 2 +- writeengine/bulk/we_tableinfo.h | 12 +- writeengine/server/we_ddlcommandproc.cpp | 6 +- writeengine/server/we_dmlcommandproc.cpp | 2 +- writeengine/xml/we_xmljob.cpp | 6 +- writeengine/xml/we_xmljob.h | 6 +- 120 files changed, 1022 insertions(+), 695 deletions(-) create mode 100644 mysql-test/columnstore/basic/r/type_timestamp.result create mode 100644 mysql-test/columnstore/basic/t/type_timestamp.test diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp index c33269dbb..59bc2bf21 100644 --- a/datatypes/mcs_datatype.cpp +++ b/datatypes/mcs_datatype.cpp @@ -592,7 +592,7 @@ string TypeHandlerDatetime::format(const SimpleValue& v, const SystemCatalog::Ty string TypeHandlerTimestamp::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const { - return DataConvert::timestampToString(v.toSInt64(), v.tzname()); + return DataConvert::timestampToString(v.toSInt64(), v.timeZone()); } string TypeHandlerTime::format(const SimpleValue& v, const SystemCatalog::TypeAttributesStd& attr) const @@ -893,8 +893,8 @@ class SimpleConverter : public boost::any public: SimpleConverter(const SessionParam& sp, const TypeHandler* h, const SystemCatalog::TypeAttributesStd& attr, const char* str) - : boost::any( - h->convertFromString(attr, ConvertFromStringParam(sp.tzname(), true, false), str, initPushWarning())) + : boost::any(h->convertFromString(attr, ConvertFromStringParam(sp.timeZone(), true, false), str, + initPushWarning())) { } round_style_t roundStyle() const @@ -1059,7 +1059,7 @@ SimpleValue TypeHandlerTimestamp::toSimpleValue(const SessionParam& sp, { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); SimpleConverter anyVal(sp, this, attr, str); - return SimpleValueTimestamp(anyVal.to_uint64(), sp.tzname()); + return SimpleValueTimestamp(anyVal.to_uint64(), sp.timeZone()); } SimpleValue TypeHandlerTime::toSimpleValue(const SessionParam& sp, diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 38a113db4..a2686b45e 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -561,30 +561,30 @@ enum class round_style_t : uint8_t class SessionParam { - const char* m_tzname; + long m_timeZone; public: - SessionParam(const char* tzname) : m_tzname(tzname) + SessionParam(long timeZone) : m_timeZone(timeZone) { } - const char* tzname() const + long timeZone() const { - return m_tzname; + return m_timeZone; } }; class ConvertFromStringParam { - const std::string& m_timeZone; + const long m_timeZone; const bool m_noRoundup; const bool m_isUpdate; public: - ConvertFromStringParam(const std::string& timeZone, bool noRoundup, bool isUpdate) + ConvertFromStringParam(long timeZone, bool noRoundup, bool isUpdate) : m_timeZone(timeZone), m_noRoundup(noRoundup), m_isUpdate(isUpdate) { } - const std::string& timeZone() const + long timeZone() const { return m_timeZone; } @@ -602,14 +602,14 @@ class SimpleValue { int64_t m_sint64; int128_t m_sint128; - const char* m_tzname; + long m_timeZone; public: - SimpleValue(const int64_t sint64, const int128_t& sint128, const char* tzname) - : m_sint64(sint64), m_sint128(sint128), m_tzname(tzname) + SimpleValue(const int64_t sint64, const int128_t& sint128, long timeZone) + : m_sint64(sint64), m_sint128(sint128), m_timeZone(timeZone) { } - SimpleValue() : m_sint64(0), m_sint128(0), m_tzname(0) + SimpleValue() : m_sint64(0), m_sint128(0), m_timeZone(0) { } int64_t toSInt64() const @@ -624,16 +624,16 @@ class SimpleValue { return m_sint128; } - const char* tzname() const + long timeZone() const { - return m_tzname; + return m_timeZone; } }; class SimpleValueSInt64 : public SimpleValue { public: - SimpleValueSInt64(int64_t value) : SimpleValue(value, 0, NULL) + SimpleValueSInt64(int64_t value) : SimpleValue(value, 0, 0) { } }; @@ -641,7 +641,7 @@ class SimpleValueSInt64 : public SimpleValue class SimpleValueUInt64 : public SimpleValue { public: - SimpleValueUInt64(uint64_t value) : SimpleValue(static_cast(value), 0, NULL) + SimpleValueUInt64(uint64_t value) : SimpleValue(static_cast(value), 0, 0) { } }; @@ -649,7 +649,7 @@ class SimpleValueUInt64 : public SimpleValue class SimpleValueSInt128 : public SimpleValue { public: - SimpleValueSInt128(int128_t value) : SimpleValue(0, value, NULL) + SimpleValueSInt128(int128_t value) : SimpleValue(0, value, 0) { } }; @@ -657,8 +657,7 @@ class SimpleValueSInt128 : public SimpleValue class SimpleValueTimestamp : public SimpleValue { public: - SimpleValueTimestamp(uint64_t value, const char* tzname) - : SimpleValue(static_cast(value), 0, tzname) + SimpleValueTimestamp(uint64_t value, long timeZone) : SimpleValue(static_cast(value), 0, timeZone) { } }; diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index d780c2a6d..7ad8f273c 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -1308,9 +1308,20 @@ struct AlterTableStatement : public SqlStatement return fTableName->fSchema; } + long getTimeZone() const + { + return fTimeZone; + } + void setTimeZone(long timeZone) + { + fTimeZone = timeZone; + } + QualifiedName* fTableName; AlterTableActionList fActions; - std::string fTimeZone; + private: + long fTimeZone; + public: }; /** @brief This is used during parsing when constraint attributes diff --git a/dbcon/ddlpackage/serialize.cpp b/dbcon/ddlpackage/serialize.cpp index c719e8f96..796b57cd0 100644 --- a/dbcon/ddlpackage/serialize.cpp +++ b/dbcon/ddlpackage/serialize.cpp @@ -118,7 +118,9 @@ int AlterTableStatement::unserialize(ByteStream& bytestream) // read table name fTableName->unserialize(bytestream); - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; // read alter action list quadbyte action_count; @@ -225,7 +227,8 @@ int AlterTableStatement::serialize(ByteStream& bytestream) // write table name fTableName->serialize(bytestream); - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; write_vec(fActions, bytestream); diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index 0e321e063..aca76bf4b 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -1023,7 +1023,8 @@ void AlterTableProcessor::addColumn(uint32_t sessionID, execplan::CalpontSystemC bs << (ByteStream::byte)column_iterator->colType.colDataType; bs << (uint32_t)column_iterator->colType.colWidth; bs << (ByteStream::byte)column_iterator->colType.compressionType; - bs << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bs << timeZone; // cout << "sending command fillcolumn " << endl; uint32_t msgRecived = 0; fWEClient->write_to_all(bs); diff --git a/dbcon/ddlpackageproc/altertableprocessor.h b/dbcon/ddlpackageproc/altertableprocessor.h index 9ee0bd50f..7a7f708ad 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.h +++ b/dbcon/ddlpackageproc/altertableprocessor.h @@ -148,7 +148,7 @@ class AlterTableProcessor : public DDLPackageProcessor ddlpackage::AtaTableComment& ataTableComment, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); - std::string fTimeZone; + long fTimeZone; protected: void rollBackAlter(const std::string& error, BRM::TxnID txnID, int sessionId, DDLResult& result, diff --git a/dbcon/dmlpackage/calpontdmlpackage.h b/dbcon/dmlpackage/calpontdmlpackage.h index 650dd75a1..651c151a7 100644 --- a/dbcon/dmlpackage/calpontdmlpackage.h +++ b/dbcon/dmlpackage/calpontdmlpackage.h @@ -222,14 +222,14 @@ class CalpontDMLPackage * * @param the timezone to set */ - void set_TimeZone(const std::string& timeZone) + void set_TimeZone(const long timeZone) { fTimeZone = timeZone; } /** @brief get the timezone */ - const std::string get_TimeZone() const + long get_TimeZone() const { return fTimeZone; } @@ -367,7 +367,7 @@ class CalpontDMLPackage void initializeTable(); std::string fSchemaName; - std::string fTimeZone; + long fTimeZone; std::string fTableName; std::string fDMLStatement; std::string fSQLStatement; diff --git a/dbcon/dmlpackage/commanddmlpackage.cpp b/dbcon/dmlpackage/commanddmlpackage.cpp index cf9847396..16c435008 100644 --- a/dbcon/dmlpackage/commanddmlpackage.cpp +++ b/dbcon/dmlpackage/commanddmlpackage.cpp @@ -60,7 +60,8 @@ int CommandDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fSQLStatement; // for cleartablelock, this is table lockID bytestream << (uint8_t)fLogging; bytestream << fSchemaName; - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; bytestream << fTableName; bytestream << fTableOid; bytestream << static_cast(fIsAutocommitOn); @@ -83,7 +84,9 @@ int CommandDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> logging; fLogging = (logging != 0); bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; bytestream >> fTableName; bytestream >> fTableOid; bytestream >> reinterpret_cast(fIsAutocommitOn); diff --git a/dbcon/dmlpackage/deletedmlpackage.cpp b/dbcon/dmlpackage/deletedmlpackage.cpp index 2ab6c6f2a..cdbce4a94 100644 --- a/dbcon/dmlpackage/deletedmlpackage.cpp +++ b/dbcon/dmlpackage/deletedmlpackage.cpp @@ -69,7 +69,8 @@ int DeleteDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fSQLStatement; bytestream << fSchemaName; - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; if (fTable != 0) { @@ -105,7 +106,9 @@ int DeleteDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; fTable = new DMLTable(); retval = fTable->read(bytestream); diff --git a/dbcon/dmlpackage/insertdmlpackage.cpp b/dbcon/dmlpackage/insertdmlpackage.cpp index 6d4693d4d..ef8024590 100644 --- a/dbcon/dmlpackage/insertdmlpackage.cpp +++ b/dbcon/dmlpackage/insertdmlpackage.cpp @@ -64,7 +64,8 @@ int InsertDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fDMLStatement; bytestream << fSchemaName; - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; bytestream << (uint8_t)fLogging; bytestream << (uint8_t)fLogending; @@ -95,7 +96,9 @@ int InsertDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; uint8_t logging; bytestream >> logging; fLogging = (logging != 0); @@ -125,7 +128,9 @@ void InsertDMLPackage::readMetaData(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; uint8_t logging; bytestream >> logging; fLogging = (logging != 0); diff --git a/dbcon/dmlpackage/updatedmlpackage.cpp b/dbcon/dmlpackage/updatedmlpackage.cpp index 2400e6b5a..d2096b689 100644 --- a/dbcon/dmlpackage/updatedmlpackage.cpp +++ b/dbcon/dmlpackage/updatedmlpackage.cpp @@ -68,7 +68,8 @@ int UpdateDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fSQLStatement; bytestream << fSchemaName; - bytestream << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bytestream << timeZone; bytestream << (uint8_t)fIsFromCol; if (fTable != 0) @@ -105,7 +106,9 @@ int UpdateDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; - bytestream >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bytestream >> timeZone; + fTimeZone = timeZone; uint8_t isFromCol; bytestream >> isFromCol; fIsFromCol = (isFromCol != 0); diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index c3919b893..e94588955 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -183,7 +183,8 @@ void AggregateColumn::serialize(messageqcpp::ByteStream& b) const (*rcit)->serialize(b); b << fData; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; // b << fAlias; b << fTableAlias; b << static_cast(fAsc); @@ -236,7 +237,9 @@ void AggregateColumn::unserialize(messageqcpp::ByteStream& b) } b >> fData; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; // b >> fAlias; b >> fTableAlias; b >> reinterpret_cast(fAsc); diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index 4ab0e1a3c..9b0a75c93 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -313,12 +313,12 @@ class AggregateColumn : public ReturnedColumn return false; } - inline const std::string timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -346,7 +346,7 @@ class AggregateColumn : public ReturnedColumn ColumnList fGroupByColList; ColumnList fProjectColList; SRCP fConstCol; - std::string fTimeZone; + long fTimeZone; public: /*********************************************************** diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index f3e235f14..56280e891 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -72,7 +72,8 @@ ostream& operator<<(ostream& output, const ArithmeticOperator& rhs) void ArithmeticOperator::serialize(messageqcpp::ByteStream& b) const { b << (ObjectReader::id_t)ObjectReader::ARITHMETICOPERATOR; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; const messageqcpp::ByteStream::byte tmp = fDecimalOverflowCheck; b << tmp; Operator::serialize(b); @@ -81,7 +82,9 @@ void ArithmeticOperator::serialize(messageqcpp::ByteStream& b) const void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b) { ObjectReader::checkType(b, ObjectReader::ARITHMETICOPERATOR); - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; messageqcpp::ByteStream::byte tmp; b >> tmp; fDecimalOverflowCheck = tmp; diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index aa07fbc9b..8a433361b 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -59,11 +59,11 @@ class ArithmeticOperator : public Operator return new ArithmeticOperator(*this); } - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -207,7 +207,7 @@ class ArithmeticOperator : public Operator template inline result_t execute(result_t op1, result_t op2, bool& isNull); inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); - std::string fTimeZone; + long fTimeZone; bool fDecimalOverflowCheck; }; diff --git a/dbcon/execplan/calpontselectexecutionplan.cpp b/dbcon/execplan/calpontselectexecutionplan.cpp index de4291c0e..c53cd86ab 100644 --- a/dbcon/execplan/calpontselectexecutionplan.cpp +++ b/dbcon/execplan/calpontselectexecutionplan.cpp @@ -500,7 +500,8 @@ void CalpontSelectExecutionPlan::serialize(messageqcpp::ByteStream& b) const b << fDJSPartitionSize; b << fUMMemLimit; b << (uint8_t)fIsDML; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; } void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) @@ -695,7 +696,9 @@ void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) b >> fUMMemLimit; b >> tmp8; fIsDML = tmp8; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; } bool CalpontSelectExecutionPlan::operator==(const CalpontSelectExecutionPlan& t) const diff --git a/dbcon/execplan/calpontselectexecutionplan.h b/dbcon/execplan/calpontselectexecutionplan.h index 91a6d627c..e870cf661 100644 --- a/dbcon/execplan/calpontselectexecutionplan.h +++ b/dbcon/execplan/calpontselectexecutionplan.h @@ -706,11 +706,11 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan return fIsDML; } - void timeZone(const std::string& timezone) + void timeZone(const long timezone) { fTimeZone = timezone; } - const std::string timeZone() const + long timeZone() const { return fTimeZone; } @@ -923,7 +923,7 @@ class CalpontSelectExecutionPlan : public CalpontExecutionPlan int64_t fUMMemLimit; bool fIsDML; - std::string fTimeZone; + long fTimeZone; std::vector fDynamicParseTreeVec; }; diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index a2f5ea525..500f78119 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -6105,8 +6105,8 @@ const string CalpontSystemCatalog::ColType::toString() const } boost::any CalpontSystemCatalog::ColType::convertColumnData(const std::string& data, bool& pushWarning, - const std::string& timeZone, bool nulFlag, - bool noRoundup, bool isUpdate) const + long timeZone, bool nulFlag, bool noRoundup, + bool isUpdate) const { pushWarning = false; const datatypes::TypeHandler* h = typeHandler(); diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 352758aaa..f4c69f803 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -205,7 +205,6 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog */ struct ColType : public datatypes::SystemCatalog::TypeHolderStd { - ColType(); ConstraintType constraintType; DictOID ddn; std::string defaultValue; @@ -216,11 +215,25 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog uint64_t nextvalue; // next autoincrement value uint32_t charsetNumber; const CHARSET_INFO* cs; + private: + long timeZone; + public: + ColType(); ColType(const ColType& rhs); ColType& operator=(const ColType& rhs); CHARSET_INFO* getCharset(); + + long getTimeZone() const + { + return timeZone; + } + void setTimeZone(long timeZone_) + { + timeZone = timeZone_; + } + // for F&E use. only serialize necessary info for now void serialize(messageqcpp::ByteStream& b) const { @@ -254,7 +267,7 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog * @param nRoundtrip * @param isUpdate */ - boost::any convertColumnData(const std::string& data, bool& bSaturate, const std::string& timeZone, + boost::any convertColumnData(const std::string& data, bool& bSaturate, long timeZone, bool nulFlag = false, bool noRoundup = false, bool isUpdate = false) const; const std::string toString() const; diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index 65406b108..a3094285c 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -254,7 +254,8 @@ void ConstantColumn::serialize(messageqcpp::ByteStream& b) const b << (uint32_t)fType; // b << fAlias; b << fData; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; b << static_cast(fReturnAll); b << (uint64_t)fResult.intVal; b << fResult.uintVal; @@ -278,7 +279,9 @@ void ConstantColumn::unserialize(messageqcpp::ByteStream& b) b >> fConstval; b >> (uint32_t&)fType; b >> fData; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; b >> reinterpret_cast(fReturnAll); b >> (uint64_t&)fResult.intVal; b >> fResult.uintVal; diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index e246eb390..970ffd2a4 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -113,14 +113,14 @@ class ConstantColumn : public ReturnedColumn /** * accessor */ - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } /** * mutator */ - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -204,7 +204,7 @@ class ConstantColumn : public ReturnedColumn std::string fConstval; int fType; std::string fData; - std::string fTimeZone; + long fTimeZone; /*********************************************************** * F&E framework * diff --git a/dbcon/execplan/functioncolumn.cpp b/dbcon/execplan/functioncolumn.cpp index 6aa153528..9740a82aa 100644 --- a/dbcon/execplan/functioncolumn.cpp +++ b/dbcon/execplan/functioncolumn.cpp @@ -272,7 +272,8 @@ void FunctionColumn::serialize(messageqcpp::ByteStream& b) const b << fTableAlias; b << fData; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; } void FunctionColumn::unserialize(messageqcpp::ByteStream& b) @@ -303,10 +304,11 @@ void FunctionColumn::unserialize(messageqcpp::ByteStream& b) b >> fTableAlias; b >> fData; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; FuncExp* funcExp = FuncExp::instance(); fFunctor = funcExp->getFunctor(fFunctionName); - fFunctor->timeZone(fTimeZone); fFunctor->fix(*this); // @bug 3506. Special treatment for rand() function. reset the seed diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 887352021..e0b73dae3 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -121,12 +121,12 @@ class FunctionColumn : public ReturnedColumn fTableAlias = tableAlias; } - inline const std::string timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -181,7 +181,7 @@ class FunctionColumn : public ReturnedColumn std::string fFunctionName; /// function name std::string fTableAlias; /// table alias which has the column std::string fData; /// SQL representation - std::string fTimeZone; + long fTimeZone; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -217,31 +217,38 @@ class FunctionColumn : public ReturnedColumn public: virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); fResult.strVal = fFunctor->getStrVal(row, fFunctionParms, isNull, fOperationType); return fResult.strVal; } virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getIntVal(row, fFunctionParms, isNull, fOperationType); } virtual uint64_t getUintVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getUintVal(row, fFunctionParms, isNull, fOperationType); } virtual float getFloatVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getFloatVal(row, fFunctionParms, isNull, fOperationType); } virtual double getDoubleVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getDoubleVal(row, fFunctionParms, isNull, fOperationType); } virtual long double getLongDoubleVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getLongDoubleVal(row, fFunctionParms, isNull, fOperationType); } virtual IDB_Decimal getDecimalVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); IDB_Decimal decimal = fFunctor->getDecimalVal(row, fFunctionParms, isNull, fOperationType); if (UNLIKELY(fResultType.colWidth == utils::MAXLEGACYWIDTH && fResultType.scale == decimal.scale)) @@ -276,10 +283,9 @@ class FunctionColumn : public ReturnedColumn if (fResultType.scale > decimal.scale) decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; else - decimal.value = - (int64_t)(decimal.value > 0 - ? (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 - : (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); + decimal.value = (int64_t)( + decimal.value > 0 ? (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 + : (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); } decimal.scale = fResultType.scale; @@ -288,22 +294,27 @@ class FunctionColumn : public ReturnedColumn } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getBoolVal(row, fFunctionParms, isNull, fOperationType); } virtual int32_t getDateIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getDateIntVal(row, fFunctionParms, isNull, fOperationType); } virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); } virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getTimestampIntVal(row, fFunctionParms, isNull, fOperationType); } virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { + fOperationType.setTimeZone(fTimeZone); return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); } diff --git a/dbcon/execplan/mcsanalyzetableexecutionplan.cpp b/dbcon/execplan/mcsanalyzetableexecutionplan.cpp index 982135ed9..d548c8245 100644 --- a/dbcon/execplan/mcsanalyzetableexecutionplan.cpp +++ b/dbcon/execplan/mcsanalyzetableexecutionplan.cpp @@ -97,7 +97,8 @@ void MCSAnalyzeTableExecutionPlan::serialize(messageqcpp::ByteStream& bs) const bs << fSchemaName; bs << fTableName; bs << fLocalQuery; - bs << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bs << timeZone; bs << fTraceFlags; } @@ -149,7 +150,9 @@ void MCSAnalyzeTableExecutionPlan::unserialize(messageqcpp::ByteStream& bs) bs >> fSchemaName; bs >> fTableName; bs >> fLocalQuery; - bs >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bs >> timeZone; + fTimeZone = timeZone; bs >> fTraceFlags; } } // namespace execplan diff --git a/dbcon/execplan/mcsanalyzetableexecutionplan.h b/dbcon/execplan/mcsanalyzetableexecutionplan.h index ce95809f4..5dc6d8270 100644 --- a/dbcon/execplan/mcsanalyzetableexecutionplan.h +++ b/dbcon/execplan/mcsanalyzetableexecutionplan.h @@ -169,12 +169,12 @@ class MCSAnalyzeTableExecutionPlan : public CalpontExecutionPlan return fUuid; } - void timeZone(const std::string& timezone) + void timeZone(long timezone) { fTimeZone = timezone; } - const std::string timeZone() const + long timeZone() const { return fTimeZone; } @@ -256,7 +256,7 @@ class MCSAnalyzeTableExecutionPlan : public CalpontExecutionPlan std::string fTableName; uint32_t fTraceFlags; boost::uuids::uuid fUuid; - std::string fTimeZone; + long fTimeZone; uint32_t fStatementID; uint64_t fStringScanThreshold; std::string fData; diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index d08144d75..6ecb8b1cb 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -345,7 +345,8 @@ void SimpleColumn::serialize(messageqcpp::ByteStream& b) const b << fColumnName; b << fIndexName; b << fViewName; - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; b << (uint32_t)fOid; b << fData; b << fTableAlias; @@ -362,7 +363,9 @@ void SimpleColumn::unserialize(messageqcpp::ByteStream& b) b >> fColumnName; b >> fIndexName; b >> fViewName; - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; b >> (uint32_t&)fOid; b >> fData; b >> fTableAlias; diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index a7bcaa80a..8f35d3f1f 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -151,11 +151,11 @@ class SimpleColumn : public ReturnedColumn if (lower_case_table_names) boost::algorithm::to_lower(fViewName); } - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -259,7 +259,7 @@ class SimpleColumn : public ReturnedColumn std::string fIndexName; // if belong to view, view name is non-empty std::string fViewName; - std::string fTimeZone; + long fTimeZone; bool fisColumnStore; /** @brief parse SimpleColumn text @@ -349,7 +349,7 @@ class SimpleColumn : public ReturnedColumn inline int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); - return TreeNode::getDatetimeIntVal(); + return TreeNode::getDatetimeIntVal(fTimeZone); } inline int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index ed1459b25..4d48ad212 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -55,7 +55,7 @@ SimpleFilter::SimpleFilter(const string& sql) : Filter(sql) parse(sql); } -SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const string& timeZone) +SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const long timeZone) : fOp(op), fLhs(lhs), fRhs(rhs), fIndexFlag(NOINDEX), fJoinFlag(EQUA), fTimeZone(timeZone) { convertConstant(); @@ -314,7 +314,8 @@ void SimpleFilter::serialize(messageqcpp::ByteStream& b) const b << static_cast(fIndexFlag); b << static_cast(fJoinFlag); - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; } void SimpleFilter::unserialize(messageqcpp::ByteStream& b) @@ -330,7 +331,9 @@ void SimpleFilter::unserialize(messageqcpp::ByteStream& b) fRhs = dynamic_cast(ObjectReader::createTreeNode(b)); b >> reinterpret_cast(fIndexFlag); b >> reinterpret_cast(fJoinFlag); - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; fSimpleColumnList.clear(); fAggColumnList.clear(); diff --git a/dbcon/execplan/simplefilter.h b/dbcon/execplan/simplefilter.h index 900afec2d..85d2a06c9 100644 --- a/dbcon/execplan/simplefilter.h +++ b/dbcon/execplan/simplefilter.h @@ -67,7 +67,7 @@ class SimpleFilter : public Filter SimpleFilter(); SimpleFilter(const std::string& sql); - SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const std::string& timeZone = ""); + SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const long timeZone = 0); SimpleFilter(const SimpleFilter& rhs); virtual ~SimpleFilter(); @@ -92,12 +92,12 @@ class SimpleFilter : public Filter return fLhs; } - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -220,7 +220,7 @@ class SimpleFilter : public Filter ReturnedColumn* fRhs; /// right operand int fIndexFlag; /// which side col is index int fJoinFlag; /// hash join type - std::string fTimeZone; + long fTimeZone; void parse(std::string); diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 496a6436f..105765aed 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -325,7 +325,7 @@ class TreeNode } inline bool getBoolVal(); - inline const std::string& getStrVal(const std::string& timeZone); + inline const std::string& getStrVal(const long timeZone); inline int64_t getIntVal(); inline uint64_t getUintVal(); inline float getFloatVal(); @@ -333,7 +333,7 @@ class TreeNode inline long double getLongDoubleVal(); inline IDB_Decimal getDecimalVal(); inline int32_t getDateIntVal(); - inline int64_t getDatetimeIntVal(); + inline int64_t getDatetimeIntVal(long timeZone = 0); inline int64_t getTimestampIntVal(); inline int64_t getTimeIntVal(); @@ -457,7 +457,7 @@ inline bool TreeNode::getBoolVal() return fResult.boolVal; } -inline const std::string& TreeNode::getStrVal(const std::string& timeZone) +inline const std::string& TreeNode::getStrVal(const long timeZone) { switch (fResultType.colDataType) { @@ -1052,7 +1052,7 @@ inline IDB_Decimal TreeNode::getDecimalVal() return fResult.decimalVal; } -inline int64_t TreeNode::getDatetimeIntVal() +inline int64_t TreeNode::getDatetimeIntVal(long timeZone) { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) return (fResult.intVal & 0x00000000FFFFFFC0LL) << 32; @@ -1083,6 +1083,17 @@ inline int64_t TreeNode::getDatetimeIntVal() else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) // return (fResult.intVal & 0xFFFFFFFFFFF00000LL); return (fResult.intVal); + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + { + dataconvert::TimeStamp timestamp(fResult.intVal); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone); + dataconvert::DateTime dt(m_time.year, m_time.month, m_time.day, m_time.hour, m_time.minute, m_time.second, + timestamp.msecond); + memcpy(&fResult.intVal, &dt, 8); + return fResult.intVal; + } else return getIntVal(); } diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index 00b2e9ac5..dc39677d8 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -288,7 +288,8 @@ void WindowFunctionColumn::serialize(messageqcpp::ByteStream& b) const fOrderBy.serialize(b); udafContext.serialize(b); - b << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + b << timeZone; } void WindowFunctionColumn::unserialize(messageqcpp::ByteStream& b) @@ -320,7 +321,9 @@ void WindowFunctionColumn::unserialize(messageqcpp::ByteStream& b) fOrderBy.unserialize(b); udafContext.unserialize(b); - b >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + b >> timeZone; + fTimeZone = timeZone; } void WindowFunctionColumn::addToPartition(vector& groupByList) diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 5f2b21294..b40c2dd27 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -146,12 +146,12 @@ class WindowFunctionColumn : public ReturnedColumn return udafContext; } - inline const std::string timeZone() const + inline long timeZone() const { return fTimeZone; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(const long timeZone) { fTimeZone = timeZone; } @@ -180,7 +180,7 @@ class WindowFunctionColumn : public ReturnedColumn // UDAnF support mcsv1sdk::mcsv1Context udafContext; - std::string fTimeZone; + long fTimeZone; /*********************************************************** * F&E framework * ***********************************************************/ diff --git a/dbcon/joblist/groupconcat.h b/dbcon/joblist/groupconcat.h index 24f14348d..6db944b57 100644 --- a/dbcon/joblist/groupconcat.h +++ b/dbcon/joblist/groupconcat.h @@ -127,7 +127,7 @@ class GroupConcator int64_t fGroupConcatLen; int64_t fConstantLen; boost::scoped_array fOutputString; - std::string fTimeZone; + long fTimeZone; }; // For GROUP_CONCAT withour distinct or orderby diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index 19984cbdc..d85c40a4f 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -372,7 +372,7 @@ struct JobInfo int64_t largeSideLimit; uint64_t partitionSize; bool isDML; - std::string timeZone; + long timeZone; // This is for tracking any dynamically allocated ParseTree objects // in simpleScalarFilterToParseTree() for later deletion in diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index a9c0c254b..4ba29aa56 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -132,7 +132,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo); /* This looks like an inefficient way to get NULL values. Much easier ways to do it. */ template -void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone, T& val) +void valueNullNum(const CalpontSystemCatalog::ColType& ct, const long timeZone, T& val) { T& n = val; bool pushWarning = false; @@ -274,7 +274,7 @@ void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZon template void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, - const string& timeZone, T& v) + const long timeZone, T& v) { if (str.size() == 0 || isNull) { diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index 5b612d4ae..91d8a5de3 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -64,7 +64,7 @@ using namespace joblist; namespace { -void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row, const string& timeZone) +void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row, const long timeZone) { ostringstream oss; int64_t data = 0; diff --git a/dbcon/joblist/jobstep.h b/dbcon/joblist/jobstep.h index d9e95992e..ce051dcaf 100644 --- a/dbcon/joblist/jobstep.h +++ b/dbcon/joblist/jobstep.h @@ -417,11 +417,11 @@ class JobStep fOnClauseFilter = b; } - void timeZone(const std::string& timezone) + void timeZone(const long timezone) { fTimeZone = timezone; } - const std::string timeZone() const + long timeZone() const { return fTimeZone; } @@ -496,7 +496,7 @@ class JobStep uint64_t fProgress; int64_t fStartTime; int64_t fLastStepTeleTime; - std::string fTimeZone; + long fTimeZone; private: static boost::mutex fLogMutex; diff --git a/dbcon/joblist/tupleunion.h b/dbcon/joblist/tupleunion.h index 2ac031381..23a00b273 100644 --- a/dbcon/joblist/tupleunion.h +++ b/dbcon/joblist/tupleunion.h @@ -199,7 +199,7 @@ class TupleUnion : public JobStep, public TupleDeliveryStep bool runRan, joinRan; boost::shared_ptr sessionMemLimit; - std::string fTimeZone; + long fTimeZone; }; } // namespace joblist diff --git a/dbcon/mysql/ha_exists_sub.cpp b/dbcon/mysql/ha_exists_sub.cpp index 36c44f13d..da934318d 100644 --- a/dbcon/mysql/ha_exists_sub.cpp +++ b/dbcon/mysql/ha_exists_sub.cpp @@ -96,7 +96,7 @@ execplan::ParseTree* ExistsSub::transform() csep->subType(CalpontSelectExecutionPlan::EXISTS_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index a3065d6c9..c836a5cd6 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -44,7 +44,7 @@ using namespace execplan; namespace cal_impl_if { -void derivedTableOptimization(THD* thd, SCSEP& csep) +void derivedTableOptimization(gp_walk_info* gwip, SCSEP& csep) { // @bug5634. replace the unused column with ConstantColumn from derived table column list, // ExeMgr will not project ConstantColumn. Only count for local derived column. @@ -135,8 +135,7 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) else { cols[i].reset(new ConstantColumn(val)); - (reinterpret_cast(cols[i].get())) - ->timeZone(thd->variables.time_zone->get_name()->ptr()); + (reinterpret_cast(cols[i].get()))->timeZone(gwip->timeZone); } for (uint j = 0; j < unionColVec.size(); j++) @@ -156,8 +155,7 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) else { unionColVec[j][i].reset(new ConstantColumn(val)); - (reinterpret_cast(unionColVec[j][i].get())) - ->timeZone(thd->variables.time_zone->get_name()->ptr()); + (reinterpret_cast(unionColVec[j][i].get()))->timeZone(gwip->timeZone); } } } @@ -173,15 +171,13 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) if (!cols.empty()) { cols[0].reset(new ConstantColumn(val)); - (reinterpret_cast(cols[0].get())) - ->timeZone(thd->variables.time_zone->get_name()->ptr()); + (reinterpret_cast(cols[0].get()))->timeZone(gwip->timeZone); nonConstCols.push_back(cols[0]); for (uint j = 0; j < unionColVec.size(); j++) { unionColVec[j][0].reset(new ConstantColumn(val)); - (reinterpret_cast(unionColVec[j][0].get())) - ->timeZone(thd->variables.time_zone->get_name()->ptr()); + (reinterpret_cast(unionColVec[j][0].get()))->timeZone(gwip->timeZone); nonConstUnionColVec[j].push_back(unionColVec[j][0]); } } @@ -229,7 +225,7 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) if (horizontalOptimization && pt) { pt->walk(setDerivedTable); - setDerivedFilter(thd, pt, derivedTbFilterMap, derivedTbList); + setDerivedFilter(gwip, pt, derivedTbFilterMap, derivedTbList); csep->filters(pt); } @@ -301,7 +297,7 @@ void derivedTableOptimization(THD* thd, SCSEP& csep) for (uint i = 0; i < csep->subSelectList().size(); i++) { SCSEP subselect(boost::dynamic_pointer_cast(csep->subSelectList()[i])); - derivedTableOptimization(thd, subselect); + derivedTableOptimization(gwip, subselect); } } @@ -339,7 +335,7 @@ void setDerivedTable(execplan::ParseTree* n) } } -ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, map& filterMap, +ParseTree* setDerivedFilter(gp_walk_info* gwip, ParseTree*& n, map& filterMap, CalpontSelectExecutionPlan::SelectList& derivedTbList) { if (!(n->derivedTable().empty())) @@ -381,7 +377,7 @@ ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, map& fi int64_t val = 1; n = new ParseTree(new ConstantColumn(val)); - (dynamic_cast(n->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(n->data()))->timeZone(gwip->timeZone); } else { @@ -397,10 +393,10 @@ ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, map& fi ParseTree* rhs = n->right(); if (lhs) - n->left(setDerivedFilter(thd, lhs, filterMap, derivedTbList)); + n->left(setDerivedFilter(gwip, lhs, filterMap, derivedTbList)); if (rhs) - n->right(setDerivedFilter(thd, rhs, filterMap, derivedTbList)); + n->right(setDerivedFilter(gwip, rhs, filterMap, derivedTbList)); } } @@ -428,7 +424,7 @@ SCSEP FromSubQuery::transform() csep->subType(CalpontSelectExecutionPlan::FROM_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; gwi.viewName = fGwip.viewName; diff --git a/dbcon/mysql/ha_in_sub.cpp b/dbcon/mysql/ha_in_sub.cpp index cfb4fb3be..df4017165 100644 --- a/dbcon/mysql/ha_in_sub.cpp +++ b/dbcon/mysql/ha_in_sub.cpp @@ -151,7 +151,7 @@ execplan::ParseTree* InSub::transform() csep->subType(CalpontSelectExecutionPlan::IN_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index dd10c9f36..a061c5abb 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -163,6 +163,8 @@ ha_mcs::ha_mcs(handlerton* hton, TABLE_SHARE* table_arg) HA_CAN_TABLE_CONDITION_PUSHDOWN | HA_CAN_DIRECT_UPDATE_AND_DELETE) , m_lock_type(F_UNLCK) { + const char* timeZone = current_thd->variables.time_zone->get_name()->ptr(); + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &time_zone); } /** @@ -318,7 +320,7 @@ int ha_mcs::write_row(const uchar* buf) int rc; try { - rc = ha_mcs_impl_write_row(buf, table, rows_changed); + rc = ha_mcs_impl_write_row(buf, table, rows_changed, time_zone); } catch (std::runtime_error& e) { @@ -652,7 +654,7 @@ int ha_mcs::rnd_next(uchar* buf) int rc; try { - rc = ha_mcs_impl_rnd_next(buf, table); + rc = ha_mcs_impl_rnd_next(buf, table, time_zone); } catch (std::runtime_error& e) { diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index b87414d81..6e6301564 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -51,6 +51,7 @@ class ha_mcs : public handler // call on Ubuntu18. std::vector condStack; int m_lock_type; + long time_zone; int impl_external_lock(THD* thd, TABLE* table, int lock_type); int impl_rnd_init(TABLE* table, const std::vector& condStack); diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h index 1e9079530..c0390877e 100644 --- a/dbcon/mysql/ha_mcs_datatype.h +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -29,9 +29,11 @@ class StoreFieldMariaDB : public StoreField { Field* m_field; const CalpontSystemCatalog::ColType& m_type; + long m_timeZone; public: - StoreFieldMariaDB(Field* f, const CalpontSystemCatalog::ColType& type) : m_field(f), m_type(type) + StoreFieldMariaDB(Field* f, const CalpontSystemCatalog::ColType& type, const long timeZone) + : m_field(f), m_type(type), m_timeZone(timeZone) { } @@ -76,8 +78,7 @@ class StoreFieldMariaDB : public StoreField int store_timestamp(int64_t val) override { char tmp[256]; - DataConvert::timestampToString(val, tmp, sizeof(tmp), current_thd->variables.time_zone->get_name()->ptr(), - m_type.precision); + DataConvert::timestampToString(val, tmp, sizeof(tmp), m_timeZone, m_type.precision); return store_string(tmp, strlen(tmp)); } @@ -212,8 +213,10 @@ class WriteBatchFieldMariaDB : public WriteBatchField Field* m_field; const CalpontSystemCatalog::ColType& m_type; uint32_t m_mbmaxlen; - WriteBatchFieldMariaDB(Field* field, const CalpontSystemCatalog::ColType& type, uint32_t mbmaxlen) - : m_field(field), m_type(type), m_mbmaxlen(mbmaxlen) + long m_timeZone; + WriteBatchFieldMariaDB(Field* field, const CalpontSystemCatalog::ColType& type, uint32_t mbmaxlen, + const long timeZone) + : m_field(field), m_type(type), m_mbmaxlen(mbmaxlen), m_timeZone(timeZone) { } size_t ColWriteBatchDate(const uchar* buf, bool nullVal, ColBatchWriter& ci) override @@ -325,7 +328,7 @@ class WriteBatchFieldMariaDB : public WriteBatchField my_timestamp_from_binary(&tm, buf, m_field->decimals()); MySQLTime time; - gmtSecToMySQLTime(tm.tv_sec, time, current_thd->variables.time_zone->get_name()->ptr()); + gmtSecToMySQLTime(tm.tv_sec, time, m_timeZone); if (!tm.tv_usec) { diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 465fdf47d..54ee97a17 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -649,11 +649,14 @@ bool anyNullInTheColumn(THD* thd, string& schema, string& table, string& columnN csep.returnedCols(returnedColumnList); SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(thd->variables.time_zone->get_name()->ptr()); + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + sf->timeZone(timeZoneOffset); boost::shared_ptr sop(new PredicateOperator("isnull")); sf->op(sop); ConstantColumn* rhs = new ConstantColumn("", ConstantColumn::NULLDATA); - rhs->timeZone(thd->variables.time_zone->get_name()->ptr()); + rhs->timeZone(timeZoneOffset); sf->lhs(col[0]->clone()); sf->rhs(rhs); @@ -799,6 +802,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl if (valConfig.compare("YES") == 0) isVarbinaryAllowed = true; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + //@Bug 1771. error out for not supported feature. if (typeid(stmt) == typeid(CreateTableStatement)) { @@ -901,9 +908,9 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = colType.convertColumnData( - createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, - thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = + colType.convertColumnData(createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, + pushWarning, timeZoneOffset, false, false, false); } catch (std::exception&) { @@ -1143,7 +1150,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl algorithm::to_lower(alterTable->fTableName->fName); } - alterTable->fTimeZone.assign(thd->variables.time_zone->get_name()->ptr()); + alterTable->setTimeZone(timeZoneOffset); if (schema.length() == 0) { @@ -1313,9 +1320,8 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = colType.convertColumnData( - addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, - thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(addColumnPtr->fColumnDef->fDefaultValue->fValue, + pushWarning, timeZoneOffset, false, false, false); } catch (std::exception&) { @@ -1698,9 +1704,8 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = colType.convertColumnData( - addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, - thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(addColumnsPtr->fColumns[0]->fDefaultValue->fValue, + pushWarning, timeZoneOffset, false, false, false); } catch (std::exception&) { diff --git a/dbcon/mysql/ha_mcs_dml.cpp b/dbcon/mysql/ha_mcs_dml.cpp index 05cbaad07..554e03bba 100644 --- a/dbcon/mysql/ha_mcs_dml.cpp +++ b/dbcon/mysql/ha_mcs_dml.cpp @@ -337,7 +337,10 @@ int doProcessInsertValues(TABLE* table, uint32_t size, cal_connection_info& ci, pDMLPackage->set_TableName(name); name = table->s->db.str; pDMLPackage->set_SchemaName(name); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + pDMLPackage->set_TimeZone(timeZoneOffset); if (thd->lex->sql_command == SQLCOM_INSERT_SELECT) pDMLPackage->set_isInsertSelect(true); @@ -676,7 +679,8 @@ int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_connection_info& } } -int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci) +int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, + long timeZone) { ByteStream rowData; int rc = 0; @@ -746,7 +750,7 @@ int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::ca Field* fieldPtr = table->field[colpos]; uint32_t mbmaxlen = (fieldPtr->charset() && fieldPtr->charset()->mbmaxlen) ? fieldPtr->charset()->mbmaxlen : 0; - datatypes::WriteBatchFieldMariaDB field(fieldPtr, colType, mbmaxlen); + datatypes::WriteBatchFieldMariaDB field(fieldPtr, colType, mbmaxlen, timeZone); idbassert(table == table->field[colpos]->table); buf += h->ColWriteBatch(&field, buf, nullVal, writer); } diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 7a7a8978b..7b396360e 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -1509,7 +1509,7 @@ uint32_t buildJoin(gp_walk_info& gwi, List& join_list, return 0; } -ParseTree* buildRowPredicate(THD* thd, RowColumn* lhs, RowColumn* rhs, string predicateOp) +ParseTree* buildRowPredicate(gp_walk_info* gwip, RowColumn* lhs, RowColumn* rhs, string predicateOp) { PredicateOperator* po = new PredicateOperator(predicateOp); boost::shared_ptr sop(po); @@ -1523,7 +1523,7 @@ ParseTree* buildRowPredicate(THD* thd, RowColumn* lhs, RowColumn* rhs, string pr ParseTree* pt = new ParseTree(lo); sop->setOpType(lhs->columnVec()[0]->resultType(), rhs->columnVec()[0]->resultType()); SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[0].get(), rhs->columnVec()[0].get()); - sf->timeZone(thd->variables.time_zone->get_name()->ptr()); + sf->timeZone(gwip->timeZone); pt->left(new ParseTree(sf)); for (uint32_t i = 1; i < lhs->columnVec().size(); i++) @@ -1531,7 +1531,7 @@ ParseTree* buildRowPredicate(THD* thd, RowColumn* lhs, RowColumn* rhs, string pr sop.reset(po->clone()); sop->setOpType(lhs->columnVec()[i]->resultType(), rhs->columnVec()[i]->resultType()); SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[i].get(), rhs->columnVec()[i].get()); - sf->timeZone(thd->variables.time_zone->get_name()->ptr()); + sf->timeZone(gwip->timeZone); pt->right(new ParseTree(sf)); if (i + 1 < lhs->columnVec().size()) @@ -1551,7 +1551,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It { // (c1,c2,..) = (v1,v2,...) transform to: c1=v1 and c2=v2 and ... assert(!lhs->columnVec().empty() && lhs->columnVec().size() == rhs->columnVec().size()); - gwip->ptWorkStack.push(buildRowPredicate(gwip->thd, rhs, lhs, ifp->func_name())); + gwip->ptWorkStack.push(buildRowPredicate(gwip, rhs, lhs, ifp->func_name())); } else if (ifp->functype() == Item_func::IN_FUNC) { @@ -1595,7 +1595,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It RowColumn* vals = dynamic_cast(tmpStack.top()); valVec.push_back(vals); tmpStack.pop(); - ParseTree* pt = buildRowPredicate(gwip->thd, columns, vals, predicateOp); + ParseTree* pt = buildRowPredicate(gwip, columns, vals, predicateOp); while (!tmpStack.empty()) { @@ -1604,7 +1604,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It vals = dynamic_cast(tmpStack.top()); valVec.push_back(vals); tmpStack.pop(); - pt1->right(buildRowPredicate(gwip->thd, columns->clone(), vals, predicateOp)); + pt1->right(buildRowPredicate(gwip, columns->clone(), vals, predicateOp)); pt = pt1; } @@ -1646,8 +1646,8 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It break; sop->setOpType(sc->resultType(), valVec[j]->columnVec()[i]->resultType()); - cf->pushFilter(new SimpleFilter(sop, sc->clone(), valVec[j]->columnVec()[i]->clone(), - gwip->thd->variables.time_zone->get_name()->ptr())); + cf->pushFilter( + new SimpleFilter(sop, sc->clone(), valVec[j]->columnVec()[i]->clone(), gwip->timeZone)); } if (j < valVec.size()) @@ -1792,7 +1792,7 @@ bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, execplan::ReturnedCol } SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sf->timeZone(gwip->timeZone); //@bug 2101 for when there are only constants in a delete or update where clause (eg "where 5 < 6"). // There will be no field column and it will get here only if the comparison is true. @@ -1906,11 +1906,11 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator(">")); sop->setOpType(filterCol->resultType(), rhs->resultType()); sfr = new SimpleFilter(sop, filterCol, rhs); - sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfr->timeZone(gwip->timeZone); sop.reset(new PredicateOperator("<")); sop->setOpType(filterCol->resultType(), lhs->resultType()); sfl = new SimpleFilter(sop, filterCol->clone(), lhs); - sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfl->timeZone(gwip->timeZone); ParseTree* ptp = new ParseTree(new LogicOperator("or")); ptp->left(sfr); ptp->right(sfl); @@ -1921,11 +1921,11 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator("<=")); sop->setOpType(filterCol->resultType(), rhs->resultType()); sfr = new SimpleFilter(sop, filterCol, rhs); - sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfr->timeZone(gwip->timeZone); sop.reset(new PredicateOperator(">=")); sop->setOpType(filterCol->resultType(), lhs->resultType()); sfl = new SimpleFilter(sop, filterCol->clone(), lhs); - sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfl->timeZone(gwip->timeZone); ParseTree* ptp = new ParseTree(new LogicOperator("and")); ptp->left(sfr); ptp->right(sfl); @@ -1985,8 +1985,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) cf->op(sop); sop.reset(new PredicateOperator(eqop)); sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter( - new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); + cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->timeZone)); while (!gwip->rcWorkStack.empty()) { @@ -1998,8 +1997,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) gwip->rcWorkStack.pop(); sop.reset(new PredicateOperator(eqop)); sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter( - new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); + cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->timeZone)); } if (!gwip->rcWorkStack.empty()) @@ -2065,8 +2063,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { gwip->rcWorkStack.push(new ConstantColumn((int64_t)udf->val_int())); } - (dynamic_cast(gwip->rcWorkStack.top())) - ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->timeZone); } else { @@ -2088,8 +2085,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { gwip->rcWorkStack.push(new ConstantColumn(buf.ptr(), ConstantColumn::NUM)); } - (dynamic_cast(gwip->rcWorkStack.top())) - ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->timeZone); return false; } @@ -2265,19 +2261,19 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) SimpleFilter* sfo = 0; // b IS NULL ConstantColumn* nlhs1 = new ConstantColumn("", ConstantColumn::NULLDATA); - nlhs1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + nlhs1->timeZone(gwip->timeZone); sop.reset(new PredicateOperator("isnull")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfn1 = new SimpleFilter(sop, rhs, nlhs1); - sfn1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfn1->timeZone(gwip->timeZone); ParseTree* ptpl = new ParseTree(sfn1); // a IS NULL ConstantColumn* nlhs2 = new ConstantColumn("", ConstantColumn::NULLDATA); - nlhs2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + nlhs2->timeZone(gwip->timeZone); sop.reset(new PredicateOperator("isnull")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfn2 = new SimpleFilter(sop, lhs, nlhs2); - sfn2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfn2->timeZone(gwip->timeZone); ParseTree* ptpr = new ParseTree(sfn2); // AND them both ParseTree* ptpn = new ParseTree(new LogicOperator("and")); @@ -2287,7 +2283,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator("=")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfo = new SimpleFilter(sop, lhs->clone(), rhs->clone()); - sfo->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sfo->timeZone(gwip->timeZone); // OR with the NULL comparison tree ParseTree* ptp = new ParseTree(new LogicOperator("or")); ptp->left(sfo); @@ -2337,7 +2333,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) bool buildConstPredicate(Item_func* ifp, ReturnedColumn* rhs, gp_walk_info* gwip) { SimpleFilter* sf = new SimpleFilter(); - sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + sf->timeZone(gwip->timeZone); boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); ConstantColumn* lhs = 0; @@ -2356,7 +2352,7 @@ bool buildConstPredicate(Item_func* ifp, ReturnedColumn* rhs, gp_walk_info* gwip lhs = new ConstantColumn((int64_t)0, ConstantColumn::NUM); sop.reset(new PredicateOperator("=")); } - lhs->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + lhs->timeZone(gwip->timeZone); CalpontSystemCatalog::ColType opType = rhs->resultType(); @@ -2431,7 +2427,7 @@ SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) sc->tableAlias(gwi.tbList[i].alias); sc->viewName(viewName, lower_case_table_names); sc->resultType(ct); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); break; } } @@ -2490,7 +2486,7 @@ SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) sc->tableName(csep->derivedTbAlias()); sc->colPosition(j); sc->tableAlias(csep->derivedTbAlias()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); if (!viewName.empty()) { sc->viewName(viewName, lower_case_table_names); @@ -2613,7 +2609,7 @@ void collectAllCols(gp_walk_info& gwi, Item_field* ifp) sc->tableAlias(csep->derivedTbAlias()); sc->viewName(gwi.tbList[i].view); sc->resultType(cols[j]->resultType()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); // @bug5634 derived table optimization cols[j]->incRefCount(); @@ -2661,7 +2657,7 @@ void collectAllCols(gp_walk_info& gwi, Item_field* ifp) sc->resultType(ct); sc->tableAlias(gwi.tbList[i].alias, lower_case_table_names); sc->viewName(viewName, lower_case_table_names); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); srcp.reset(sc); gwi.returnedCols.push_back(srcp); gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); @@ -2994,7 +2990,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, sc->columnName(rc->alias()); sc->sequence(0); sc->tableAlias(tan.alias); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); sc->derivedTable(csep->derivedTbAlias()); sc->derivedRefCol(rc); return sc; @@ -3013,7 +3009,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, tan.fisColumnStore, gwi.sessionid, lower_case_table_names); sc->tableAlias(table->alias.ptr(), lower_case_table_names); sc->isColumnStore(false); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); sc->resultType(fieldType_MysqlToIDB(field)); sc->oid(field->field_index + 1); return sc; @@ -3043,7 +3039,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); sc->tableAlias(tan.alias); sc->viewName(tan.view); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); sc->resultType(csc->colType(oidlist[minWidthColOffset].objnum)); sc->charsetNumber(table->field[minWidthColOffset]->charset()->number); return sc; @@ -3209,7 +3205,7 @@ ReturnedColumn* buildReturnedColumnNull(gp_walk_info& gwi) return new SimpleColumn("noop"); ConstantColumn* rc = new ConstantColumnNull(); if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); return rc; } @@ -3346,7 +3342,7 @@ static ConstantColumn* buildConstantColumnMaybeNullFromValStr(const Item* item, { ConstantColumn* rc = newConstantColumnMaybeNullFromValStrNoTz(item, valStr, gwi); if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); return rc; } @@ -3365,7 +3361,7 @@ static ConstantColumn* buildConstantColumnNotNullUsingValNative(Item* item, gp_w { ConstantColumn* rc = newConstantColumnNotNullUsingValNativeNoTz(item, gwi); if (rc) - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); return rc; } @@ -3565,7 +3561,7 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool ArithmeticColumn* ac = new ArithmeticColumn(); Item** sfitempp = item->arguments(); ArithmeticOperator* aop = new ArithmeticOperator(item->func_name()); - aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + aop->timeZone(gwi.timeZone); aop->setOverflowCheck(get_decimal_overflow_check(gwi.thd)); ParseTree* pt = new ParseTree(aop); // ReturnedColumn *lhs = 0, *rhs = 0; @@ -3692,7 +3688,7 @@ ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool else { ConstantColumn* cc = new ConstantColumn(string("0"), (int64_t)0); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + cc->timeZone(gwi.timeZone); if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause { @@ -4065,15 +4061,14 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non THD* thd = current_thd; sptp.reset( new ParseTree(new ConstantColumn(static_cast(thd->variables.default_week_format)))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } // add the keyword unit argument for interval function if (funcName == "date_add_interval" || funcName == "extract" || funcName == "timestampdiff") { - addIntervalArgs(gwi.thd, ifp, funcParms); + addIntervalArgs(&gwi, ifp, funcParms); } // check for unsupported arguments add the keyword unit argument for extract functions @@ -4123,19 +4118,19 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non // add the keyword unit argument and char length for cast functions if (funcName == "cast_as_char") { - castCharArgs(gwi.thd, ifp, funcParms); + castCharArgs(&gwi, ifp, funcParms); } // add the length and scale arguments if (funcName == "decimal_typecast") { - castDecimalArgs(gwi.thd, ifp, funcParms); + castDecimalArgs(&gwi, ifp, funcParms); } // add the type argument if (funcName == "get_format") { - castTypeArgs(gwi.thd, ifp, funcParms); + castTypeArgs(&gwi, ifp, funcParms); } // add my_time_zone @@ -4150,8 +4145,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non // FIXME: Get GMT offset (in seconds east of GMT) in Windows... sptp.reset(new ParseTree(new ConstantColumn(static_cast(0), ConstantColumn::NUM))); #endif - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } @@ -4161,12 +4155,10 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (funcParms.size() == 0) { sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed1, ConstantColumn::NUM))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed2, ConstantColumn::NUM))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); gwi.no_parm_func_list.push_back(fc); } @@ -4220,8 +4212,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non tzinfo = string((char*)buf, length); } sptp.reset(new ParseTree(new ConstantColumn(tzinfo))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); tzinfo.clear(); if (to_tzinfo) @@ -4233,8 +4224,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non tzinfo = string((char*)buf, length); } sptp.reset(new ParseTree(new ConstantColumn(tzinfo))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); tzinfo.clear(); } @@ -4253,8 +4243,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non sign = -1; } sptp.reset(new ParseTree(new ConstantColumn(sign))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } @@ -4327,16 +4316,23 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non #endif - fc->operationType(functor->operationType(funcParms, fc->resultType())); - // For some reason, MDB has MYSQL_TYPE_DATETIME2 for functions on a TIMESTAMP - if (fc->operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) + execplan::CalpontSystemCatalog::ColType& resultType = fc->resultType(); + resultType.setTimeZone(gwi.timeZone); + fc->operationType(functor->operationType(funcParms, resultType)); + + // For floor/ceiling/truncate/round functions applied on TIMESTAMP columns, set the + // function result type to TIMESTAMP + if ((funcName == "floor" || funcName == "ceiling" || funcName == "truncate" || funcName == "round") && + fc->operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) { CalpontSystemCatalog::ColType ct = fc->resultType(); ct.colDataType = CalpontSystemCatalog::TIMESTAMP; ct.colWidth = 8; fc->resultType(ct); } + fc->expressionId(ci->expressionId++); + // A few functions use a different collation than that found in // the base ifp class if (funcName == "locate" || funcName == "find_in_set" || funcName == "strcmp") @@ -4409,7 +4405,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non } } - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + fc->timeZone(gwi.timeZone); return fc; } @@ -4549,11 +4545,13 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS Func* functor = funcexp->getFunctor(funcName); fc->resultType(colType_MysqlToIDB(item)); - fc->operationType(functor->operationType(funcParms, fc->resultType())); + execplan::CalpontSystemCatalog::ColType& resultType = fc->resultType(); + resultType.setTimeZone(gwi.timeZone); + fc->operationType(functor->operationType(funcParms, resultType)); fc->functionName(funcName); fc->functionParms(funcParms); fc->expressionId(ci->expressionId++); - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + fc->timeZone(gwi.timeZone); // For function join. If any argument has non-zero joininfo, set it to the function. fc->setSimpleColumnList(); @@ -4695,7 +4693,7 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) sc->alias(ifp->name.str); sc->isColumnStore(prm.columnStore()); - sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi.timeZone); if (!prm.columnStore() && ifp->field) sc->oid(ifp->field->field_index + 1); // ExeMgr requires offset started from 1 @@ -4803,7 +4801,7 @@ static void processAggregateColumnConstArg(gp_walk_info& gwi, SRCP& parm, Aggreg { // Explicit NULL or a const function that evaluated to NULL cc = new ConstantColumnNull(); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + cc->timeZone(gwi.timeZone); parm.reset(cc); ac->constCol(SRCP(rt)); return; @@ -4878,7 +4876,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) ac = new AggregateColumn(gwi.sessionid); } - ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + ac->timeZone(gwi.timeZone); if (isp->name.length) ac->alias(isp->name.str); @@ -5397,7 +5395,7 @@ because it has multiple arguments."; return ac; } -void addIntervalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) +void addIntervalArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { string funcName = ifp->func_name(); int interval_type = -1; @@ -5409,7 +5407,7 @@ void addIntervalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) else if (funcName == "extract") interval_type = ((Item_extract*)ifp)->int_type; - functionParms.push_back(getIntervalType(thd, interval_type)); + functionParms.push_back(getIntervalType(gwip, interval_type)); SPTP sptp; if (funcName == "date_add_interval") @@ -5417,42 +5415,42 @@ void addIntervalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) if (((Item_date_add_interval*)ifp)->date_sub_interval) { sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_SUB))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } else { sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_ADD))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } } } -SPTP getIntervalType(THD* thd, int interval_type) +SPTP getIntervalType(gp_walk_info* gwip, int interval_type) { SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)interval_type))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); return sptp; } -void castCharArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) +void castCharArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { Item_char_typecast* idai = (Item_char_typecast*)ifp; SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->get_cast_length()))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } -void castDecimalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) +void castDecimalArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { Item_decimal_typecast* idai = (Item_decimal_typecast*)ifp; SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->decimals))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); // max length including sign and/or decimal points @@ -5460,12 +5458,12 @@ void castDecimalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 1))); else sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 2))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } -void castTypeArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) +void castTypeArgs(gp_walk_info* gwip, Item_func* ifp, FunctionParm& functionParms) { Item_func_get_format* get_format = (Item_func_get_format*)ifp; SPTP sptp; @@ -5474,7 +5472,7 @@ void castTypeArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) sptp.reset(new ParseTree(new ConstantColumn("DATE"))); else sptp.reset(new ParseTree(new ConstantColumn("DATETIME"))); - (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwip->timeZone); functionParms.push_back(sptp); } @@ -5575,7 +5573,7 @@ void gp_walk(const Item* item, void* arg) Item_hex_hybrid* hip = reinterpret_cast(const_cast(item)); gwip->rcWorkStack.push(new ConstantColumn((int64_t)hip->val_int(), ConstantColumn::NUM)); ConstantColumn* cc = dynamic_cast(gwip->rcWorkStack.top()); - cc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + cc->timeZone(gwip->timeZone); break; } @@ -5594,8 +5592,7 @@ void gp_walk(const Item* item, void* arg) } gwip->rcWorkStack.push(new ConstantColumn(cval)); - (dynamic_cast(gwip->rcWorkStack.top())) - ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->timeZone); break; } @@ -5630,7 +5627,7 @@ void gp_walk(const Item* item, void* arg) { // push noop for unhandled item SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwip->timeZone); gwip->rcWorkStack.push(rc); break; } @@ -5650,14 +5647,13 @@ void gp_walk(const Item* item, void* arg) { // push noop for unhandled item SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwip->timeZone); gwip->rcWorkStack.push(rc); break; } gwip->rcWorkStack.push(new ConstantColumn("", ConstantColumn::NULLDATA)); - (dynamic_cast(gwip->rcWorkStack.top())) - ->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->timeZone); break; } @@ -6175,7 +6171,7 @@ void gp_walk(const Item* item, void* arg) { // push noop for unhandled item SimpleColumn* rc = new SimpleColumn("noop"); - rc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwip->timeZone); gwip->rcWorkStack.push(rc); break; } @@ -6664,7 +6660,7 @@ int processFrom(bool& isUnion, SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& plan->data(csep->data()); // gwi for the union unit - gp_walk_info union_gwi; + gp_walk_info union_gwi(gwi.timeZone); union_gwi.thd = gwi.thd; uint32_t err = 0; @@ -6777,8 +6773,7 @@ int processWhere(SELECT_LEX& select_lex, gp_walk_info& gwi, SCSEP& csep, const s else if (join && join->zero_result_cause) { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); - (dynamic_cast(gwi.rcWorkStack.top())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.timeZone); } for (Item* item : gwi.condList) @@ -7226,7 +7221,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.sessionid = sessionID; boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); - csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + csep->timeZone(gwi.timeZone); gwi.csc = csc; CalpontSelectExecutionPlan::SelectList derivedTbList; @@ -7607,7 +7602,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i selectSubList.push_back(ssub); SimpleColumn* rc = new SimpleColumn(); rc->colSource(rc->colSource() | SELECT_SUB); - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); if (sub->get_select_lex()->get_table_list()) { @@ -8414,7 +8409,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i sc1->tableAlias(sc->tableAlias()); sc1->viewName(sc->viewName()); sc1->colPosition(0); - sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc1->timeZone(gwi.timeZone); minSc.reset(sc1); } } @@ -8473,12 +8468,12 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return 0; } -int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) +int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti, long timeZone) { gp_walk_info* gwi = ti.condInfo; if (!gwi) - gwi = new gp_walk_info(); + gwi = new gp_walk_info(timeZone); gwi->thd = thd; LEX* lex = thd->lex; @@ -8506,7 +8501,7 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) boost::algorithm::to_lower(alias); } sc->tableAlias(alias); - sc->timeZone(gwi->thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(gwi->timeZone); assert(sc); boost::shared_ptr spsc(sc); gwi->returnedCols.push_back(spsc); @@ -8581,7 +8576,10 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) { SELECT_LEX* select_lex = gi.groupByTables->select_lex; - gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + gp_walk_info gwi(timeZoneOffset); gwi.thd = thd; gwi.isGroupByHandler = true; int status = getGroupPlan(gwi, *select_lex, csep, gi); @@ -8597,7 +8595,7 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) else if (status < 0) return status; // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); + derivedTableOptimization(&gwi, csep); return 0; } @@ -8618,7 +8616,7 @@ int cs_get_derived_plan(ha_columnstore_derived_handler* handler, THD* thd, SCSEP cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; #endif // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); + derivedTableOptimization(&gwi, csep); return 0; } @@ -8649,7 +8647,7 @@ int cs_get_select_plan(ha_columnstore_select_handler* handler, THD* thd, SCSEP& cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; #endif // Derived table projection and filter optimization. - derivedTableOptimization(thd, csep); + derivedTableOptimization(&gwi, csep); return 0; } @@ -8954,8 +8952,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro else if (join && join->zero_result_cause) { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); - (dynamic_cast(gwi.rcWorkStack.top())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.timeZone); } SELECT_LEX tmp_select_lex; @@ -9443,7 +9440,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro selectSubList.push_back(ssub); SimpleColumn* rc = new SimpleColumn(); rc->colSource(rc->colSource() | SELECT_SUB); - rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + rc->timeZone(gwi.timeZone); if (sub->get_select_lex()->get_table_list()) { @@ -10401,7 +10398,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro sc1->tableName(sc->tableName()); sc1->tableAlias(sc->tableAlias()); sc1->viewName(sc->viewName()); - sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + sc1->timeZone(gwi.timeZone); sc1->colPosition(0); minSc.reset(sc1); } diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 5ef57cf43..9e7292949 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -295,7 +295,8 @@ bool onlyOneTableinTM(cal_impl_if::cal_connection_info* ci) return true; } -int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false) +int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, long timeZone, + bool handler_flag = false) { int rc = HA_ERR_END_OF_FILE; int num_attr = ti.msTablePtr->s->fields; @@ -436,7 +437,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h { // fetch and store data (*f)->set_notnull(); - datatypes::StoreFieldMariaDB mf(*f, colType); + datatypes::StoreFieldMariaDB mf(*f, colType, timeZone); h->storeValueToField(row, s, &mf); } } @@ -891,6 +892,10 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c bool isFromSameTable = true; execplan::SCSEP updateCP(new execplan::CalpontSelectExecutionPlan()); + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + updateCP->isDML(true); //@Bug 2753. the memory already freed by destructor of UpdateSqlStatement @@ -1014,9 +1019,9 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c // sysdate() etc. if (!hasNonSupportItem && !cal_impl_if::nonConstFunc(ifp) && tmpVec.size() == 0) { - gp_walk_info gwi; - gwi.thd = thd; - SRCP srcp(buildReturnedColumn(value, gwi, gwi.fatalParseError)); + gp_walk_info gwi2(gwi.timeZone); + gwi2.thd = thd; + SRCP srcp(buildReturnedColumn(value, gwi2, gwi2.fatalParseError)); ConstantColumn* constCol = dynamic_cast(srcp.get()); if (constCol) @@ -1163,7 +1168,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c char buf[64]; gettimeofday(&tv, 0); MySQLTime time; - gmtSecToMySQLTime(tv.tv_sec, time, thd->variables.time_zone->get_name()->ptr()); + gmtSecToMySQLTime(tv.tv_sec, time, timeZoneOffset); sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", time.year, time.month, time.day, time.hour, time.minute, time.second, tv.tv_usec); columnAssignmentPtr->fScalarExpression = buf; @@ -1314,7 +1319,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c pDMLPackage->set_TableName(tableName); pDMLPackage->set_SchemaName(schemaName); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + pDMLPackage->set_TimeZone(timeZoneOffset); pDMLPackage->set_IsFromCol(true); // cout << " setting isFromCol to " << isFromCol << endl; @@ -1516,7 +1521,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c CalpontSystemCatalog::TableColName tcn = csc->colName(colrids[minWidthColOffset].objnum); SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); sc->tableAlias(aliasName); - sc->timeZone(thd->variables.time_zone->get_name()->ptr()); + sc->timeZone(timeZoneOffset); sc->resultType(csc->colType(colrids[minWidthColOffset].objnum)); SRCP srcp; srcp.reset(sc); @@ -1660,7 +1665,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c // << endl; VendorDMLStatement cmdStmt("CTRL+C", DML_COMMAND, sessionID); CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + pDMLPackage->set_TimeZone(timeZoneOffset); ByteStream bytestream; bytestream << static_cast(sessionID); pDMLPackage->write(bytestream); @@ -1783,7 +1788,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c { VendorDMLStatement cmdStmt(command, DML_COMMAND, sessionID); CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); - pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); + pDMLPackage->set_TimeZone(timeZoneOffset); pDMLPackage->setTableOid(ci->tableOid); ByteStream bytestream; bytestream << static_cast(sessionID); @@ -2008,6 +2013,10 @@ int ha_mcs_impl_analyze(THD* thd, TABLE* table) execplan::MCSAnalyzeTableExecutionPlan::ReturnedColumnList returnedColumnList; execplan::MCSAnalyzeTableExecutionPlan::ColumnMap columnMap; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + // Iterate over table oid list and create a `SimpleColumn` for every column with supported type. for (uint32_t i = 0, e = oidlist.size(); i < e; ++i) { @@ -2026,7 +2035,7 @@ int ha_mcs_impl_analyze(THD* thd, TABLE* table) simpleColumn->oid(objNum); simpleColumn->alias(tableColName.column); simpleColumn->resultType(colType); - simpleColumn->timeZone(thd->variables.time_zone->get_name()->ptr()); + simpleColumn->timeZone(timeZoneOffset); returnedColumn.reset(simpleColumn); returnedColumnList.push_back(returnedColumn); @@ -2040,7 +2049,7 @@ int ha_mcs_impl_analyze(THD* thd, TABLE* table) caep->schemaName(table->s->db.str, lower_case_table_names); caep->tableName(table->s->table_name.str, lower_case_table_names); - caep->timeZone(thd->variables.time_zone->get_name()->ptr()); + caep->timeZone(timeZoneOffset); SessionManager sm; BRM::TxnID txnID; @@ -2160,7 +2169,10 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected_rows, const std::vector& condStack) { THD* thd = current_thd; - cal_impl_if::gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + cal_impl_if::gp_walk_info gwi(timeZoneOffset); gwi.thd = thd; int rc = 0; @@ -2189,8 +2201,10 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) { IDEBUG(cout << "rnd_init for table " << table->s->table_name.str << endl); THD* thd = current_thd; - - gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + gp_walk_info gwi(timeZoneOffset); gwi.thd = thd; if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) @@ -2333,7 +2347,7 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector& condStack) ti.msTablePtr = table; // send plan whenever rnd_init is called - cp_get_table_plan(thd, ti.csep, ti); + cp_get_table_plan(thd, ti.csep, ti, timeZoneOffset); } IDEBUG(cerr << table->s->table_name.str << " send plan:" << endl); @@ -2563,7 +2577,7 @@ internal_error: return ER_INTERNAL_ERROR; } -int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table) +int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table, long timeZone) { THD* thd = current_thd; @@ -2606,7 +2620,7 @@ int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table) try { - rc = fetchNextRow(buf, ti, ci); + rc = fetchNextRow(buf, ti, ci, timeZone); } catch (std::exception& e) { @@ -2846,7 +2860,7 @@ int ha_mcs_impl_delete_table(const char* name) int rc = ha_mcs_impl_delete_table_(dbName, name, *ci); return rc; } -int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed) +int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed, long timeZone) { THD* thd = current_thd; @@ -2893,7 +2907,7 @@ int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed) ((thd->lex)->sql_command == SQLCOM_LOAD) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) || ci->isCacheInsert)) { - rc = ha_mcs_impl_write_batch_row_(buf, table, *ci); + rc = ha_mcs_impl_write_batch_row_(buf, table, *ci, timeZone); } else { @@ -3894,7 +3908,10 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector& condSt #ifdef DEBUG_WALK_COND { - gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + gp_walk_info gwi(timeZoneOffset); gwi.condPush = true; gwi.sessionid = tid2sid(thd->thread_id); cout << "------------------ cond push -----------------------" << endl; @@ -3906,7 +3923,12 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector& condSt if (!ti.csep) { if (!ti.condInfo) - ti.condInfo = new gp_walk_info(); + { + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + ti.condInfo = new gp_walk_info(timeZoneOffset); + } gp_walk_info* gwi = ti.condInfo; gwi->dropCond = false; @@ -4554,7 +4576,7 @@ internal_error: * HA_ERR_END_OF_FILE if the record set has come to an end * others if something went wrong whilst getting the result set ***********************************************************/ -int ha_mcs_impl_group_by_next(TABLE* table) +int ha_mcs_impl_group_by_next(TABLE* table, long timeZone) { THD* thd = current_thd; @@ -4594,7 +4616,7 @@ int ha_mcs_impl_group_by_next(TABLE* table) { // fetchNextRow interface forces to use buf. unsigned char buf; - rc = fetchNextRow(&buf, ti, ci, true); + rc = fetchNextRow(&buf, ti, ci, timeZone, true); } catch (std::exception& e) { @@ -4801,7 +4823,10 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table) if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command)) return 0; - gp_walk_info gwi; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + gp_walk_info gwi(timeZoneOffset); gwi.thd = thd; bool err = false; @@ -5237,7 +5262,7 @@ internal_error: return ER_INTERNAL_ERROR; } -int ha_mcs_impl_select_next(uchar* buf, TABLE* table) +int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone) { THD* thd = current_thd; @@ -5336,7 +5361,7 @@ int ha_mcs_impl_select_next(uchar* buf, TABLE* table) try { - rc = fetchNextRow(buf, ti, ci); + rc = fetchNextRow(buf, ti, ci, timeZone); } catch (std::exception& e) { diff --git a/dbcon/mysql/ha_mcs_impl.h b/dbcon/mysql/ha_mcs_impl.h index a776c783a..dc8da092d 100644 --- a/dbcon/mysql/ha_mcs_impl.h +++ b/dbcon/mysql/ha_mcs_impl.h @@ -29,9 +29,9 @@ extern int ha_mcs_impl_delete_table(const char* name); extern int ha_mcs_impl_analyze(THD* thd, TABLE* table); extern int ha_mcs_impl_open(const char* name, int mode, uint32_t test_if_locked); extern int ha_mcs_impl_close(void); -extern int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table); +extern int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table, long timeZone); extern int ha_mcs_impl_rnd_end(TABLE* table, bool is_derived_hand = false); -extern int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed); +extern int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed, long timeZone); extern void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table, bool is_cache_insert = false); extern int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table); extern int ha_mcs_impl_rename_table(const char* from, const char* to); @@ -45,9 +45,9 @@ extern int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected extern int ha_mcs_impl_delete_row(); extern int ha_mcs_impl_rnd_pos(uchar* buf, uchar* pos); extern int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table); -extern int ha_mcs_impl_select_next(uchar* buf, TABLE* table); +extern int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone); extern int ha_mcs_impl_group_by_init(mcs_handler_info* handler_info, TABLE* table); -extern int ha_mcs_impl_group_by_next(TABLE* table); +extern int ha_mcs_impl_group_by_next(TABLE* table, long timeZone); extern int ha_mcs_impl_group_by_end(TABLE* table); #endif @@ -59,7 +59,8 @@ extern int ha_mcs_impl_group_by_end(TABLE* table); extern int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_impl_if::cal_connection_info& ci); extern int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, ha_rows& rowsInserted); -extern int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci); +extern int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, + long timeZone); extern int ha_mcs_impl_write_last_batch(TABLE* table, cal_impl_if::cal_connection_info& ci, bool abort); extern int ha_mcs_impl_commit_(handlerton* hton, THD* thd, bool all, cal_impl_if::cal_connection_info& ci); extern int ha_mcs_impl_rollback_(handlerton* hton, THD* thd, bool all, cal_impl_if::cal_connection_info& ci); diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index e3ac09932..f3ebd7f42 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -165,6 +165,7 @@ struct gp_walk_info bool cs_vtable_impossible_where_on_union; bool isGroupByHandler; + long timeZone; // MCOL-4617 The below 2 fields are used for in-to-exists // predicate creation and injection. See usage in InSub::transform() @@ -176,7 +177,7 @@ struct gp_walk_info TableOnExprList tableOnExprList; std::vector condList; - gp_walk_info() + gp_walk_info(long timeZone_) : sessionid(0) , fatalParseError(false) , condPush(false) @@ -197,6 +198,7 @@ struct gp_walk_info , cs_vtable_is_update_with_derive(false) , cs_vtable_impossible_where_on_union(false) , isGroupByHandler(false) + , timeZone(timeZone_) , inSubQueryLHS(nullptr) , inSubQueryLHSItem(nullptr) { @@ -391,7 +393,7 @@ const std::string infinidb_err_msg = "distributed syntax or consider changing the MariaDB Columnstore Operating Mode (infinidb_vtable_mode)."; int cp_get_plan(THD* thd, execplan::SCSEP& csep); -int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); +int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti, long timeZone); int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_group_info& gi); int cs_get_derived_plan(ha_columnstore_derived_handler* handler, THD* thd, execplan::SCSEP& csep, gp_walk_info& gwi); @@ -428,14 +430,14 @@ execplan::ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi); execplan::ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& nonSupport); execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, uint32_t pseudoType); -void addIntervalArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); -void castCharArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); -void castDecimalArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); -void castTypeArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); +void addIntervalArgs(gp_walk_info* gwip, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castCharArgs(gp_walk_info* gwip, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castDecimalArgs(gp_walk_info* gwip, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castTypeArgs(gp_walk_info* gwip, Item_func* ifp, funcexp::FunctionParm& functionParms); // void parse_item (Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& // parseInfo); bool isPredicateFunction(Item* item, gp_walk_info* gwip); -execplan::ParseTree* buildRowPredicate(THD* thd, execplan::RowColumn* lhs, execplan::RowColumn* rhs, +execplan::ParseTree* buildRowPredicate(gp_walk_info* gwip, execplan::RowColumn* lhs, execplan::RowColumn* rhs, std::string predicateOp); bool buildRowColumnFilter(gp_walk_info* gwip, execplan::RowColumn* rhs, execplan::RowColumn* lhs, Item_func* ifp); @@ -448,13 +450,13 @@ std::string getViewName(TABLE_LIST* table_ptr); bool buildConstPredicate(Item_func* ifp, execplan::ReturnedColumn* rhs, gp_walk_info* gwip); execplan::CalpontSystemCatalog::ColType fieldType_MysqlToIDB(const Field* field); execplan::CalpontSystemCatalog::ColType colType_MysqlToIDB(const Item* item); -execplan::SPTP getIntervalType(THD* thd, int interval_type); +execplan::SPTP getIntervalType(gp_walk_info* gwip, int interval_type); uint32_t isPseudoColumn(std::string funcName); void setDerivedTable(execplan::ParseTree* n); -execplan::ParseTree* setDerivedFilter(THD* thd, execplan::ParseTree*& n, +execplan::ParseTree* setDerivedFilter(gp_walk_info* gwip, execplan::ParseTree*& n, std::map& obj, execplan::CalpontSelectExecutionPlan::SelectList& derivedTbList); -void derivedTableOptimization(THD* thd, execplan::SCSEP& csep); +void derivedTableOptimization(gp_walk_info* gwip, execplan::SCSEP& csep); bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, execplan::ReturnedColumn* rhs, gp_walk_info* gwip, boost::shared_ptr& sop, const Item_func::Functype& funcType, const std::vector& itemList, bool isInSubs = false); diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index 8ed8c59f3..c65c79f46 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -385,7 +385,10 @@ void partitionByValue_common(UDF_ARGS* args, // inp csc->identity(CalpontSystemCatalog::FE); OID_t oid = csc->lookupOID(tcn); CalpontSystemCatalog::ColType ct = csc->colType(oid); - datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + const char* timeZone = current_thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + datatypes::SessionParam sp(timeZoneOffset); datatypes::SimpleValue startVal; datatypes::SimpleValue endVal; datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; @@ -1252,7 +1255,10 @@ extern "C" string schema, table, column; CalpontSystemCatalog::ColType ct; string errMsg; - datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + const char* timeZone = current_thd->variables.time_zone->get_name()->ptr(); + long timeZoneOffset; + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &timeZoneOffset); + datatypes::SessionParam sp(timeZoneOffset); datatypes::SimpleValue startVal; datatypes::SimpleValue endVal; datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index cefe23986..619caff4c 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -581,6 +581,8 @@ ha_columnstore_derived_handler::ha_columnstore_derived_handler(THD* thd, TABLE_L : derived_handler(thd, mcs_hton) { derived = dt; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &time_zone); } /*********************************************************** @@ -625,7 +627,7 @@ int ha_columnstore_derived_handler::next_row() { DBUG_ENTER("ha_columnstore_derived_handler::next_row"); - int rc = ha_mcs_impl_rnd_next(table->record[0], table); + int rc = ha_mcs_impl_rnd_next(table->record[0], table, time_zone); DBUG_RETURN(rc); } @@ -670,6 +672,8 @@ ha_mcs_group_by_handler::ha_mcs_group_by_handler(THD* thd_arg, Query* query) , order_by(query->order_by) , having(query->having) { + const char* timeZone = thd_arg->variables.time_zone->get_name()->ptr(); + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &time_zone); } /*********************************************************** @@ -705,7 +709,7 @@ int ha_mcs_group_by_handler::init_scan() int ha_mcs_group_by_handler::next_row() { DBUG_ENTER("ha_mcs_group_by_handler::next_row"); - int rc = ha_mcs_impl_group_by_next(table); + int rc = ha_mcs_impl_group_by_next(table, time_zone); DBUG_RETURN(rc); } @@ -985,6 +989,8 @@ ha_columnstore_select_handler::ha_columnstore_select_handler(THD* thd, SELECT_LE , pushdown_init_rc(0) { select = select_lex; + const char* timeZone = thd->variables.time_zone->get_name()->ptr(); + dataconvert::timeZoneToOffset(timeZone, strlen(timeZone), &time_zone); } /*********************************************************** @@ -1028,7 +1034,7 @@ int ha_columnstore_select_handler::next_row() { DBUG_ENTER("ha_columnstore_select_handler::next_row"); - int rc = ha_mcs_impl_select_next(table->record[0], table); + int rc = ha_mcs_impl_select_next(table->record[0], table, time_zone); DBUG_RETURN(rc); } diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index dc127bb2e..80ca3cff8 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -76,6 +76,9 @@ struct mcs_handler_info ***********************************************************/ class ha_mcs_group_by_handler : public group_by_handler { + private: + long time_zone; + public: ha_mcs_group_by_handler(THD* thd_arg, Query* query); ~ha_mcs_group_by_handler(); @@ -109,6 +112,7 @@ class ha_columnstore_derived_handler : public derived_handler { private: COLUMNSTORE_SHARE* share; + long time_zone; public: ha_columnstore_derived_handler(THD* thd_arg, TABLE_LIST* tbl); @@ -138,6 +142,7 @@ class ha_columnstore_select_handler : public select_handler COLUMNSTORE_SHARE* share; bool prepared; bool scan_ended; + long time_zone; public: bool scan_initialized; diff --git a/dbcon/mysql/ha_pseudocolumn.cpp b/dbcon/mysql/ha_pseudocolumn.cpp index fb87a76cc..26fb82264 100644 --- a/dbcon/mysql/ha_pseudocolumn.cpp +++ b/dbcon/mysql/ha_pseudocolumn.cpp @@ -494,7 +494,7 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& cc = new ConstantColumn(localPm); else cc = new ConstantColumn("", ConstantColumn::NULLDATA); - cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + cc->timeZone(gwi.timeZone); cc->alias(ifp->full_name() ? ifp->full_name() : ""); return cc; @@ -556,7 +556,7 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& parms.push_back(sptp); fc->functionParms(parms); fc->expressionId(ci->expressionId++); - fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + fc->timeZone(gwi.timeZone); // string result type CalpontSystemCatalog::ColType ct; diff --git a/dbcon/mysql/ha_scalar_sub.cpp b/dbcon/mysql/ha_scalar_sub.cpp index 4a4f08fca..b12ea4791 100644 --- a/dbcon/mysql/ha_scalar_sub.cpp +++ b/dbcon/mysql/ha_scalar_sub.cpp @@ -109,8 +109,7 @@ execplan::ParseTree* ScalarSub::transform() { fSub = (Item_subselect*)(fFunc->arguments()[0]); fColumn.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); - (dynamic_cast(fColumn.get())) - ->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(fColumn.get()))->timeZone(fGwip.timeZone); delete rhs; return buildParseTree(op); } @@ -176,7 +175,7 @@ execplan::ParseTree* ScalarSub::transform_between() SOP sop; sop.reset(op_LE); rhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op3)); - (dynamic_cast(rhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(rhs->data()))->timeZone(fGwip.timeZone); } SubSelect* sub1 = dynamic_cast(op2); @@ -192,7 +191,7 @@ execplan::ParseTree* ScalarSub::transform_between() SOP sop; sop.reset(op_GE); lhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op2)); - (dynamic_cast(lhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(lhs->data()))->timeZone(fGwip.timeZone); } if (!rhs || !lhs) @@ -245,7 +244,7 @@ execplan::ParseTree* ScalarSub::buildParseTree(PredicateOperator* op) csep->subType(CalpontSelectExecutionPlan::SINGLEROW_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_select_sub.cpp b/dbcon/mysql/ha_select_sub.cpp index 897da428e..2bce37765 100644 --- a/dbcon/mysql/ha_select_sub.cpp +++ b/dbcon/mysql/ha_select_sub.cpp @@ -67,7 +67,7 @@ SCSEP SelectSubQuery::transform() csep->subType(CalpontSelectExecutionPlan::SELECT_SUBS); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fGwip.timeZone); gwi.thd = fGwip.thd; gwi.subQuery = this; diff --git a/dbcon/mysql/ha_view.cpp b/dbcon/mysql/ha_view.cpp index c37907e98..4912fd71d 100644 --- a/dbcon/mysql/ha_view.cpp +++ b/dbcon/mysql/ha_view.cpp @@ -65,7 +65,7 @@ void View::transform() csep->sessionID(fParentGwip->sessionid); // gwi for the sub query - gp_walk_info gwi; + gp_walk_info gwi(fParentGwip->timeZone); gwi.thd = fParentGwip->thd; uint32_t sessionID = csep->sessionID(); diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 8111e14b0..e94372cea 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -145,27 +145,25 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi // put interval val column to bound (dynamic_cast(rc))->functionName(funcName); - (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(rc))->timeZone(gwi.timeZone); sptp.reset(new ParseTree(order->clone())); funcParms.push_back(sptp); sptp.reset(new ParseTree(intervalCol->val()->clone())); funcParms.push_back(sptp); - funcParms.push_back(getIntervalType(gwi.thd, intervalCol->intervalType())); + funcParms.push_back(getIntervalType(&gwi, intervalCol->intervalType())); SRCP srcp(intervalCol->val()); bound.fVal = srcp; if (addOp) { sptp.reset(new ParseTree(new ConstantColumn("ADD"))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } else { sptp.reset(new ParseTree(new ConstantColumn("SUB"))); - (dynamic_cast(sptp->data())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(sptp->data()))->timeZone(gwi.timeZone); funcParms.push_back(sptp); } @@ -187,7 +185,7 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi else aop = new ArithmeticOperator("-"); - aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + aop->timeZone(gwi.timeZone); ParseTree* pt = new ParseTree(aop); ParseTree *lhs = 0, *rhs = 0; lhs = new ParseTree(order->clone()); @@ -314,7 +312,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n Item_sum* item_sum = wf->window_func(); string funcName = ConvertFuncName(item_sum); WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); - ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + ac->timeZone(gwi.timeZone); ac->distinct(item_sum->has_with_distinct()); Window_spec* win_spec = wf->window_spec; SRCP srcp; @@ -407,45 +405,45 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n sprintf(sRespectNulls, "%lu", bRespectNulls); srcp.reset(new ConstantColumn(sRespectNulls, (uint64_t)bRespectNulls, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; } case Item_sum::FIRST_VALUE_FUNC: srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM_FIRST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset( new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; case Item_sum::LAST_VALUE_FUNC: srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("0", (uint64_t)0, ConstantColumn::NUM)); // FROM_LAST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset( new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; case Item_sum::NTH_VALUE_FUNC: // When the front end supports these paramters, this needs modification srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM FIRST/LAST 1 => FIRST - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset( new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; @@ -453,11 +451,11 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case Item_sum::LAG_FUNC: // When the front end supports these paramters, this needs modification srcp.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); // Default to fill in for NULL values - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); srcp.reset( new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT - (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); funcParms.push_back(srcp); break; @@ -802,8 +800,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n bound = 1; srcp.reset(new ConstantColumn((int64_t)bound)); - (dynamic_cast(srcp.get())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); frm.fStart.fVal = srcp; frm.fStart.fBound.reset(buildBoundExp(frm.fStart, srcp, gwi)); @@ -819,8 +816,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n bound = 1; srcp.reset(new ConstantColumn((int64_t)bound)); - (dynamic_cast(srcp.get())) - ->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + (dynamic_cast(srcp.get()))->timeZone(gwi.timeZone); frm.fEnd.fVal = srcp; frm.fEnd.fBound.reset(buildBoundExp(frm.fEnd, srcp, gwi)); diff --git a/ddlproc/ddlprocessor.cpp b/ddlproc/ddlprocessor.cpp index de9f0fc1e..eb6f3d09b 100644 --- a/ddlproc/ddlprocessor.cpp +++ b/ddlproc/ddlprocessor.cpp @@ -541,7 +541,7 @@ class PackageHandler qts.schema_name = alterTableStmt.schemaName(); fQtc.postQueryTele(qts); - processor->fTimeZone = alterTableStmt.fTimeZone; + processor->fTimeZone = alterTableStmt.getTimeZone(); result = processor->processPackage(alterTableStmt); diff --git a/mysql-test/columnstore/basic/r/type_timestamp.result b/mysql-test/columnstore/basic/r/type_timestamp.result new file mode 100644 index 000000000..e56cb083f --- /dev/null +++ b/mysql-test/columnstore/basic/r/type_timestamp.result @@ -0,0 +1,141 @@ +# +# Test cases for the TIMESTAMP datatype +# +DROP DATABASE IF EXISTS timestamp_test; +CREATE DATABASE timestamp_test; +USE timestamp_test; +CREATE TABLE ctimestamp (a timestamp); +SET time_zone='-5:00'; +INSERT INTO ctimestamp VALUES ('2019-01-01 01:02:03'), ('2019-05-05 01:01:01'); +SET time_zone='+1:00'; +SELECT a FROM ctimestamp ORDER BY a; +a +2019-01-01 07:02:03 +2019-05-05 07:01:01 +SET time_zone='-2:00'; +SELECT a FROM ctimestamp ORDER BY a; +a +2019-01-01 04:02:03 +2019-05-05 04:01:01 +CREATE TABLE ctimestamp2 (a timestamp DEFAULT 0); +INSERT INTO ctimestamp2 SELECT * FROM ctimestamp; +SELECT a FROM ctimestamp2 ORDER BY a; +a +2019-01-01 04:02:03 +2019-05-05 04:01:01 +CREATE TABLE ctimestamp3 (a timestamp); +INSERT INTO ctimestamp3 VALUES (19940101), (940101), +(19940101010203), (940101010203), ('1994-01-01T01:02:03'); +SELECT a FROM ctimestamp3 ORDER BY a; +a +1994-01-01 00:00:00 +1994-01-01 00:00:00 +1994-01-01 01:02:03 +1994-01-01 01:02:03 +1994-01-01 01:02:03 +CREATE TABLE ctimestamp4 (a timestamp(6) default 0); +INSERT INTO ctimestamp4 VALUES (0), ('2019-01-01 01:01:01.123456'); +SELECT a, microsecond(a) FROM ctimestamp4 ORDER BY a; +a microsecond(a) +0000-00-00 00:00:00.000000 0 +2019-01-01 01:01:01.123456 123456 +DROP DATABASE IF EXISTS timestamp_test; +CREATE DATABASE timestamp_test; +USE timestamp_test; +CREATE TABLE ctimestamp (a timestamp); +SET time_zone='+0:00'; +INSERT INTO ctimestamp VALUES ('2019-01-02 00:02:03'), +('2019-01-02 01:02:03'), ('2019-01-02 10:11:12'); +SET time_zone='+1:00'; +SELECT a, a BETWEEN '2019-01-02 02:00:00' AND '2019-01-02 13:00:00' +FROM ctimestamp ORDER BY a; +a a BETWEEN '2019-01-02 02:00:00' AND '2019-01-02 13:00:00' +2019-01-02 01:02:03 0 +2019-01-02 02:02:03 1 +2019-01-02 11:11:12 1 +SELECT a, IF(a < '2019-01-02 02:00:00', 'yes', 'no'), +ADDTIME(a, '1:1:1'), STR_TO_DATE(a, '%Y-%m-%d %H:%i:%s'), +EXTRACT(DAY_HOUR FROM a), EXTRACT(MINUTE_SECOND FROM a), +TIME_FORMAT(a, '%H:\%i:\%s'), a RLIKE '02:03', IFNULL(NULL, a), +CASE a WHEN '2019-01-02 01:02:03' THEN 'found' WHEN '2019-01-02 11:11:12' +THEN 'found2' ELSE 'notfound' END, CHAR_LENGTH(a), +CAST(a AS UNSIGNED INT), CAST(a AS CHAR), CAST(a AS DATE), +TIME(CAST(a AS DATETIME)), TIME(COALESCE(NULL, a)), HEX(a), +NULLIF(a, '2019-01-02 01:02:03'), TIMEDIFF(a, '2019-01-01 01:02:03') +FROM ctimestamp ORDER BY a; +a IF(a < '2019-01-02 02:00:00', 'yes', 'no') ADDTIME(a, '1:1:1') STR_TO_DATE(a, '%Y-%m-%d %H:%i:%s') EXTRACT(DAY_HOUR FROM a) EXTRACT(MINUTE_SECOND FROM a) TIME_FORMAT(a, '%H:\%i:\%s') a RLIKE '02:03' IFNULL(NULL, a) CASE a WHEN '2019-01-02 01:02:03' THEN 'found' WHEN '2019-01-02 11:11:12' +THEN 'found2' ELSE 'notfound' END CHAR_LENGTH(a) CAST(a AS UNSIGNED INT) CAST(a AS CHAR) CAST(a AS DATE) TIME(CAST(a AS DATETIME)) TIME(COALESCE(NULL, a)) HEX(a) NULLIF(a, '2019-01-02 01:02:03') TIMEDIFF(a, '2019-01-01 01:02:03') +2019-01-02 01:02:03 yes 2019-01-02 02:03:04 2019-01-02 01:02:03 201 203 01:\02:\03 1 2019-01-02 01:02:03 found 19 20190102010203 2019-01-02 01:02:03 2019-01-02 01:02:03 01:02:03 323031392D30312D30322030313A30323A3033 NULL 24:00:00 +2019-01-02 02:02:03 no 2019-01-02 03:03:04 2019-01-02 02:02:03 202 203 02:\02:\03 1 2019-01-02 02:02:03 notfound 19 20190102020203 2019-01-02 02:02:03 2019-01-02 02:02:03 02:02:03 323031392D30312D30322030323A30323A3033 2019-01-02 02:02:03 25:00:00 +2019-01-02 11:11:12 no 2019-01-02 12:12:13 2019-01-02 11:11:12 211 1112 11:\11:\12 0 2019-01-02 11:11:12 found2 19 20190102111112 2019-01-02 11:11:12 2019-01-02 11:11:12 11:11:12 323031392D30312D30322031313A31313A3132 2019-01-02 11:11:12 34:09:09 +INSERT INTO ctimestamp VALUES ('2020-01-03 12:12:12'), +('2020-05-06 12:12:12'), ('2020-10-28 12:12:12'); +SELECT a, DAYNAME(a), DAYOFWEEK(a), DATE_FORMAT(a, '%W %M %Y'), +MONTHNAME(a), DATE(a), YEARWEEK(a), DAYOFYEAR(a), YEAR(a), +a + INTERVAL 1 DAY, TIMESTAMPDIFF(DAY, a, '2020-01-01'), +LAST_DAY(a), TRUNCATE(a, -2), a IN ('2019-01-02 01:02:03', a), +TO_DAYS(a), DAY(a), WEEK(a), WEEKDAY(a), GREATEST(a, '2020-07-01'), +MONTH(a), QUARTER(a), DATE_ADD(a, INTERVAL 1 SECOND) +FROM ctimestamp WHERE a > '2020-01-01' ORDER BY a; +a DAYNAME(a) DAYOFWEEK(a) DATE_FORMAT(a, '%W %M %Y') MONTHNAME(a) DATE(a) YEARWEEK(a) DAYOFYEAR(a) YEAR(a) a + INTERVAL 1 DAY TIMESTAMPDIFF(DAY, a, '2020-01-01') LAST_DAY(a) TRUNCATE(a, -2) a IN ('2019-01-02 01:02:03', a) TO_DAYS(a) DAY(a) WEEK(a) WEEKDAY(a) GREATEST(a, '2020-07-01') MONTH(a) QUARTER(a) DATE_ADD(a, INTERVAL 1 SECOND) +2020-01-03 12:12:12 Friday 6 Friday January 2020 January 2020-01-03 201952 3 2020 2020-01-04 12:12:12 -2 2020-01-31 2020-01-03 12:12:12 1 737792 3 0 4 2020-07-01 00:00:00 1 1 2020-01-03 12:12:13 +2020-05-06 12:12:12 Wednesday 4 Wednesday May 2020 May 2020-05-06 202018 127 2020 2020-05-07 12:12:12 -126 2020-05-31 2020-05-06 12:12:12 1 737916 6 18 2 2020-07-01 00:00:00 5 2 2020-05-06 12:12:13 +2020-10-28 12:12:12 Wednesday 4 Wednesday October 2020 October 2020-10-28 202043 302 2020 2020-10-29 12:12:12 -301 2020-10-31 2020-10-28 12:12:12 1 738091 28 43 2 2020-10-28 12:12:12 10 4 2020-10-28 12:12:13 +SELECT UNIX_TIMESTAMP(a), TIME_TO_SEC(a), CEIL(a), +CAST(LEAST(a, '2019-03-03 00:00:00') AS DATETIME), +ROUND(a), SECOND(a), MINUTE(a), HOUR(a), FLOOR(a) +FROM ctimestamp ORDER BY a; +UNIX_TIMESTAMP(a) TIME_TO_SEC(a) CEIL(a) CAST(LEAST(a, '2019-03-03 00:00:00') AS DATETIME) ROUND(a) SECOND(a) MINUTE(a) HOUR(a) FLOOR(a) +1546387323 3723 2019-01-02 01:02:03 2019-01-02 01:02:03 2019-01-02 01:02:03 3 2 1 2019-01-02 01:02:03 +1546390923 7323 2019-01-02 02:02:03 2019-01-02 02:02:03 2019-01-02 02:02:03 3 2 2 2019-01-02 02:02:03 +1546423872 40272 2019-01-02 11:11:12 2019-01-02 11:11:12 2019-01-02 11:11:12 12 11 11 2019-01-02 11:11:12 +1578049932 43932 2020-01-03 12:12:12 2019-03-03 00:00:00 2020-01-03 12:12:12 12 12 12 2020-01-03 12:12:12 +1588763532 43932 2020-05-06 12:12:12 2019-03-03 00:00:00 2020-05-06 12:12:12 12 12 12 2020-05-06 12:12:12 +1603883532 43932 2020-10-28 12:12:12 2019-03-03 00:00:00 2020-10-28 12:12:12 12 12 12 2020-10-28 12:12:12 +DROP DATABASE IF EXISTS timestamp_test; +CREATE DATABASE timestamp_test; +USE timestamp_test; +CREATE TABLE ctimestamp (a timestamp, b int); +SET time_zone='+0:00'; +INSERT INTO ctimestamp VALUES ('2019-01-03 12:12:12', 1), +('2019-01-04 12:12:12', 2), ('2019-01-03 12:12:12', 4), +('2019-01-03 12:12:12', 2), ('2019-01-04 12:12:12', 1); +SELECT a, b, SUM(b) over (PARTITION BY a ORDER BY a, b +ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) c +FROM ctimestamp; +a b c +2019-01-03 12:12:12 1 1 +2019-01-03 12:12:12 2 3 +2019-01-03 12:12:12 4 7 +2019-01-04 12:12:12 1 1 +2019-01-04 12:12:12 2 3 +SELECT a, b, MAX(a) over (PARTITION BY b ORDER BY a desc +ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) c +FROM ctimestamp; +a b c +2019-01-04 12:12:12 1 2019-01-04 12:12:12 +2019-01-03 12:12:12 1 2019-01-04 12:12:12 +2019-01-04 12:12:12 2 2019-01-04 12:12:12 +2019-01-03 12:12:12 2 2019-01-04 12:12:12 +2019-01-03 12:12:12 4 2019-01-03 12:12:12 +DROP DATABASE IF EXISTS timestamp_test; +CREATE DATABASE timestamp_test; +USE timestamp_test; +CREATE TABLE ctimestamp (a int, b timestamp); +INSERT INTO ctimestamp VALUES (1, 20190101), (1, 20200202), +(2, 20190202), (2, 20200202), (2, 20190101); +SELECT b, count(*) FROM ctimestamp GROUP BY b ORDER BY b; +b count(*) +2019-01-01 00:00:00 2 +2019-02-02 00:00:00 1 +2020-02-02 00:00:00 2 +SELECT b, max(a), min(a) FROM ctimestamp GROUP BY b ORDER BY b; +b max(a) min(a) +2019-01-01 00:00:00 2 1 +2019-02-02 00:00:00 2 2 +2020-02-02 00:00:00 2 1 +SELECT a, max(b), min(b) FROM ctimestamp GROUP BY a ORDER BY a; +a max(b) min(b) +1 2020-02-02 00:00:00 2019-01-01 00:00:00 +2 2020-02-02 00:00:00 2019-01-01 00:00:00 +DROP DATABASE timestamp_test; diff --git a/mysql-test/columnstore/basic/t/type_timestamp.test b/mysql-test/columnstore/basic/t/type_timestamp.test new file mode 100644 index 000000000..2789f6e7c --- /dev/null +++ b/mysql-test/columnstore/basic/t/type_timestamp.test @@ -0,0 +1,131 @@ +--source ../include/have_columnstore.inc +--source ../include/combinations.myisam-columnstore.inc + +--echo # +--echo # Test cases for the TIMESTAMP datatype +--echo # + +# Test bulk insert/literals/microseconds +--disable_warnings +DROP DATABASE IF EXISTS timestamp_test; +--enable_warnings + +CREATE DATABASE timestamp_test; +USE timestamp_test; + +## Test the effect of changing timezones on timestamp values +CREATE TABLE ctimestamp (a timestamp); +SET time_zone='-5:00'; +INSERT INTO ctimestamp VALUES ('2019-01-01 01:02:03'), ('2019-05-05 01:01:01'); +SET time_zone='+1:00'; +SELECT a FROM ctimestamp ORDER BY a; +SET time_zone='-2:00'; +SELECT a FROM ctimestamp ORDER BY a; + +## Test bulk insert using cpimport +CREATE TABLE ctimestamp2 (a timestamp DEFAULT 0); +INSERT INTO ctimestamp2 SELECT * FROM ctimestamp; +SELECT a FROM ctimestamp2 ORDER BY a; + +## Test literals +CREATE TABLE ctimestamp3 (a timestamp); +INSERT INTO ctimestamp3 VALUES (19940101), (940101), +(19940101010203), (940101010203), ('1994-01-01T01:02:03'); +SELECT a FROM ctimestamp3 ORDER BY a; + +## Test microseconds +CREATE TABLE ctimestamp4 (a timestamp(6) default 0); +INSERT INTO ctimestamp4 VALUES (0), ('2019-01-01 01:01:01.123456'); +SELECT a, microsecond(a) FROM ctimestamp4 ORDER BY a; + +# Test distributed functions +--disable_warnings +DROP DATABASE IF EXISTS timestamp_test; +--enable_warnings + +CREATE DATABASE timestamp_test; +USE timestamp_test; + +CREATE TABLE ctimestamp (a timestamp); +SET time_zone='+0:00'; +INSERT INTO ctimestamp VALUES ('2019-01-02 00:02:03'), +('2019-01-02 01:02:03'), ('2019-01-02 10:11:12'); +SET time_zone='+1:00'; + +SELECT a, a BETWEEN '2019-01-02 02:00:00' AND '2019-01-02 13:00:00' +FROM ctimestamp ORDER BY a; + +SELECT a, IF(a < '2019-01-02 02:00:00', 'yes', 'no'), +ADDTIME(a, '1:1:1'), STR_TO_DATE(a, '%Y-%m-%d %H:%i:%s'), +EXTRACT(DAY_HOUR FROM a), EXTRACT(MINUTE_SECOND FROM a), +TIME_FORMAT(a, '%H:\%i:\%s'), a RLIKE '02:03', IFNULL(NULL, a), +CASE a WHEN '2019-01-02 01:02:03' THEN 'found' WHEN '2019-01-02 11:11:12' +THEN 'found2' ELSE 'notfound' END, CHAR_LENGTH(a), +CAST(a AS UNSIGNED INT), CAST(a AS CHAR), CAST(a AS DATE), +TIME(CAST(a AS DATETIME)), TIME(COALESCE(NULL, a)), HEX(a), +NULLIF(a, '2019-01-02 01:02:03'), TIMEDIFF(a, '2019-01-01 01:02:03') +FROM ctimestamp ORDER BY a; + +INSERT INTO ctimestamp VALUES ('2020-01-03 12:12:12'), +('2020-05-06 12:12:12'), ('2020-10-28 12:12:12'); + +SELECT a, DAYNAME(a), DAYOFWEEK(a), DATE_FORMAT(a, '%W %M %Y'), +MONTHNAME(a), DATE(a), YEARWEEK(a), DAYOFYEAR(a), YEAR(a), +a + INTERVAL 1 DAY, TIMESTAMPDIFF(DAY, a, '2020-01-01'), +LAST_DAY(a), TRUNCATE(a, -2), a IN ('2019-01-02 01:02:03', a), +TO_DAYS(a), DAY(a), WEEK(a), WEEKDAY(a), GREATEST(a, '2020-07-01'), +MONTH(a), QUARTER(a), DATE_ADD(a, INTERVAL 1 SECOND) +FROM ctimestamp WHERE a > '2020-01-01' ORDER BY a; + +SELECT UNIX_TIMESTAMP(a), TIME_TO_SEC(a), CEIL(a), +CAST(LEAST(a, '2019-03-03 00:00:00') AS DATETIME), +ROUND(a), SECOND(a), MINUTE(a), HOUR(a), FLOOR(a) +FROM ctimestamp ORDER BY a; + +# Test window functions +--disable_warnings +DROP DATABASE IF EXISTS timestamp_test; +--enable_warnings + +CREATE DATABASE timestamp_test; +USE timestamp_test; + +CREATE TABLE ctimestamp (a timestamp, b int); +SET time_zone='+0:00'; +INSERT INTO ctimestamp VALUES ('2019-01-03 12:12:12', 1), +('2019-01-04 12:12:12', 2), ('2019-01-03 12:12:12', 4), +('2019-01-03 12:12:12', 2), ('2019-01-04 12:12:12', 1); + +## Test SUM + +SELECT a, b, SUM(b) over (PARTITION BY a ORDER BY a, b +ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) c +FROM ctimestamp; + +## Test MAX + +SELECT a, b, MAX(a) over (PARTITION BY b ORDER BY a desc +ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) c +FROM ctimestamp; + +# Test aggregate functions +--disable_warnings +DROP DATABASE IF EXISTS timestamp_test; +--enable_warnings + +CREATE DATABASE timestamp_test; +USE timestamp_test; + +CREATE TABLE ctimestamp (a int, b timestamp); +INSERT INTO ctimestamp VALUES (1, 20190101), (1, 20200202), +(2, 20190202), (2, 20200202), (2, 20190101); + +# Test count(*) +SELECT b, count(*) FROM ctimestamp GROUP BY b ORDER BY b; + +# Test max/min +SELECT b, max(a), min(a) FROM ctimestamp GROUP BY b ORDER BY b; +SELECT a, max(b), min(b) FROM ctimestamp GROUP BY a ORDER BY a; + +# Cleanup +DROP DATABASE timestamp_test; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 5d5e9d346..141954333 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1193,7 +1193,7 @@ bool stringToTimeStruct(const string& data, Time& dtime, long decimals) return true; } -bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const string& timeZone) +bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, long timeZone) { // special handling for 0000-00-00 00:00:00 // "0" is sent by the server when checking for default value @@ -1879,10 +1879,11 @@ int64_t DataConvert::convertColumnDatetime(const char* dataOrg, CalpontDateTimeF // Most of this code is taken from DataConvert::convertColumnDatetime //------------------------------------------------------------------------------ int64_t DataConvert::convertColumnTimestamp(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, - int& status, unsigned int dataOrgLen, const std::string& timeZone) + int& status, unsigned int dataOrgLen, long timeZone) { char tmbuf[64]; std::string dataOrgTemp = dataOrg; + status = 0; if (dataOrgTemp.substr(0, 19) == "0000-00-00 00:00:00") { return 0; @@ -1902,7 +1903,6 @@ int64_t DataConvert::convertColumnTimestamp(const char* dataOrg, CalpontDateTime dataOrgLen = strlen(tmbuf); } - status = 0; const char* p; p = dataOrg; char fld[10]; @@ -2258,8 +2258,7 @@ std::string DataConvert::datetimeToString(long long datetimevalue, long decimals return buf; } -std::string DataConvert::timestampToString(long long timestampvalue, const std::string& timezone, - long decimals) +std::string DataConvert::timestampToString(long long timestampvalue, long timezone, long decimals) { // 10 is default which means we don't need microseconds if (decimals > 6 || decimals < 0) @@ -2343,7 +2342,7 @@ std::string DataConvert::datetimeToString1(long long datetimevalue) return buf; } -std::string DataConvert::timestampToString1(long long timestampvalue, const std::string& timezone) +std::string DataConvert::timestampToString1(long long timestampvalue, long timezone) { const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 char buf[TIMESTAMPTOSTRING1_LEN]; @@ -2381,7 +2380,7 @@ int64_t DataConvert::datetimeToInt(const string& datetime) return stringToDatetime(datetime); } -int64_t DataConvert::timestampToInt(const string& timestamp, const string& timeZone) +int64_t DataConvert::timestampToInt(const string& timestamp, long timeZone) { return stringToTimestamp(timestamp, timeZone); } @@ -2414,7 +2413,7 @@ int64_t DataConvert::stringToDatetime(const string& data, bool* date) return -1; } -int64_t DataConvert::stringToTimestamp(const string& data, const string& timeZone) +int64_t DataConvert::stringToTimestamp(const string& data, long timeZone) { TimeStamp aTimestamp; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 1df077670..5653f59a7 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -289,6 +289,14 @@ inline void unserializeTimezoneInfo(messageqcpp::ByteStream& bs, TIME_ZONE_INFO* bs >> (uint&)tz->revcnt; }; +inline long systemTimeZoneOffset() +{ + time_t t = time(NULL); + struct tm lt; + localtime_r(&t, <); + return lt.tm_gmtoff; +} + /** * This function converts the timezone represented as a string * in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds @@ -296,20 +304,32 @@ inline void unserializeTimezoneInfo(messageqcpp::ByteStream& bs, TIME_ZONE_INFO* */ inline bool timeZoneToOffset(const char* str, std::string::size_type length, long* offset) { + if (strcmp(str, "SYSTEM") == 0) + { + *offset = systemTimeZoneOffset(); + return 0; + } + const char* end = str + length; bool negative; unsigned long number_tmp; long offset_tmp; if (length < 4) + { + *offset = 0; return 1; + } if (*str == '+') negative = 0; else if (*str == '-') negative = 1; else + { + *offset = 0; return 1; + } str++; number_tmp = 0; @@ -321,7 +341,10 @@ inline bool timeZoneToOffset(const char* str, std::string::size_type length, lon } if (str + 1 >= end || *str != ':') + { + *offset = 0; return 1; + } str++; offset_tmp = number_tmp * 60L; @@ -334,7 +357,10 @@ inline bool timeZoneToOffset(const char* str, std::string::size_type length, lon } if (str != end) + { + *offset = 0; return 1; + } offset_tmp = (offset_tmp + number_tmp) * 60L; @@ -347,10 +373,12 @@ inline bool timeZoneToOffset(const char* str, std::string::size_type length, lon */ if (number_tmp > 59 || offset_tmp < -13 * 3600L + 1 || offset_tmp > 13 * 3600L) + { + *offset = 0; return 1; + } *offset = offset_tmp; - return 0; } @@ -483,10 +511,10 @@ inline bool isTimestampValid(uint64_t second, uint64_t microsecond) * * @param seconds the value to be converted * @param time the broken-down representation of the timestamp - * @param timeZone a string with the server timezone of the machine - * which initiated the query + @param offset a timeZone offset (in seconds) relative to UTC. + For example, for EST which is UTC-5:00, offset will be -18000s. */ -inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, const std::string& timeZone) +inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, long offset) { if (seconds == 0) { @@ -494,78 +522,52 @@ inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, const std::strin return; } - if (timeZone == "SYSTEM") + int64_t days; + int32_t rem; + int32_t y; + int32_t yleap; + const unsigned int* ip; + + days = (int64_t)(seconds / SECS_PER_DAY); + rem = (int32_t)(seconds % SECS_PER_DAY); + + rem += offset; + while (rem < 0) { - struct tm tmp_tm; - time_t tmp_t = (time_t)seconds; - localtime_r(&tmp_t, &tmp_tm); - time.second_part = 0; - time.year = (int)((tmp_tm.tm_year + 1900) % 10000); - time.month = (int)tmp_tm.tm_mon + 1; - time.day = (int)tmp_tm.tm_mday; - time.hour = (int)tmp_tm.tm_hour; - time.minute = (int)tmp_tm.tm_min; - time.second = (int)tmp_tm.tm_sec; - time.time_type = CALPONTDATETIME_ENUM; - if (time.second == 60 || time.second == 61) - time.second = 59; + rem += SECS_PER_DAY; + days--; } - else + while (rem >= SECS_PER_DAY) { - long offset; - if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) - { - time.reset(); - return; - } - - int64_t days; - int32_t rem; - int32_t y; - int32_t yleap; - const unsigned int* ip; - - days = (int64_t)(seconds / SECS_PER_DAY); - rem = (int32_t)(seconds % SECS_PER_DAY); - - rem += offset; - while (rem < 0) - { - rem += SECS_PER_DAY; - days--; - } - while (rem >= SECS_PER_DAY) - { - rem -= SECS_PER_DAY; - days++; - } - time.hour = (unsigned int)(rem / SECS_PER_HOUR); - rem = rem % SECS_PER_HOUR; - time.minute = (unsigned int)(rem / SECS_PER_MIN); - time.second = (unsigned int)(rem % SECS_PER_MIN); - - y = EPOCH_YEAR; - while (days < 0 || days >= (int64_t)(year_lengths[yleap = isLeapYear(y)])) - { - int32_t newy; - - newy = y + days / DAYS_PER_NYEAR; - if (days < 0) - newy--; - days -= (newy - y) * DAYS_PER_NYEAR + leapsThruEndOf(newy - 1) - leapsThruEndOf(y - 1); - y = newy; - } - time.year = y; - - ip = mon_lengths[yleap]; - for (time.month = 0; days >= (int64_t)ip[time.month]; time.month++) - days -= (int64_t)ip[time.month]; - time.month++; - time.day = (unsigned int)(days + 1); - - time.second_part = 0; - time.time_type = CALPONTDATETIME_ENUM; + rem -= SECS_PER_DAY; + days++; } + time.hour = (unsigned int)(rem / SECS_PER_HOUR); + rem = rem % SECS_PER_HOUR; + time.minute = (unsigned int)(rem / SECS_PER_MIN); + time.second = (unsigned int)(rem % SECS_PER_MIN); + + y = EPOCH_YEAR; + while (days < 0 || days >= (int64_t)(year_lengths[yleap = isLeapYear(y)])) + { + int32_t newy; + + newy = y + days / DAYS_PER_NYEAR; + if (days < 0) + newy--; + days -= (newy - y) * DAYS_PER_NYEAR + leapsThruEndOf(newy - 1) - leapsThruEndOf(y - 1); + y = newy; + } + time.year = y; + + ip = mon_lengths[yleap]; + for (time.month = 0; days >= (int64_t)ip[time.month]; time.month++) + days -= (int64_t)ip[time.month]; + time.month++; + time.day = (unsigned int)(days + 1); + + time.second_part = 0; + time.time_type = CALPONTDATETIME_ENUM; } /** @@ -593,41 +595,15 @@ inline int64_t secSinceEpoch(int year, int month, int day, int hour, int min, in return ((days * HOURS_PER_DAY + hour) * MINS_PER_HOUR + min) * SECS_PER_MIN + sec; } -// This is duplicate of funchelpers.h:calc_mysql_daynr, -// with one additional function parameter -inline uint32_t calc_mysql_daynr(uint32_t year, uint32_t month, uint32_t day, bool& isValid) -{ - int temp; - int y = year; - long delsum; - - if (!isDateValid(day, month, year)) - { - isValid = false; - return 0; - } - - delsum = (long)(365 * y + 31 * ((int)month - 1) + (int)day); - - if (month <= 2) - y--; - else - delsum -= (long)((int)month * 4 + 23) / 10; - - temp = (int)((y / 100 + 1) * 3) / 4; - - return delsum + (int)y / 4 - temp; -} - /** * @brief converts a timestamp from broken-down representation * to seconds since UTC epoch * * @param time the broken-down representation of the timestamp - @param timeZone a string with the server timezone of the machine - which initiated the query + @param offset a timeZone offset (in seconds) relative to UTC. + For example, for EST which is UTC-5:00, offset will be -18000s. */ -inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, const std::string& timeZone, bool& isValid) +inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, long offset, bool& isValid) { int64_t seconds; @@ -637,88 +613,7 @@ inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, const std::string& timeZ return 0; } - if (timeZone == "SYSTEM") - { - // This is mirror of code in func_unix_timestamp.cpp - uint32_t loop; - time_t tmp_t = 0; - int shift = 0; - struct tm *l_time, tm_tmp; - int64_t diff; - localtime_r(&tmp_t, &tm_tmp); - // Get the system timezone offset at 0 seconds since epoch - int64_t my_time_zone = tm_tmp.tm_gmtoff; - int day = time.day; - - if ((time.year == MAX_TIMESTAMP_YEAR) && (time.month == 1) && (day > 4)) - { - day -= 2; - shift = 2; - } - - tmp_t = (time_t)(((calc_mysql_daynr(time.year, time.month, day, isValid) - 719528) * 86400L + - (int64_t)time.hour * 3600L + (int64_t)(time.minute * 60 + time.second)) - - (time_t)my_time_zone); - if (!isValid) - return 0; - - localtime_r(&tmp_t, &tm_tmp); - l_time = &tm_tmp; - - for (loop = 0; - loop < 2 && (time.hour != (uint32_t)l_time->tm_hour || time.minute != (uint32_t)l_time->tm_min || - time.second != (uint32_t)l_time->tm_sec); - loop++) - { - int days = day - l_time->tm_mday; - - if (days < -1) - days = 1; /* Month has wrapped */ - else if (days > 1) - days = -1; - - diff = (3600L * (int64_t)(days * 24 + ((int)time.hour - (int)l_time->tm_hour)) + - (int64_t)(60 * ((int)time.minute - (int)l_time->tm_min)) + - (int64_t)((int)time.second - (int)l_time->tm_sec)); - tmp_t += (time_t)diff; - localtime_r(&tmp_t, &tm_tmp); - l_time = &tm_tmp; - } - - if (loop == 2 && time.hour != (uint32_t)l_time->tm_hour) - { - int days = day - l_time->tm_mday; - - if (days < -1) - days = 1; /* Month has wrapped */ - else if (days > 1) - days = -1; - - diff = (3600L * (int64_t)(days * 24 + ((int)time.hour - (int)l_time->tm_hour)) + - (int64_t)(60 * ((int)time.minute - (int)l_time->tm_min)) + - (int64_t)((int)time.second - (int)l_time->tm_sec)); - - if (diff == 3600) - tmp_t += 3600 - time.minute * 60 - time.second; /* Move to next hour */ - else if (diff == -3600) - tmp_t -= time.minute * 60 + time.second; /* Move to previous hour */ - } - - /* shift back, if we were dealing with boundary dates */ - tmp_t += shift * 86400L; - - seconds = (int64_t)tmp_t; - } - else - { - long offset; - if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) - { - isValid = false; - return -1; - } - seconds = secSinceEpoch(time.year, time.month, time.day, time.hour, time.minute, time.second) - offset; - } + seconds = secSinceEpoch(time.year, time.month, time.day, time.hour, time.minute, time.second) - offset; /* make sure we have legit timestamps (i.e. we didn't over/underflow anywhere above) */ if (seconds >= MIN_TIMESTAMP_VALUE && seconds <= MAX_TIMESTAMP_VALUE) return seconds; @@ -1151,11 +1046,11 @@ struct TimeStamp { } - int64_t convertToMySQLint(const std::string& timeZone) const; + int64_t convertToMySQLint(long timeZone) const; void reset(); }; -inline int64_t TimeStamp::convertToMySQLint(const std::string& timeZone) const +inline int64_t TimeStamp::convertToMySQLint(long timeZone) const { const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 char buf[TIMESTAMPTOSTRING1_LEN]; @@ -1262,10 +1157,9 @@ class DataConvert * @param type the columns database type * @param data the columns string representation of it's data */ - EXPORT static std::string timestampToString(long long timestampvalue, const std::string& timezone, - long decimals = 0); + EXPORT static std::string timestampToString(long long timestampvalue, long timezone, long decimals = 0); static inline void timestampToString(long long timestampvalue, char* buf, unsigned int buflen, - const std::string& timezone, long decimals = 0); + long timezone, long decimals = 0); /** * @brief convert a columns data from native format to a string @@ -1300,9 +1194,9 @@ class DataConvert * @param type the columns database type * @param data the columns string representation of it's data */ - EXPORT static std::string timestampToString1(long long timestampvalue, const std::string& timezone); + EXPORT static std::string timestampToString1(long long timestampvalue, long timezone); static inline void timestampToString1(long long timestampvalue, char* buf, unsigned int buflen, - const std::string& timezone); + long timezone); /** * @brief convert a columns data from native format to a string @@ -1352,11 +1246,11 @@ class DataConvert * @param datetimeFormat the format the date value in * @param status 0 - success, -1 - fail * @param dataOrgLen length specification of dataOrg - * @param timeZone the timezone used for conversion to native format + * @param timeZone an offset (in seconds) relative to UTC. + For example, for EST which is UTC-5:00, offset will be -18000s. */ EXPORT static int64_t convertColumnTimestamp(const char* dataOrg, CalpontDateTimeFormat datetimeFormat, - int& status, unsigned int dataOrgLen, - const std::string& timeZone); + int& status, unsigned int dataOrgLen, long timeZone); /** * @brief convert a time column data, represented as a string, @@ -1385,7 +1279,7 @@ class DataConvert // convert string to datetime EXPORT static int64_t stringToDatetime(const std::string& data, bool* isDate = NULL); // convert string to timestamp - EXPORT static int64_t stringToTimestamp(const std::string& data, const std::string& timeZone); + EXPORT static int64_t stringToTimestamp(const std::string& data, long timeZone); // convert integer to date EXPORT static int64_t intToDate(int64_t data); // convert integer to datetime @@ -1396,7 +1290,7 @@ class DataConvert EXPORT static int64_t dateToInt(const std::string& date); // convert string to datetime. alias to datetimeToInt EXPORT static int64_t datetimeToInt(const std::string& datetime); - EXPORT static int64_t timestampToInt(const std::string& timestamp, const std::string& timeZone); + EXPORT static int64_t timestampToInt(const std::string& timestamp, long timeZone); EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime(const std::string& data); // bug4388, union type conversion @@ -1467,7 +1361,7 @@ inline void DataConvert::datetimeToString(long long datetimevalue, char* buf, un } inline void DataConvert::timestampToString(long long timestampvalue, char* buf, unsigned int buflen, - const std::string& timezone, long decimals) + long timezone, long decimals) { // 10 is default which means we don't need microseconds if (decimals > 6 || decimals < 0) @@ -1545,7 +1439,7 @@ inline void DataConvert::datetimeToString1(long long datetimevalue, char* buf, u } inline void DataConvert::timestampToString1(long long timestampvalue, char* buf, unsigned int buflen, - const std::string& timezone) + long timezone) { TimeStamp timestamp(timestampvalue); int64_t seconds = timestamp.second; diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index 09b1d2d0c..5a2506b3f 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -171,7 +171,7 @@ int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm TimeStamp timestamp(val1); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); dt1.year = m_time.year; dt1.month = m_time.month; dt1.day = m_time.day; diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index 3bd2b907b..3ce66eb60 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -328,7 +328,7 @@ CalpontSystemCatalog::ColType Func_between::operationType(FunctionParm& fp, if (cc) { Result result = cc->result(); - result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, timeZone()); + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, resultType.getTimeZone()); cc->result(result); } } diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 76d09b9bd..c10030b98 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -105,7 +105,8 @@ static datatypes::TUInt64Null DecimalToBitOperand(Row& row, const execplan::SPTP // and could be extracted into a utility class with its own header // if that is the case - this is left as future exercise datatypes::TUInt64Null GenericToBitOperand(Row& row, const execplan::SPTP& parm, - const funcexp::Func& thisFunc, bool temporalRounding) + const funcexp::Func& thisFunc, bool temporalRounding, + long timeZone) { switch (parm->data()->resultType().colDataType) { @@ -186,7 +187,7 @@ datatypes::TUInt64Null GenericToBitOperand(Row& row, const execplan::SPTP& parm, return datatypes::TUInt64Null(); TimeStamp dt(time); - int64_t value = dt.convertToMySQLint(thisFunc.timeZone()); + int64_t value = dt.convertToMySQLint(timeZone); if (temporalRounding && dt.msecond >= 500000) value++; return datatypes::TUInt64Null((uint64_t)value); @@ -222,8 +223,8 @@ class BitOperandGeneric : public datatypes::TUInt64Null BitOperandGeneric() { } - BitOperandGeneric(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) - : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, true)) + BitOperandGeneric(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, long timeZone) + : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, true, timeZone)) { } }; @@ -236,8 +237,9 @@ class BitOperandGenericShiftAmount : public datatypes::TUInt64Null BitOperandGenericShiftAmount() { } - BitOperandGenericShiftAmount(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) - : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, false)) + BitOperandGenericShiftAmount(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, + long timeZone) + : TUInt64Null(GenericToBitOperand(row, parm, thisFunc, false, timeZone)) { } }; @@ -327,7 +329,7 @@ class Func_bitand_return_uint64 : public Func_bitand CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Lazy args(row, parm, *this); + Arg2Lazy args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)(args.a & args.b).nullSafeValue(isNull); } }; @@ -353,7 +355,7 @@ class Func_leftshift_return_uint64 : public Func_leftshift CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); + Arg2Eager args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)args.a.MariaDBShiftLeft(args.b).nullSafeValue(isNull); } }; @@ -378,7 +380,7 @@ class Func_rightshift_return_uint64 : public Func_rightshift CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); + Arg2Eager args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)args.a.MariaDBShiftRight(args.b).nullSafeValue(isNull); } }; @@ -409,7 +411,7 @@ class Func_bitor_return_uint64 : public Func_bitor CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Lazy args(row, parm, *this); + Arg2Lazy args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)(args.a | args.b).nullSafeValue(isNull); } }; @@ -435,7 +437,7 @@ class Func_bitxor_return_uint64 : public Func_bitxor CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 2); - Arg2Eager args(row, parm, *this); + Arg2Eager args(row, parm, *this, operationColType.getTimeZone()); return (int64_t)(args.a ^ args.b).nullSafeValue(isNull); } }; @@ -475,7 +477,7 @@ class Func_bit_count_return_uint64 : public Func_bit_count CalpontSystemCatalog::ColType& operationColType) override { idbassert(parm.size() == 1); - return bitCount((uint64_t)TA(row, parm[0], *this).nullSafeValue(isNull)); + return bitCount((uint64_t)TA(row, parm[0], *this, operationColType.getTimeZone()).nullSafeValue(isNull)); } }; diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index b3917b2a5..4b224c6d9 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -186,7 +186,7 @@ int64_t Func_cast_signed::getIntVal(Row& row, FunctionParm& parm, bool& isNull, int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); TimeStamp dt(time); - return dt.convertToMySQLint(timeZone()); + return dt.convertToMySQLint(operationColType.getTimeZone()); } break; @@ -304,7 +304,7 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, FunctionParm& parm, bool& isNu int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); TimeStamp dt(time); - return dt.convertToMySQLint(timeZone()); + return dt.convertToMySQLint(operationColType.getTimeZone()); } break; @@ -433,7 +433,7 @@ string Func_cast_char::getStrVal(Row& row, FunctionParm& parm, bool& isNull, case execplan::CalpontSystemCatalog::TIMESTAMP: { return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), - timeZone()) + operationColType.getTimeZone()) .substr(0, length); } break; @@ -568,7 +568,7 @@ int32_t Func_cast_date::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bo case execplan::CalpontSystemCatalog::TIMESTAMP: { int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - string value = dataconvert::DataConvert::timestampToString(val1, timeZone()); + string value = dataconvert::DataConvert::timestampToString(val1, op_ct.getTimeZone()); value = value.substr(0, 10); return dataconvert::DataConvert::stringToDate(value); } @@ -691,7 +691,7 @@ int64_t Func_cast_date::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, operationColType.getTimeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -847,7 +847,7 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, operationColType.getTimeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -958,7 +958,7 @@ int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, operationColType.getTimeZone()); Time time; time.hour = m_time.hour; time.minute = m_time.minute; @@ -1362,7 +1362,7 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, FunctionParm& parm, bool& int32_t s = 0; string value = dataconvert::DataConvert::timestampToString1( - parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + parm[0]->data()->getTimestampIntVal(row, isNull), operationColType.getTimeZone()); // strip off micro seconds string date = value.substr(0, 14); @@ -1475,8 +1475,8 @@ double Func_cast_double::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull case execplan::CalpontSystemCatalog::TIMESTAMP: { - string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + string str = DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), + operationColType.getTimeZone()); // strip off micro seconds str = str.substr(0, 14); diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index f0c016847..dc559a5db 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -571,7 +571,7 @@ IDB_Decimal Func_ceil::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); if (!isNull) - ret.value = dt.convertToMySQLint(timeZone()); + ret.value = dt.convertToMySQLint(op_ct.getTimeZone()); } break; diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index 749914188..553d54399 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -99,7 +99,7 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool case execplan::CalpontSystemCatalog::TIMESTAMP: { string date = dataconvert::DataConvert::timestampToString( - parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + parm[0]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); return (int64_t)date.size(); } diff --git a/utils/funcexp/func_convert_tz.cpp b/utils/funcexp/func_convert_tz.cpp index 424b7494c..03efdad88 100644 --- a/utils/funcexp/func_convert_tz.cpp +++ b/utils/funcexp/func_convert_tz.cpp @@ -154,7 +154,9 @@ int64_t Func_convert_tz::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } else { - seconds = dataconvert::mySQLTimeToGmtSec(my_start_time, from_tz, valid); + long from_tz_offset; + dataconvert::timeZoneToOffset(from_tz.c_str(), from_tz.size(), &from_tz_offset); + seconds = dataconvert::mySQLTimeToGmtSec(my_start_time, from_tz_offset, valid); if (!valid) { if (seconds != 0) @@ -196,7 +198,9 @@ int64_t Func_convert_tz::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } else { - dataconvert::gmtSecToMySQLTime(seconds, my_time_tmp, to_tz); + long to_tz_offset; + dataconvert::timeZoneToOffset(to_tz.c_str(), to_tz.size(), &to_tz_offset); + dataconvert::gmtSecToMySQLTime(seconds, my_time_tmp, to_tz_offset); } dataconvert::DateTime result_datetime(my_time_tmp.year, my_time_tmp.month, my_time_tmp.day, diff --git a/utils/funcexp/func_date.cpp b/utils/funcexp/func_date.cpp index 8ebbd2250..5544e8b31 100644 --- a/utils/funcexp/func_date.cpp +++ b/utils/funcexp/func_date.cpp @@ -47,7 +47,7 @@ CalpontSystemCatalog::ColType Func_date::operationType(FunctionParm& fp, } int64_t Func_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType; @@ -75,7 +75,7 @@ int64_t Func_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul case execplan::CalpontSystemCatalog::TIMESTAMP: { int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - value = dataconvert::DataConvert::timestampToString(val1, timeZone()); + value = dataconvert::DataConvert::timestampToString(val1, ct.getTimeZone()); value = value.substr(0, 10); break; } diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index c98d54581..2e715acc7 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -759,7 +759,7 @@ int64_t Func_date_add::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index 0aff75d9e..962fc811d 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -242,7 +242,7 @@ CalpontSystemCatalog::ColType Func_date_format::operationType(FunctionParm& fp, } string Func_date_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { int64_t val = 0; DateTime dt = 0; @@ -273,7 +273,7 @@ string Func_date_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& TimeStamp timestamp(val); int64_t seconds = timestamp.second; MySQLTime time; - gmtSecToMySQLTime(seconds, time, timeZone()); + gmtSecToMySQLTime(seconds, time, ct.getTimeZone()); dt.year = time.year; dt.month = time.month; dt.day = time.day; @@ -412,7 +412,7 @@ int64_t Func_date_format::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& pa int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), ct.getTimeZone()); } } // namespace funcexp diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index 29baf4466..bd4a484d8 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -64,7 +64,7 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.day; } diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index 9a94882c6..07078a25f 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -76,7 +76,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, op_ct.getTimeZone()); year = time.year; month = time.month; day = time.day; diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index 64f04bcf3..441bfeebc 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -75,7 +75,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, ct.getTimeZone()); year = time.year; month = time.month; day = time.day; diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index 487635c7c..fb9d97428 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -74,7 +74,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 066151aa3..a6f8d6353 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -202,7 +202,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); dataconvert::DateTime dt; dt.year = m_time.year; dt.month = m_time.month; diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index c212d19c0..7c0dacc82 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -225,7 +225,7 @@ uint64_t Func_floor::getUintVal(Row& row, FunctionParm& parm, bool& isNull, case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); // strip off micro seconds str = str.substr(0, 14); @@ -519,7 +519,7 @@ IDB_Decimal Func_floor::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); // strip off micro seconds str = str.substr(0, 14); diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index b9ef7f823..7d2da7411 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -117,7 +117,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.hour; } diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 3e208dc7d..98fe8a334 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -35,7 +35,7 @@ using namespace rowgroup; namespace { -bool boolVal(SPTP& parm, Row& row, const string& timeZone) +bool boolVal(SPTP& parm, Row& row, long timeZone) { bool ret = true; bool isNull = false; // Keep it local. We don't want to mess with the global one here. @@ -122,9 +122,9 @@ CalpontSystemCatalog::ColType Func_if::operationType(FunctionParm& fp, return ct; } -int64_t Func_if::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +int64_t Func_if::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getIntVal(row, isNull); } @@ -134,9 +134,9 @@ int64_t Func_if::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSy } } -string Func_if::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +string Func_if::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getStrVal(row, isNull); } @@ -146,9 +146,10 @@ string Func_if::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSys } } -IDB_Decimal Func_if::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +IDB_Decimal Func_if::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getDecimalVal(row, isNull); } @@ -158,9 +159,9 @@ IDB_Decimal Func_if::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, C } } -double Func_if::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +double Func_if::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getDoubleVal(row, isNull); } @@ -171,9 +172,9 @@ double Func_if::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, Calpont } long double Func_if::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getLongDoubleVal(row, isNull); } @@ -183,9 +184,9 @@ long double Func_if::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull } } -int32_t Func_if::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +int32_t Func_if::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getDateIntVal(row, isNull); } @@ -195,9 +196,10 @@ int32_t Func_if::getDateIntVal(Row& row, FunctionParm& parm, bool& isNull, Calpo } } -int64_t Func_if::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +int64_t Func_if::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getDatetimeIntVal(row, isNull); } @@ -208,9 +210,9 @@ int64_t Func_if::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull, C } int64_t Func_if::getTimestampIntVal(Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getTimestampIntVal(row, isNull); } @@ -220,9 +222,9 @@ int64_t Func_if::getTimestampIntVal(Row& row, FunctionParm& parm, bool& isNull, } } -int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) +int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - if (boolVal(parm[0], row, timeZone())) + if (boolVal(parm[0], row, ct.getTimeZone())) { return parm[1]->data()->getTimeIntVal(row, isNull); } diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index 9040c47d2..89f173a77 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -74,7 +74,7 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 5f822d539..7d60ae507 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -1786,7 +1786,7 @@ string Func_format::getStrVal(Row& row, FunctionParm& parm, bool& isNull, case execplan::CalpontSystemCatalog::TIMESTAMP: { value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), - timeZone()); + operationColType.getTimeZone()); } break; diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index 4a5f594a7..af310c876 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -117,7 +117,7 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.minute; } diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index 3f9136a04..50ee3600e 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -63,7 +63,7 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.month; } diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 3f76db8ca..8ea7c3e37 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -97,7 +97,7 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, op_ct.getTimeZone()); return time.month; } diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 77c07c674..ba4ab740f 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -955,7 +955,8 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionPar string value; if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), timeZone()); + value = + DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); else value = DataConvert::datetimeToString1(parm[1]->data()->getDatetimeIntVal(row, isNull)); diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index c4417497f..04c4babb5 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -67,7 +67,7 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); month = m_time.month; break; } diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index f918aa7b4..cb495c8c1 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -48,7 +48,7 @@ using namespace logging; namespace { inline bool getBool(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull, - CalpontSystemCatalog::ColType& ct, const string& timeZone) + CalpontSystemCatalog::ColType& ct, long timeZone) { string expr; string pattern; @@ -244,7 +244,7 @@ CalpontSystemCatalog::ColType Func_regexp::operationType(FunctionParm& fp, bool Func_regexp::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBool(row, pm, isNull, ct, timeZone()) && !isNull; + return getBool(row, pm, isNull, ct, ct.getTimeZone()) && !isNull; } } // namespace funcexp diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 293159570..16855324e 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -569,7 +569,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull string value; if (op_ct.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), - timeZone()); + op_ct.getTimeZone()); else value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); @@ -709,5 +709,11 @@ int64_t Func_round::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull return parm[0]->data()->getIntVal(row, isNull); } +int64_t Func_round::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + return parm[0]->data()->getTimestampIntVal(row, isNull); +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index e07d29176..7731ea66c 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -116,7 +116,7 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.second; } diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index ff1f53356..0580021ff 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -42,7 +42,7 @@ namespace using namespace funcexp; dataconvert::DateTime getDateTime(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType& ct, const string& timeZone) + CalpontSystemCatalog::ColType& ct, long timeZone) { TimeExtractor extractor; dataconvert::DateTime dateTime; @@ -186,7 +186,7 @@ string Func_str_to_date::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); string convertedDate = dataconvert::DataConvert::datetimeToString(*((long long*)&dateTime)); return convertedDate; } @@ -195,7 +195,7 @@ int32_t Func_str_to_date::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return ((((int32_t)(time >> 32)) & 0xFFFFFFC0) | 0x3E); } @@ -204,7 +204,7 @@ int64_t Func_str_to_date::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& pa CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return time; } @@ -213,7 +213,7 @@ int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, FunctionParm& p CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); dataconvert::TimeStamp timestamp; dataconvert::MySQLTime m_time; m_time.year = dateTime.year; @@ -223,7 +223,7 @@ int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, FunctionParm& p m_time.minute = dateTime.minute; m_time.second = dateTime.second; bool isValid = true; - int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone(), isValid); + int64_t seconds = mySQLTimeToGmtSec(m_time, ct.getTimeZone(), isValid); if (!isValid) { timestamp = -1; @@ -243,7 +243,7 @@ int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, { dataconvert::DateTime dateTime; dataconvert::Time retTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); retTime.day = 0; retTime.is_neg = false; retTime.hour = dateTime.hour; @@ -258,7 +258,7 @@ int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, timeZone()); + dateTime = getDateTime(row, parm, isNull, ct, ct.getTimeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return time; } diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index a859c4843..14d2246ae 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -45,7 +45,7 @@ CalpontSystemCatalog::ColType Func_time::operationType(FunctionParm& fp, } string Func_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { int64_t val = 0; @@ -126,7 +126,7 @@ string Func_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); dataconvert::Time time; time.hour = m_time.hour; time.minute = m_time.minute; diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index 5b971b958..d075658b0 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -45,7 +45,7 @@ CalpontSystemCatalog::ColType Func_time_format::operationType(FunctionParm& fp, } string Func_time_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct) { // assume 256 is enough. assume not allowing incomplete date char buf[256]; @@ -72,7 +72,7 @@ string Func_time_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, ct.getTimeZone()); hour = m_time.hour; min = m_time.minute; sec = m_time.second; diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index d6b5de7cf..388ef2ca6 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -70,7 +70,7 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, op_ct.getTimeZone()); hour = time.hour; min = time.minute; sec = time.second; diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 2477943ab..d3f891397 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -142,16 +142,11 @@ string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is case execplan::CalpontSystemCatalog::TIMESTAMP: { - if (type1 != type2) - { - isNull = true; - break; - } int64_t temp = parm[0]->data()->getTimestampIntVal(row, isNull); dataconvert::TimeStamp timestamp(temp); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, ct.getTimeZone()); dataconvert::DateTime dt; dt.year = time.year; dt.month = time.month; @@ -239,7 +234,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(temp); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, time, ct.getTimeZone()); dataconvert::DateTime dt; dt.year = time.year; dt.month = time.month; @@ -328,7 +323,7 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, int64_t Func_timediff::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), ct.getTimeZone()); } int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index 83024c069..07e7eda73 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -55,7 +55,7 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bo TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); dt1.year = m_time.year; dt1.month = m_time.month; dt1.day = m_time.day; @@ -82,7 +82,7 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bo TimeStamp timestamp(parm[1]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, timeZone()); + gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); dt2.year = m_time.year; dt2.month = m_time.month; dt2.day = m_time.day; diff --git a/utils/funcexp/func_to_days.cpp b/utils/funcexp/func_to_days.cpp index 79fc165b4..f597c5fd7 100644 --- a/utils/funcexp/func_to_days.cpp +++ b/utils/funcexp/func_to_days.cpp @@ -85,7 +85,7 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index f9b7826d0..87f045af0 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -560,7 +560,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN int64_t x = 0; string value = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), op_ct.getTimeZone()); s = parm[1]->data()->getIntVal(row, isNull); @@ -714,5 +714,11 @@ string Func_truncate::getStrVal(Row& row, FunctionParm& parm, bool& isNull, return x.toString(true); } +int64_t Func_truncate::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + return parm[0]->data()->getTimestampIntVal(row, isNull); +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index f0fe91afc..8f7a3af08 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -76,7 +76,7 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index a6fbbcceb..302293b8e 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -73,7 +73,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 0b64c2534..2f4fc827b 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -63,7 +63,7 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); return m_time.year; } diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 3e59dd120..3b85692bf 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -78,7 +78,7 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); + dataconvert::gmtSecToMySQLTime(seconds, m_time, op_ct.getTimeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index ce83903cb..4eed16886 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -105,9 +105,9 @@ uint64_t Func::stringToDatetime(const string str) return ret; } -uint64_t Func::stringToTimestamp(const string str) +uint64_t Func::stringToTimestamp(const std::string& str, long timeZone) { - int64_t ret = DataConvert::stringToTimestamp(str, timeZone()); + int64_t ret = DataConvert::stringToTimestamp(str, timeZone); if (ret == -1) { diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 26b7d04c7..1cc73879f 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -73,17 +73,6 @@ class Func fFuncName = funcName; } - const std::string timeZone() const - { - std::unique_lock l(tzMutex); - return fTimeZone; - } - void timeZone(const std::string timeZone) - { - std::unique_lock l(tzMutex); - fTimeZone = timeZone; - } - void raiseIllegalParameterDataTypeError(const execplan::CalpontSystemCatalog::ColType& colType) const { std::ostringstream oss; @@ -177,7 +166,7 @@ class Func protected: virtual uint32_t stringToDate(std::string); virtual uint64_t stringToDatetime(std::string); - virtual uint64_t stringToTimestamp(std::string); + virtual uint64_t stringToTimestamp(const std::string&, long); virtual int64_t stringToTime(std::string); virtual uint32_t intToDate(int64_t); @@ -205,9 +194,6 @@ class Func float fFloatNullVal; double fDoubleNullVal; long double fLongDoubleNullVal; - - std::string fTimeZone; - mutable std::mutex tzMutex; }; class ParmTSInt64 : public datatypes::TSInt64Null @@ -216,7 +202,7 @@ class ParmTSInt64 : public datatypes::TSInt64Null ParmTSInt64() { } - ParmTSInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) + ParmTSInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, long timeZone) : TSInt64Null(parm->data()->toTSInt64Null(row)) { } @@ -228,7 +214,7 @@ class ParmTUInt64 : public datatypes::TUInt64Null ParmTUInt64() { } - ParmTUInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc) + ParmTUInt64(rowgroup::Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc, long timeZone) : TUInt64Null(parm->data()->toTUInt64Null(row)) { } @@ -240,8 +226,8 @@ class Arg2Lazy public: TA a; TB b; - Arg2Lazy(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc) - : a(row, parm[0], thisFunc), b(a.isNull() ? TB() : TB(row, parm[1], thisFunc)) + Arg2Lazy(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc, long timeZone) + : a(row, parm[0], thisFunc, timeZone), b(a.isNull() ? TB() : TB(row, parm[1], thisFunc, timeZone)) { } }; @@ -252,8 +238,8 @@ class Arg2Eager public: TA a; TB b; - Arg2Eager(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc) - : a(row, parm[0], thisFunc), b(row, parm[1], thisFunc) + Arg2Eager(rowgroup::Row& row, FunctionParm& parm, const Func& thisFunc, long timeZone) + : a(row, parm[0], thisFunc, timeZone), b(row, parm[1], thisFunc, timeZone) { } }; diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 59d4a05ab..60ccbe5e0 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -180,6 +180,9 @@ class Func_round : public Func_Real int64_t getDatetimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_truncate class @@ -214,6 +217,9 @@ class Func_truncate : public Func_Real execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_ceil class diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 4ed4abc95..1287cfd64 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -94,7 +94,7 @@ class Func_Str : public Func execplan::CalpontSystemCatalog::ColType& op_ct) { std::string str = getStrVal(row, fp, isNull, op_ct); - return (isNull ? 0 : stringToTimestamp(str)); + return (isNull ? 0 : stringToTimestamp(str, op_ct.getTimeZone())); } int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 7c64358ea..4fd94b6c3 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1,6 +1,6 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (c) 2019-2020 MariaDB Corporation + Copyright (c) 2019-2021 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -1567,7 +1567,8 @@ void RowAggregation::serialize(messageqcpp::ByteStream& bs) const for (uint64_t i = 0; i < functionCount; i++) fFunctionCols[i]->serialize(bs); - bs << fTimeZone; + messageqcpp::ByteStream::octbyte timeZone = fTimeZone; + bs << timeZone; } //------------------------------------------------------------------------------ @@ -1612,7 +1613,9 @@ void RowAggregation::deserialize(messageqcpp::ByteStream& bs) fFunctionCols.push_back(funct); } - bs >> fTimeZone; + messageqcpp::ByteStream::octbyte timeZone; + bs >> timeZone; + fTimeZone = timeZone; } //------------------------------------------------------------------------------ diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index 360f08128..a5abc4b40 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -338,7 +338,7 @@ struct GroupConcat std::vector> fOrderCond; // position to order by [asc/desc] joblist::ResourceManager* fRm; // resource manager boost::shared_ptr fSessionMemLimit; - std::string fTimeZone; + long fTimeZone; GroupConcat() : fRm(nullptr) { @@ -505,11 +505,11 @@ class RowAggregation : public messageqcpp::Serializeable return fAggMapKeyCount; } - inline void timeZone(const std::string& timeZone) + inline void timeZone(long timeZone) { fTimeZone = timeZone; } - inline const std::string& timeZone() const + inline long timeZone() const { return fTimeZone; } @@ -598,7 +598,7 @@ class RowAggregation : public messageqcpp::Serializeable bool fKeyOnHeap = false; - std::string fTimeZone; + long fTimeZone; // We need a separate copy for each thread. mcsv1sdk::mcsv1Context fRGContext; diff --git a/writeengine/bulk/cpimport.cpp b/writeengine/bulk/cpimport.cpp index bd7a62ea5..4b1fbac8d 100644 --- a/writeengine/bulk/cpimport.cpp +++ b/writeengine/bulk/cpimport.cpp @@ -648,12 +648,12 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob std::string timeZone = optarg; long offset; - if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + if (dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) { startupError(std::string("Value for option -T is invalid"), true); } - curJob.setTimeZone(timeZone); + curJob.setTimeZone(offset); break; } diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index 566b99759..ebec84dc2 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -159,7 +159,7 @@ BulkLoad::BulkLoad() , fbContinue(false) , fDisableTimeOut(false) , fUUID(boost::uuids::nil_generator()()) - , fTimeZone("SYSTEM") + , fTimeZone(dataconvert::systemTimeZoneOffset()) , fUsername("mysql") // MCOL-4328 default file owner { fTableInfo.clear(); diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index cd800dd9e..ccc362ca4 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -107,7 +107,7 @@ class BulkLoad : public FileOp void addToCmdLineImportFileList(const std::string& importFile); const std::string& getAlternateImportDir() const; const std::string& getErrorDir() const; - const std::string& getTimeZone() const; + long getTimeZone() const; const std::string& getJobDir() const; const std::string& getSchema() const; const std::string& getTempJobDir() const; @@ -145,7 +145,7 @@ class BulkLoad : public FileOp void setTruncationAsError(bool bTruncationAsError); void setJobUUID(const std::string& jobUUID); void setErrorDir(const std::string& errorDir); - void setTimeZone(const std::string& timeZone); + void setTimeZone(long timeZone); void setS3Key(const std::string& key); void setS3Secret(const std::string& secret); void setS3Bucket(const std::string& bucket); @@ -229,7 +229,9 @@ class BulkLoad : public FileOp bool fDisableTimeOut; // disable timeout when waiting for table lock boost::uuids::uuid fUUID; // job UUID static bool fNoConsoleOutput; // disable output to console - std::string fTimeZone; // Timezone to use for TIMESTAMP data type + long fTimeZone; // Timezone offset (in seconds) relative to UTC, + // to use for TIMESTAMP data type. For example, + // for EST which is UTC-5:00, offset will be -18000s. std::string fS3Key; // S3 Key std::string fS3Secret; // S3 Secret std::string fS3Host; // S3 Host @@ -318,7 +320,7 @@ inline const std::string& BulkLoad::getErrorDir() const return fErrorDir; } -inline const std::string& BulkLoad::getTimeZone() const +inline long BulkLoad::getTimeZone() const { return fTimeZone; } @@ -486,7 +488,7 @@ inline void BulkLoad::setErrorDir(const std::string& errorDir) fErrorDir = errorDir; } -inline void BulkLoad::setTimeZone(const std::string& timeZone) +inline void BulkLoad::setTimeZone(long timeZone) { fTimeZone = timeZone; } diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 9f26a161f..ef1792a09 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -139,7 +139,7 @@ BulkLoadBuffer::BulkLoadBuffer(unsigned numberOfCols, unsigned bufferSize, Log* , fTableName(tableName) , fbTruncationAsError(false) , fImportDataMode(IMPORT_DATA_TEXT) - , fTimeZone("SYSTEM") + , fTimeZone(dataconvert::systemTimeZoneOffset()) , fFixedBinaryRecLen(0) { fData = new char[bufferSize]; diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index ea0625e39..9aa32cc1e 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -152,7 +152,9 @@ class BulkLoadBuffer // for db cols (omits default cols) bool fbTruncationAsError; // Treat string truncation as error ImportDataMode fImportDataMode; // Import data in text or binary mode - std::string fTimeZone; // Timezone used by TIMESTAMP datatype + long fTimeZone; // Timezone offset (in seconds) relative to UTC, + // to use for TIMESTAMP data type. For example, + // for EST which is UTC-5:00, offset will be -18000s. unsigned int fFixedBinaryRecLen; // Fixed rec len used in binary mode //-------------------------------------------------------------------------- @@ -388,7 +390,7 @@ class BulkLoadBuffer /** @brief set timezone. */ - void setTimeZone(const std::string& timeZone) + void setTimeZone(long timeZone) { fTimeZone = timeZone; } diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 29dc7b202..cdda59c17 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -146,7 +146,7 @@ TableInfo::TableInfo(Log* logger, const BRM::TxnID txnID, const string& processN , fKeepRbMetaFile(bKeepRbMetaFile) , fbTruncationAsError(false) , fImportDataMode(IMPORT_DATA_TEXT) - , fTimeZone("SYSTEM") + , fTimeZone(dataconvert::systemTimeZoneOffset()) , fTableLocked(false) , fReadFromStdin(false) , fReadFromS3(false) diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index 522760fe7..bba15d97d 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -127,7 +127,9 @@ class TableInfo : public WeUIDGID // data file bool fbTruncationAsError; // Treat string truncation as error ImportDataMode fImportDataMode; // Import data in text or binary mode - std::string fTimeZone; // Timezone used by TIMESTAMP data type + long fTimeZone; // Timezone offset (in seconds) relative to UTC, + // to use for TIMESTAMP data type. For example, + // for EST which is UTC-5:00, offset will be -18000s. volatile bool fTableLocked; // Do we have db table lock @@ -254,7 +256,7 @@ class TableInfo : public WeUIDGID /** @brief Get timezone. */ - const std::string& getTimeZone() const; + long getTimeZone() const; /** @brief Get number of buffers */ @@ -315,7 +317,7 @@ class TableInfo : public WeUIDGID /** @brief Set timezone. */ - void setTimeZone(const std::string& timeZone); + void setTimeZone(long timeZone); /** @brief Enable distributed mode, saving BRM updates in rptFileName */ @@ -481,7 +483,7 @@ inline ImportDataMode TableInfo::getImportDataMode() const return fImportDataMode; } -inline const std::string& TableInfo::getTimeZone() const +inline long TableInfo::getTimeZone() const { return fTimeZone; } @@ -582,7 +584,7 @@ inline void TableInfo::setImportDataMode(ImportDataMode importMode) fImportDataMode = importMode; } -inline void TableInfo::setTimeZone(const std::string& timeZone) +inline void TableInfo::setTimeZone(long timeZone) { fTimeZone = timeZone; } diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 69373fff6..1d832f9e0 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -3552,7 +3552,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) int dataWidth, scale, precision, compressionType, refColWidth, refCompressionType; string defaultValStr; ColTuple defaultVal; - string timeZone; + long timeZone; bs >> tmp32; txnID = tmp32; @@ -3581,7 +3581,9 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) refColWidth = tmp32; bs >> tmp8; refCompressionType = tmp8; - bs >> timeZone; + messageqcpp::ByteStream::octbyte timeZoneTemp; + bs >> timeZoneTemp; + timeZone = timeZoneTemp; // Find the fill in value bool isNULL = false; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index d183e3b0f..ec4aff7bc 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2681,7 +2681,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin CalpontSystemCatalog::OID oid = 0; CalpontSystemCatalog::ROPair tableRO; - std::string timeZone = cpackages[txnId].get_TimeZone(); + long timeZone = cpackages[txnId].get_TimeZone(); try { diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 0cd44b458..f4f5662d7 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -71,7 +71,11 @@ const long long columnstore_precision[19] = {0, //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -XMLJob::XMLJob() : fDebugLevel(DEBUG_0), fDeleteTempFile(false), fValidateColList(true), fTimeZone("SYSTEM") +XMLJob::XMLJob() + : fDebugLevel(DEBUG_0) + , fDeleteTempFile(false) + , fValidateColList(true) + , fTimeZone(dataconvert::systemTimeZoneOffset()) { } diff --git a/writeengine/xml/we_xmljob.h b/writeengine/xml/we_xmljob.h index 61aae0f94..ba33cd4a7 100644 --- a/writeengine/xml/we_xmljob.h +++ b/writeengine/xml/we_xmljob.h @@ -116,7 +116,7 @@ class XMLJob : public XMLOp /** * @brief Set timezone */ - void setTimeZone(const std::string& timeZone) + void setTimeZone(long timeZone) { fTimeZone = timeZone; } @@ -144,7 +144,9 @@ class XMLJob : public XMLOp JobColList fDefaultColumns; // temporary list of default cols // for table node being processed bool fValidateColList; // Validate all cols have XML tag - std::string fTimeZone; // Timezone used for TIMESTAMP datatype + long fTimeZone; // Timezone offset (in seconds) relative to UTC, + // to use for TIMESTAMP data type. For example, + // for EST which is UTC-5:00, offset will be -18000s. }; } // namespace WriteEngine From 9b686c04e1cd1f1455cc9d0f5f216cf28696e52f Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Tue, 15 Feb 2022 14:23:08 +0000 Subject: [PATCH 04/55] wrong return type --- dbcon/execplan/arithmeticcolumn.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index 70bf7be81..d7fb57027 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -283,7 +283,7 @@ const string ArithmeticColumn::nextToken(string::size_type& pos, char end) const msg.append(1, end); msg.append(" found in " + fData); throw invalid_argument(msg); - return 0; + return {}; } ostream& operator<<(ostream& output, const ArithmeticColumn& rhs) From 757aa0bd73e29ee68759e226007bbe35c56134ba Mon Sep 17 00:00:00 2001 From: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com> Date: Wed, 16 Feb 2022 18:56:21 +0300 Subject: [PATCH 05/55] Bootstrap (#2264) Development bootstrap script re-factoring patch --- build/README.md | 6 +- build/bootstrap_mcs.sh | 331 +++++++++++++++++++++--------------- build/my.cnf.in | 10 ++ build/utils.sh | 377 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 582 insertions(+), 142 deletions(-) create mode 100644 build/my.cnf.in create mode 100644 build/utils.sh diff --git a/build/README.md b/build/README.md index 9ee3599bc..1a4bbaa75 100644 --- a/build/README.md +++ b/build/README.md @@ -4,13 +4,15 @@ This is MariaDB Columnstore To build MCS from source you will need: * modern linux distribution. e.g. Ubuntu 20 + * modern C++ compiler: clang version greater than 12 or gcc version greater than 11 Clone or download this repository. git clone https://github.com/MariaDB/server -Edit a bootstrap script to fix paths to MariaDB repo cloned. - vim /some/path/server/storage/columnstore/columnstore/build/bootstrap_mcs.sh +Update Sumbodules + git submodule update --init --recursive Run the bootstrap /some/path/server/storage/columnstore/columnstore/build/bootstrap_mcs.sh +bootstrap_mcs.sh could be run with --help for list of flags diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 8187e5e0c..edd5ec52e 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -1,179 +1,230 @@ -#!/bin/sh +#!/bin/bash # This script compiles/installs MCS from scratch and it makes some assumptions: # - the server's source code is two directories above the MCS engine source. # - the script is to be run under root. -# - it is run MDB_MDB_SOURCE_PATH/storage/columnstore/columnstore, e.g. ./build/bootstrap_mcs.sh -DISTRO=$1 -MDB_BUILD_TYPE=$2 -MDB_SOURCE_PATH=`pwd`/../../../ -MCS_CONFIG_DIR=/etc/columnstore -# Needs systemd to be installed obviously. -# Feel free to ask in MariaDB Zulip how to bootstrap MCS in containers or other systemd-free environments. -systemctl stop mariadb-columnstore -MDB_GIT_URL=https://github.com/MariaDB/server.git -MDB_GIT_TAG=10.8 -if [ -z "$DISTRO" ]; then - echo "Choose a distro" + +SCRIPT_LOCATION=$(dirname "$0") +MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../..) + +source $SCRIPT_LOCATION/utils.sh + + +if [ "$EUID" -ne 0 ] + then error "Please run script as root to install MariaDb to system paths" exit 1 fi -if [ $DISTRO = 'bionic' ]; then - sudo apt-get -y update - apt-get -y install build-essential automake libboost-all-dev bison cmake \ - libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ - libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ - libsnappy-dev libcurl4-openssl-dev -elif [ $DISTRO = 'focal' ]; then - sudo apt-get -y update - apt-get -y install build-essential automake libboost-all-dev bison cmake \ - libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ - libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ - libsnappy-dev libcurl4-openssl-dev -elif [ $DISTRO = 'centos' ]; then - yum -y install epel-release \ - && yum -y groupinstall "Development Tools" \ - && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu \ - && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel -elif [ $DISTRO = 'leap' ]; then - zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ - && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel \ - && zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel \ - && zypper install -y gcc gcc-c++ git automake libtool +message "Building Mariadb Server from $MDB_SOURCE_PATH" + +BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") +DISTRO_OPTIONS=("Ubuntu" "Centos" "Debian" "openSUSE") + +optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE +optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS +optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true +source $( optparse.build ) + +if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then + getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS + MCS_BUILD_TYPE=$selectedChoice fi -if [ ! -d $MDB_SOURCE_PATH ]; then - git clone $MDB_GIT_URL $MDB_SOURCE_PATH -b $MDB_GIT_TAG +if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then + detect_distro fi -if [ ! -d $MCS_CONFIG_DIR ]; then - mkdir $MCS_CONFIG_DIR -fi +INSTALL_PREFIX="/usr/" +DATA_DIR="/var/lib/mysql/data" -if [ -z "$(grep mysql /etc/passwd)" ]; then - echo "Adding user mysql into /etc/passwd" - useradd -r -U mysql -d /var/lib/mysql - exit 1 -fi +install_deps() +{ + message "Installing deps" + if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then + sudo apt-get -y update + apt-get -y install build-essential automake libboost-all-dev bison cmake \ + libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ + libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ + libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev + elif [ $OS = 'CentOS Linux' ]; then + yum -y install epel-release \ + && yum -y groupinstall "Development Tools" \ + && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu \ + && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel gtest-devel cppunit-devel + elif [ $OS = 'OpenSuse' ]; then + zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ + && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel \ + && zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel \ + && zypper install -y gcc gcc-c++ git automake libtool gtest cppunit-devel + fi +} -if [ -z "$(grep mysql /etc/group)" ]; then - echo "You need to manually add mysql group into /etc/group, e.g. mysql:x:999" - exit 1 -fi +stop_service() +{ + message "Stopping MariaDB services" + systemctl stop mariadb + systemctl stop mariadb-columnstore +} -MCS_INSTALL_PREFIX=/var/lib/ -rm -rf /var/lib/columnstore/data1/* -rm -rf /var/lib/columnstore/data/ -rm -rf /var/lib/columnstore/local/ -rm -f /var/lib/columnstore/storagemanager/storagemanager-lock -rm -f /var/lib/columnstore/storagemanager/cs-initialized +check_service() +{ + if systemctl is-active --quiet $1; then + message "$1 service started OK" + else + error "$1 service failed" + service $1 status + fi +} -MCS_TMP_DIR=/tmp/columnstore_tmp_files -TMP_PATH=/tmp -CPUS=$(getconf _NPROCESSORS_ONLN) +start_service() +{ + message "Starting MariaDB services" + systemctl start mariadb-columnstore + systemctl start mariadb -# script -rm -rf $MCS_TMP_DIR/* -rm -rf /var/lib/mysql + check_service mariadb-columnstore + check_service mariadb +} -cd $MDB_SOURCE_PATH -if [[ "$DISTRO" = 'bionic' || "$DISTRO" = 'focal' ]]; then - #MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_TOKUDB=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_MROONGA=NO -DPLUGIN_GSSAPI=NO -DWITH_MARIABACKUP=NO -DDEB=bionic -DPLUGIN_COLUMNSTORE=YES' - MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF -DWITH_SSL=system -DDEB=bionic' - # Some development flags - #MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_GTEST=1 -DWITH_ROWGROUP_UT=1 -DWITH_DATACONVERT_UT=1 -DWITH_ARITHMETICOPERATOR_UT=1 -DWITH_ORDERBY_UT=1 -DWITH_CSDECIMAL_UT=1 -DWITH_SORTING_COMPARATORS_UT=1" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES -DWITH_BRM_UT=YES" #-DWITH_GTEST=1 -DWITH_UNITTESTS=YES - cmake . -DCMAKE_BUILD_TYPE=$MDB_BUILD_TYPE ${MDB_CMAKE_FLAGS} && \ - make -j $CPUS install -elif [ $DISTRO = 'centos' ]; then - MDB_CMAKE_FLAGS='-DWITH_SYSTEMD=yes -DPLUGIN_AUTH_GSSAPI=NO -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF -DWITH_SSL=system -DRPM=CentOS7' +clean_old_installation() +{ + message "Cleaning old installation" + rm -rf /var/lib/columnstore/data1/* + rm -rf /var/lib/columnstore/data/ + rm -rf /var/lib/columnstore/local/ + rm -f /var/lib/columnstore/storagemanager/storagemanager-lock + rm -f /var/lib/columnstore/storagemanager/cs-initialized + rm -rf /tmp/* + rm -rf /var/lib/mysql + rm -rf /var/run/mysqld + rm -rf $DATA_DIR +} + +build() +{ + message "Building sources" + local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes + -DPLUGIN_COLUMNSTORE=YES + -DPLUGIN_MROONGA=NO + -DPLUGIN_ROCKSDB=NO + -DPLUGIN_TOKUDB=NO + -DPLUGIN_CONNECT=NO + -DPLUGIN_SPIDER=NO + -DPLUGIN_OQGRAPH=NO + -DPLUGIN_SPHINX=NO + -DBUILD_CONFIG=mysql_release + -DWITH_WSREP=OFF + -DWITH_SSL=system + -DWITH_UNITTESTS=YES + -DWITH_BRM_UT=YES + -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX" + + cd $MDB_SOURCE_PATH + + if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" + elif [ $OS = 'Centos' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" + fi + local CPUS=$(getconf _NPROCESSORS_ONLN) cmake . -DCMAKE_BUILD_TYPE=$MDB_BUILD_TYPE $MDB_CMAKE_FLAGS && \ make -j $CPUS install -fi -if [ $? -ne 0 ]; then - return 1 -fi + if [ $? -ne 0 ]; then + error "!!!! BUILD FAILED" + exit 1 + fi + cd - +} -# These two lines are to handle file layout difference b/w RPM- and DEB-based distributions. -# One of the lines always fails. -mv /usr/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so -mv /usr/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so +install() +{ + message "Installing MariaDB" + if [ -z "$(grep mysql /etc/passwd)" ]; then + message "Adding user mysql into /etc/passwd" + useradd -r -U mysql -d /var/lib/mysql + fi + if [ -z "$(grep mysql /etc/group)" ]; then + echo "You need to manually add mysql group into /etc/group, e.g. mysql:x:999" + GroupID = `awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group` + message "Adding group mysql with id $GroupID" + groupadd -g GroupID mysql + fi -/usr/bin/mysql_install_db --rpm --user=mysql + mkdir -p /etc/my.cnf.d -mv /tmp/ha_columnstore_1.so /usr/lib/mysql/plugin/ha_columnstore.so -mv /tmp/ha_columnstore_2.so /usr/lib64/mysql/plugin/ha_columnstore.so + bash -c 'echo "[client-server] +socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' -cp -r /etc/mysql/conf.d /etc/my.cnf.d/ -cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml -cp $MDB_SOURCE_PATH/storage/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml -cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf -cp $MDB_SOURCE_PATH/storage/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + mv $INSTALL_PREFIX/lib/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so + message "Running mysql_install_db" + mysql_install_db --rpm --user=mysql + mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so + chown mysql:mysql $INSTALL_PREFIX/lib/plugin/ha_columnstore.so + + mkdir -p /etc/columnstore + + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + + cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/ + cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + + if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then + mkdir -p /usr/share/mysql + mkdir -p /etc/mysql/ + cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh + cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start + > /etc/mysql/debian.cnf + fi -if [[ "$DISTRO" = 'bionic' || "$DISTRO" = 'focal' ]]; then - cp ./support-files/*.service /lib/systemd/system/ - cp ./storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - cp ./storage/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - cp ./debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh - cp ./debian/additions/debian-start /etc/mysql/debian-start systemctl daemon-reload - rm -f /etc/mysql/my.cnf - cp -r /etc/mysql/conf.d/ /etc/my.cnf.d - cp -rp /etc/mysql/mariadb.conf.d/ /etc/my.cnf.d - mkdir /var/lib/columnstore/data1 - mkdir /var/lib/columnstore/data1/systemFiles - mkdir /var/lib/columnstore/data1/systemFiles/dbrm - chown -R mysql:mysql /var/lib/mysql - chown -R mysql:mysql /var/lib/mysql - chown -R mysql.mysql /var/run/mysqld - chown -R mysql:mysql /data/columnstore/* - chmod +x /usr/bin/mariadb* - cp /etc/my.cnf.d/mariadb.conf.d/columnstore.cnf /etc/my.cnf.d/ - ldconfig - columnstore-post-install - chown -R mysql:mysql /data/columnstore/* + if [ -d "/etc/mysql/mariadb.conf.d/" ]; then + message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d" + cp -rp /etc/mysql/mariadb.conf.d/* /etc/my.cnf.d + fi - /usr/sbin/install_mcs_mysql.sh + if [ -d "/etc/mysql/conf.d/" ]; then + message "Copying configs from /etc/mysql/conf.d/ to /etc/my.cnf.d" + cp -rp /etc/mysql/conf.d/* /etc/my.cnf.d + fi - chown -R syslog.syslog /var/log/mariadb/ - chmod 777 /var/log/mariadb/ - chmod 777 /var/log/mariadb/columnstore + mkdir -p /var/lib/columnstore/data1 + mkdir -p /var/lib/columnstore/data1/systemFiles + mkdir -p /var/lib/columnstore/data1/systemFiles/dbrm + mkdir -p /run/mysqld/ -elif [ $DISTRO = 'centos' ]; then - cp ./support-files/*.service /lib/systemd/system/ - cp ./storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - cp ./storage/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - systemctl daemon-reload - rm -f /etc/mysql/my.cnf - cp -r /etc/mysql/conf.d/ /etc/my.cnf.d - mkdir /var/lib/columnstore/data1 - mkdir /var/lib/columnstore/data1/systemFiles - mkdir /var/lib/columnstore/data1/systemFiles/dbrm - chown -R mysql:mysql /var/lib/mysql - chown -R mysql:mysql /data/columnstore/* - chown -R mysql.mysql /var/run/mysqld - chmod +x /usr/bin/mariadb* + mkdir -p $DATA_DIR + chown -R mysql:mysql $DATA_DIR + chown -R mysql:mysql /var/lib/columnstore/ + chown -R mysql:mysql /run/mysqld/ - ldconfig - columnstore-post-install - chown -R mysql:mysql /data/columnstore/* + chmod +x $INSTALL_PREFIX/bin/mariadb* - /usr/sbin/install_mcs_mysql.sh + ldconfig - /usr/sbin/install_mcs_mysql.sh - mkdir /var/lib/columnstore/data1 - mkdir /var/lib/columnstore/data1/systemFiles - mkdir /var/lib/columnstore/data1/systemFiles/dbrm + message "Running columnstore-post-install" + mkdir -p /var/lib/columnstore/local + columnstore-post-install --rpmmode=install + message "Running install_mcs_mysql" + install_mcs_mysql.sh - chown -R mysql:mysql /var/log/mariadb/ - chmod 777 /var/log/mariadb/ - chmod 777 /var/log/mariadb/columnstore + chown -R syslog:syslog /var/log/mariadb/ + chmod 777 /var/log/mariadb/ + chmod 777 /var/log/mariadb/columnstore +} + +if [[ $SKIP_DEPS = false ]] ; then + install_deps fi - -exit 0 +stop_service +clean_old_installation +build +install +start_service +message "FINISHED" diff --git a/build/my.cnf.in b/build/my.cnf.in new file mode 100644 index 000000000..d2d2332ae --- /dev/null +++ b/build/my.cnf.in @@ -0,0 +1,10 @@ +[client-server] + +# +# include *.cnf from the config directory +# +!includedir /etc/my.cnf.d + +[mysqld] +datadir=%DATADIR% +plugin-load-add=ha_columnstore.so \ No newline at end of file diff --git a/build/utils.sh b/build/utils.sh new file mode 100644 index 000000000..100f968b6 --- /dev/null +++ b/build/utils.sh @@ -0,0 +1,377 @@ +color_normal=$(tput sgr0) +color_bold=$(tput bold) +color_red="$color_bold$(tput setaf 1)" +color_green=$(tput setaf 2) +color_fawn=$(tput setaf 3); color_beige="$color_fawn" +color_yellow="$color_bold$color_fawn" +color_darkblue=$(tput setaf 4) +color_blue="$color_bold$color_darkblue" +color_purple=$(tput setaf 5); color_magenta="$color_purple" +color_pink="$color_bold$color_purple" +color_darkcyan=$(tput setaf 6) +color_cyan="$color_bold$color_darkcyan" +color_gray=$(tput setaf 7) +color_darkgray="$color_bold"$(tput setaf 0) +color_white="$color_bold$color_gray" + +message() +{ + echo $color_cyan -- $@$color_normal +} + +warn() +{ + echo $color_yellow -- $@$color_normal +} + +error() +{ + echo $color_red -- $@$color_normal +} + +detect_distro() +{ + if [ -f /etc/os-release ]; then + . /etc/os-release + export OS=$NAME + export OS_VERSION=$VERSION_ID + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + export OS=$(lsb_release -si) + export OS_VERSION=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + export OS=$DISTRIB_ID + OS_VERSION=$DISTRIB_RELEASE + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + OS_VERSION=$(cat /etc/debian_version) + else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + OS=$(uname -s) + OS_VERSION=$(uname -r) + fi + OS=$(echo $OS | cut -f 1 -d " ") + message "Detected $OS $OS_VERSION" +} + +menuStr="" + +function hideCursor(){ + printf "\033[?25l" + + # capture CTRL+C so cursor can be reset + trap "showCursor && exit 0" 2 +} + +function showCursor(){ + printf "\033[?25h" +} + +function clearLastMenu(){ + local msgLineCount=$(printf "$menuStr" | wc -l) + # moves the curser up N lines so the output overwrites it + echo -en "\033[${msgLineCount}A" + + # clear to end of screen to ensure there's no text left behind from previous input + [ $1 ] && tput ed +} + +function renderMenu(){ + local start=0 + local selector="" + local instruction="$1" + local selectedIndex=$2 + local listLength=$itemsLength + local longest=0 + local spaces="" + menuStr="\n $instruction\n" + + # Get the longest item from the list so that we know how many spaces to add + # to ensure there's no overlap from longer items when a list is scrolling up or down. + for (( i=0; i<$itemsLength; i++ )); do + if (( ${#menuItems[i]} > longest )); then + longest=${#menuItems[i]} + fi + done + spaces=$(printf ' %.0s' $(eval "echo {1.."$(($longest))"}")) + + if [ $3 -ne 0 ]; then + listLength=$3 + + if [ $selectedIndex -ge $listLength ]; then + start=$(($selectedIndex+1-$listLength)) + listLength=$(($selectedIndex+1)) + fi + fi + + for (( i=$start; i<$listLength; i++ )); do + local currItem="${menuItems[i]}" + currItemLength=${#currItem} + + if [[ $i = $selectedIndex ]]; then + selectedChoice="${currItem}" + selector="${color_green}ᐅ${color_normal}" + currItem="${color_green}${currItem}${color_normal}" + else + selector=" " + fi + + currItem="${spaces:0:0}${currItem}${spaces:currItemLength}" + + menuStr="${menuStr}\n ${selector} ${currItem}" + done + + menuStr="${menuStr}\n" + + # whether or not to overwrite the previous menu output + [ $4 ] && clearLastMenu + + printf "${menuStr}" +} + +function getChoice(){ + local KEY__ARROW_UP=$(echo -e "\033[A") + local KEY__ARROW_DOWN=$(echo -e "\033[B") + local KEY__ENTER=$(echo -e "\n") + local captureInput=true + local displayHelp=false + local maxViewable=0 + local instruction="Select an item from the list:" + local selectedIndex=0 + + remainingArgs=() + while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + -h|--help) + displayHelp=true + shift + ;; + -i|--index) + selectedIndex=$2 + shift 2 + ;; + -m|--max) + maxViewable=$2 + shift 2 + ;; + -o|--options) + menuItems=$2[@] + menuItems=("${!menuItems}") + shift 2 + ;; + -q|--query) + instruction="$2" + shift 2 + ;; + *) + remainingArgs+=("$1") + shift + ;; + esac + done + + # just display help + if $displayHelp; then + echo; + echo "Usage: getChoice [OPTION]..." + echo "Renders a keyboard navigable menu with a visual indicator of what's selected." + echo; + echo " -h, --help Displays this message" + echo " -i, --index The initially selected index for the options" + echo " -m, --max Limit how many options are displayed" + echo " -o, --options An Array of options for a User to choose from" + echo " -q, --query Question or statement presented to the User" + echo; + echo "Example:" + echo " foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")" + echo; + echo " getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4" + echo " printf \"\\n First choice is '\${selectedChoice}'\\n\"" + echo; + echo " getChoice -q \"Select another option in case the first isn't available\" -o foodOptions" + echo " printf \"\\n Second choice is '\${selectedChoice}'\\n\"" + echo; + + return 0 + fi + + set -- "${remainingArgs[@]}" + local itemsLength=${#menuItems[@]} + + # no menu items, at least 1 required + if [[ $itemsLength -lt 1 ]]; then + printf "\n [ERROR] No menu items provided\n" + exit 1 + fi + + renderMenu "$instruction" $selectedIndex $maxViewable + hideCursor + + while $captureInput; do + read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters. + + case "$key" in + "$KEY__ARROW_UP") + selectedIndex=$((selectedIndex-1)) + (( $selectedIndex < 0 )) && selectedIndex=$((itemsLength-1)) + + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; + + "$KEY__ARROW_DOWN") + selectedIndex=$((selectedIndex+1)) + (( $selectedIndex == $itemsLength )) && selectedIndex=0 + + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; + + "$KEY__ENTER") + clearLastMenu true + showCursor + captureInput=false + ;; + esac + done +} + + +function optparse.throw_error(){ + local message="$1" + error "OPTPARSE: ERROR: $message" + exit 1 +} + +# ----------------------------------------------------------------------------------------------------------------------------- +function optparse.define(){ + if [ $# -lt 3 ]; then + optparse.throw_error "optparse.define [] [] []" + fi + for option_id in $( seq 1 $# ) ; do + local option="$( eval "echo \$$option_id")" + local key="$( echo $option | awk -F "=" '{print $1}' )"; + local value="$( echo $option | awk -F "=" '{print $2}' )"; + + #essentials: shortname, longname, description + if [ "$key" = "short" ]; then + local shortname="$value" + if [ ${#shortname} -ne 1 ]; then + optparse.throw_error "short name expected to be one character long" + fi + local short="-${shortname}" + elif [ "$key" = "long" ]; then + local longname="$value" + if [ ${#longname} -lt 2 ]; then + optparse.throw_error "long name expected to be atleast one character long" + fi + local long="--${longname}" + elif [ "$key" = "desc" ]; then + local desc="$value" + elif [ "$key" = "default" ]; then + local default="$value" + elif [ "$key" = "variable" ]; then + local variable="$value" + elif [ "$key" = "value" ]; then + local val="$value" + fi + done + + if [ "$variable" = "" ]; then + optparse.throw_error "You must give a variable for option: ($short/$long)" + fi + + if [ "$val" = "" ]; then + val="\$OPTARG" + fi + + # build OPTIONS and help + optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")" + if [ "$default" != "" ]; then + optparse_usage="${optparse_usage} [default:$default]" + fi + optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;" + if [ "$default" != "" ]; then + optparse_defaults="${optparse_defaults}#NL${variable}=${default}" + fi + optparse_arguments_string="${optparse_arguments_string}${shortname}" + if [ "$val" = "\$OPTARG" ]; then + optparse_arguments_string="${optparse_arguments_string}:" + fi + optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;" +} + +# ----------------------------------------------------------------------------------------------------------------------------- +function optparse.build(){ + local build_file="$(mktemp -t "optparse-XXXXXX.tmp")" + + # Building getopts header here + + # Function usage + cat << EOF > $build_file +function usage(){ +cat << XXX +usage: \$0 [OPTIONS] +OPTIONS: + $optparse_usage + -? --help : usage +XXX +} +# Contract long options into short options +params="" +while [ \$# -ne 0 ]; do + param="\$1" + shift + case "\$param" in + $optparse_contractions + "-?"|--help) + usage + exit 0;; + *) + if [[ "\$param" == --* ]]; then + echo -e "Unrecognized long option: \$param" + usage + exit 1 + fi + params="\$params \"\$param\"";; + esac +done +eval set -- "\$params" +# Set default variable values +$optparse_defaults +# Process using getopts +while getopts "$optparse_arguments_string" option; do + case \$option in + # Substitute actions for different variables + $optparse_process + :) + echo "Option - \$OPTARG requires an argument" + exit 1;; + *) + usage + exit 1;; + esac +done +# Clean up after self +rm $build_file +EOF + + local -A o=( ['#NL']='\n' ['#TB']='\t' ) + + for i in "${!o[@]}"; do + sed -i "s/${i}/${o[$i]}/g" $build_file + done + + # Unset global variables + unset optparse_usage + unset optparse_process + unset optparse_arguments_string + unset optparse_defaults + unset optparse_contractions + + # Return file name to parent + echo "$build_file" +} \ No newline at end of file From 29752afc77d1fd7cfa368e5a3773c73341786bec Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Wed, 16 Feb 2022 18:17:53 +0000 Subject: [PATCH 06/55] Some bootstrap refines --- build/bootstrap_mcs.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index edd5ec52e..ccfde5e0e 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -25,6 +25,8 @@ DISTRO_OPTIONS=("Ubuntu" "Centos" "Debian" "openSUSE") optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true +optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true + source $( optparse.build ) if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then @@ -100,6 +102,7 @@ clean_old_installation() rm -rf /var/lib/mysql rm -rf /var/run/mysqld rm -rf $DATA_DIR + rm -rf /etc/mysql } build() @@ -123,11 +126,20 @@ build() cd $MDB_SOURCE_PATH + if [[ $FORCE_CMAKE_CONFIG = true ]] ; then + warn "Erasing cmake cache" + rm -f "$MDB_SOURCE_PATH/CMakeCache.txt" + rm -rf "$MDB_SOURCE_PATH/CMakeFiles" + fi + if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" elif [ $OS = 'Centos' ]; then MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" fi + + message "building with flags $MDB_CMAKE_FLAGS" + local CPUS=$(getconf _NPROCESSORS_ONLN) cmake . -DCMAKE_BUILD_TYPE=$MDB_BUILD_TYPE $MDB_CMAKE_FLAGS && \ make -j $CPUS install @@ -159,10 +171,10 @@ install() bash -c 'echo "[client-server] socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' - mv $INSTALL_PREFIX/lib/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so + mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so message "Running mysql_install_db" mysql_install_db --rpm --user=mysql - mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so + mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so chown mysql:mysql $INSTALL_PREFIX/lib/plugin/ha_columnstore.so mkdir -p /etc/columnstore From 3919c541acf83283c8f191210d7cd62d786716d8 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com> Date: Thu, 17 Feb 2022 13:08:58 +0300 Subject: [PATCH 07/55] New warnfixes (#2254) * Fix clang warnings * Remove vim tab guides * initialize variables * 'strncpy' output truncated before terminating nul copying as many bytes from a string as its length * Fix ISO C++17 does not allow 'register' storage class specifier for outdated bison * chars are unsigned on ARM, having if (ival < 0) always false * chars are unsigned by default on ARM and comparison with -1 if always true --- CMakeLists.txt | 2 +- datatypes/mcs_datatype.cpp | 1 - datatypes/mcs_datatype.h | 1 - datatypes/mcs_datatype_basic.h | 1 - datatypes/mcs_double.h | 1 - datatypes/mcs_float128.h | 1 - datatypes/mcs_int128.cpp | 1 - datatypes/mcs_int128.h | 1 - datatypes/mcs_int64.h | 1 - datatypes/mcs_longdouble.h | 1 - dbcon/ddlpackage/CMakeLists.txt | 2 +- dbcon/ddlpackageproc/altertableprocessor.cpp | 1 - dbcon/ddlpackageproc/createtableprocessor.cpp | 1 - dbcon/ddlpackageproc/ddlpackageprocessor.cpp | 1 - dbcon/ddlpackageproc/ddlpackageprocessor.h | 1 - dbcon/ddlpackageproc/droptableprocessor.cpp | 1 - dbcon/dmlpackage/CMakeLists.txt | 2 +- dbcon/dmlpackageproc/autoincrementdata.cpp | 1 - dbcon/dmlpackageproc/autoincrementdata.h | 1 - dbcon/dmlpackageproc/dmlpackageprocessor.cpp | 1 - dbcon/dmlpackageproc/insertpackageprocessor.cpp | 1 - dbcon/dmlpackageproc/tablelockdata.cpp | 1 - dbcon/dmlpackageproc/tablelockdata.h | 1 - dbcon/execplan/calpontselectexecutionplan.h | 1 - dbcon/execplan/calpontsystemcatalog.h | 1 - dbcon/execplan/clientrotator.cpp | 1 - dbcon/execplan/clientrotator.h | 1 - dbcon/execplan/constantcolumn.cpp | 1 - dbcon/execplan/constantfilter.cpp | 1 - dbcon/execplan/sessionmanager.h | 1 - dbcon/joblist/anydatalist.cpp | 1 - dbcon/joblist/batchprimitiveprocessor-jl.h | 1 - dbcon/joblist/columncommand-jl.h | 1 - dbcon/joblist/crossenginestep.cpp | 1 - dbcon/joblist/crossenginestep.h | 1 - dbcon/joblist/diskjoinstep.cpp | 4 ++-- dbcon/joblist/distributedenginecomm.cpp | 1 - dbcon/joblist/elementtype.h | 1 - dbcon/joblist/errorinfo.h | 1 - dbcon/joblist/expressionstep.cpp | 1 - dbcon/joblist/fifo.h | 1 - dbcon/joblist/groupconcat.cpp | 1 - dbcon/joblist/jlf_common.cpp | 1 - dbcon/joblist/jlf_execplantojoblist.cpp | 1 - dbcon/joblist/jlf_graphics.cpp | 1 - dbcon/joblist/jlf_subquery.cpp | 1 - dbcon/joblist/jlf_tuplejoblist.cpp | 1 - dbcon/joblist/joblist.cpp | 1 - dbcon/joblist/joblist.h | 1 - dbcon/joblist/joblistfactory.cpp | 1 - dbcon/joblist/jobstep.cpp | 1 - dbcon/joblist/jobstep.h | 1 - dbcon/joblist/lbidlist.cpp | 1 - dbcon/joblist/limitedorderby.cpp | 1 - dbcon/joblist/passthrucommand-jl.cpp | 1 - dbcon/joblist/pcolscan.cpp | 1 - dbcon/joblist/pcolstep.cpp | 1 - dbcon/joblist/pdictionaryscan.cpp | 1 - dbcon/joblist/primitivemsg.h | 1 - dbcon/joblist/primitivestep.h | 1 - dbcon/joblist/rowestimator.cpp | 4 ++-- dbcon/joblist/subquerystep.cpp | 1 - dbcon/joblist/subquerystep.h | 1 - dbcon/joblist/subquerytransformer.cpp | 1 - dbcon/joblist/subquerytransformer.h | 1 - dbcon/joblist/tuple-bps.cpp | 1 - dbcon/joblist/tupleaggregatestep.cpp | 1 - dbcon/joblist/tupleaggregatestep.h | 1 - dbcon/joblist/tupleannexstep.cpp | 1 - dbcon/joblist/tupleannexstep.h | 1 - dbcon/joblist/tupleconstantstep.cpp | 1 - dbcon/joblist/tupleconstantstep.h | 1 - dbcon/joblist/tuplehashjoin.cpp | 1 - dbcon/joblist/tuplehashjoin.h | 1 - dbcon/joblist/tuplehavingstep.cpp | 1 - dbcon/joblist/tuplehavingstep.h | 1 - dbcon/joblist/virtualtable.cpp | 1 - dbcon/joblist/windowfunctionstep.cpp | 1 - dbcon/joblist/windowfunctionstep.h | 1 - dbcon/mysql/ha_autoi.cpp | 1 - dbcon/mysql/ha_mcs_datatype.h | 1 - dbcon/mysql/ha_mcs_ddl.cpp | 1 - dbcon/mysql/ha_mcs_dml.cpp | 1 - dbcon/mysql/ha_mcs_execplan.cpp | 1 - dbcon/mysql/ha_pseudocolumn.cpp | 1 - dbcon/mysql/idb_mysql.h | 1 - ddlproc/ddlproc.cpp | 1 - ddlproc/ddlprocessor.cpp | 1 - dmlproc/batchinsertprocessor.cpp | 1 - dmlproc/batchinsertprocessor.h | 1 - dmlproc/dmlproc.cpp | 10 +--------- dmlproc/dmlprocessor.cpp | 1 - dmlproc/dmlprocessor.h | 1 - exemgr/activestatementcounter.cpp | 1 - exemgr/activestatementcounter.h | 1 - exemgr/main.cpp | 1 - oam/oamcpp/liboamcpp.cpp | 1 - oam/oamcpp/liboamcpp.h | 1 - oamapps/columnstoreDB/columnstoreDB.cpp | 2 +- oamapps/postConfigure/mycnfUpgrade.cpp | 1 - oamapps/sessionWalker/sessionwalker.cpp | 1 - primitives/blockcache/blockcacheclient.h | 1 - primitives/blockcache/blockrequestprocessor.h | 1 - primitives/blockcache/filerequest.h | 1 - primitives/blockcache/iomanager.cpp | 1 - primitives/blockcache/iomanager.h | 1 - primitives/linux-port/column.cpp | 1 - primitives/linux-port/dictionary.cpp | 1 - primitives/linux-port/index.cpp | 1 - primitives/linux-port/primitiveprocessor.h | 1 - primitives/primproc/batchprimitiveprocessor.cpp | 1 - primitives/primproc/columncommand.cpp | 1 - primitives/primproc/columncommand.h | 1 - primitives/primproc/primitiveserver.cpp | 1 - primitives/primproc/primproc.cpp | 1 - primitives/primproc/udf.cpp | 1 - storage-manager/src/smcat.cpp | 2 +- storage-manager/src/unit_tests.cpp | 4 ++-- tests/primitives_column_scan_and_filter.cpp | 1 - tests/primitives_scan_bench.cpp | 1 - tools/bincvt/li2bin.cpp | 1 - tools/clearShm/main.cpp | 1 - tools/dbbuilder/dbbuilder.cpp | 1 - tools/ddlcleanup/ddlcleanup.cpp | 1 - tools/editem/editem.cpp | 1 - tools/qfe/server.cpp | 1 - tools/sendPlan/sendplan.cpp | 1 - tools/setConfig/main.cpp | 1 - utils/cacheutils/cacheutils.cpp | 1 - utils/cacheutils/cacheutils.h | 1 - utils/common/hashfamily.h | 1 - utils/common/mcs_basic_types.h | 1 - utils/common/simd_sse.h | 1 - utils/compress/idbcompress.cpp | 1 - utils/configcpp/configcpp.cpp | 1 - utils/configcpp/configcpp.h | 1 - utils/configcpp/configstream.cpp | 1 - utils/configcpp/configstream.h | 1 - utils/configcpp/xmlparser.cpp | 1 - utils/configcpp/xmlparser.h | 1 - utils/dataconvert/dataconvert.cpp | 3 +-- utils/ddlcleanup/ddlcleanuputil.cpp | 1 - utils/ddlcleanup/ddlcleanuputil.h | 1 - utils/funcexp/func_abs.cpp | 1 - utils/funcexp/func_add_time.cpp | 1 - utils/funcexp/func_ascii.cpp | 1 - utils/funcexp/func_between.cpp | 1 - utils/funcexp/func_bitwise.cpp | 1 - utils/funcexp/func_case.cpp | 1 - utils/funcexp/func_cast.cpp | 1 - utils/funcexp/func_ceil.cpp | 1 - utils/funcexp/func_char.cpp | 1 - utils/funcexp/func_char_length.cpp | 1 - utils/funcexp/func_coalesce.cpp | 1 - utils/funcexp/func_concat.cpp | 1 - utils/funcexp/func_concat_oracle.cpp | 1 - utils/funcexp/func_concat_ws.cpp | 1 - utils/funcexp/func_conv.cpp | 1 - utils/funcexp/func_convert_tz.cpp | 1 - utils/funcexp/func_crc32.cpp | 1 - utils/funcexp/func_date.cpp | 1 - utils/funcexp/func_date_add.cpp | 1 - utils/funcexp/func_date_format.cpp | 1 - utils/funcexp/func_day.cpp | 1 - utils/funcexp/func_dayname.cpp | 1 - utils/funcexp/func_dayofweek.cpp | 1 - utils/funcexp/func_dayofyear.cpp | 1 - utils/funcexp/func_div.cpp | 1 - utils/funcexp/func_elt.cpp | 1 - utils/funcexp/func_exp.cpp | 1 - utils/funcexp/func_extract.cpp | 1 - utils/funcexp/func_find_in_set.cpp | 1 - utils/funcexp/func_floor.cpp | 1 - utils/funcexp/func_from_days.cpp | 1 - utils/funcexp/func_from_unixtime.cpp | 1 - utils/funcexp/func_get_format.cpp | 1 - utils/funcexp/func_greatest.cpp | 1 - utils/funcexp/func_hex.cpp | 1 - utils/funcexp/func_hour.cpp | 1 - utils/funcexp/func_idbpartition.cpp | 1 - utils/funcexp/func_if.cpp | 1 - utils/funcexp/func_ifnull.cpp | 1 - utils/funcexp/func_in.cpp | 1 - utils/funcexp/func_insert.cpp | 1 - utils/funcexp/func_instr.cpp | 1 - utils/funcexp/func_isnull.cpp | 1 - utils/funcexp/func_last_day.cpp | 1 - utils/funcexp/func_lcase.cpp | 1 - utils/funcexp/func_least.cpp | 1 - utils/funcexp/func_left.cpp | 1 - utils/funcexp/func_length.cpp | 1 - utils/funcexp/func_lpad.cpp | 1 - utils/funcexp/func_ltrim.cpp | 1 - utils/funcexp/func_ltrim_oracle.cpp | 1 - utils/funcexp/func_makedate.cpp | 1 - utils/funcexp/func_maketime.cpp | 1 - utils/funcexp/func_math.cpp | 1 - utils/funcexp/func_md5.cpp | 1 - utils/funcexp/func_microsecond.cpp | 1 - utils/funcexp/func_minute.cpp | 1 - utils/funcexp/func_mod.cpp | 1 - utils/funcexp/func_month.cpp | 1 - utils/funcexp/func_monthname.cpp | 1 - utils/funcexp/func_nullif.cpp | 3 +-- utils/funcexp/func_period_add.cpp | 1 - utils/funcexp/func_period_diff.cpp | 1 - utils/funcexp/func_pow.cpp | 1 - utils/funcexp/func_quarter.cpp | 1 - utils/funcexp/func_quote.cpp | 1 - utils/funcexp/func_rand.cpp | 1 - utils/funcexp/func_regexp.cpp | 1 - utils/funcexp/func_repeat.cpp | 1 - utils/funcexp/func_replace.cpp | 1 - utils/funcexp/func_replace_oracle.cpp | 1 - utils/funcexp/func_reverse.cpp | 1 - utils/funcexp/func_right.cpp | 1 - utils/funcexp/func_round.cpp | 3 +-- utils/funcexp/func_rpad.cpp | 1 - utils/funcexp/func_rtrim.cpp | 1 - utils/funcexp/func_rtrim_oracle.cpp | 1 - utils/funcexp/func_sec_to_time.cpp | 1 - utils/funcexp/func_second.cpp | 1 - utils/funcexp/func_sha.cpp | 1 - utils/funcexp/func_sign.cpp | 1 - utils/funcexp/func_space.cpp | 1 - utils/funcexp/func_str_to_date.cpp | 1 - utils/funcexp/func_strcmp.cpp | 1 - utils/funcexp/func_substr.cpp | 1 - utils/funcexp/func_substring_index.cpp | 1 - utils/funcexp/func_sysdate.cpp | 1 - utils/funcexp/func_time.cpp | 1 - utils/funcexp/func_time_format.cpp | 1 - utils/funcexp/func_time_to_sec.cpp | 1 - utils/funcexp/func_timediff.cpp | 1 - utils/funcexp/func_timestampdiff.cpp | 1 - utils/funcexp/func_to_days.cpp | 1 - utils/funcexp/func_trim.cpp | 1 - utils/funcexp/func_trim_oracle.cpp | 1 - utils/funcexp/func_truncate.cpp | 5 ++--- utils/funcexp/func_ucase.cpp | 1 - utils/funcexp/func_unhex.cpp | 1 - utils/funcexp/func_unix_timestamp.cpp | 1 - utils/funcexp/func_week.cpp | 1 - utils/funcexp/func_weekday.cpp | 1 - utils/funcexp/func_year.cpp | 1 - utils/funcexp/func_yearweek.cpp | 1 - utils/funcexp/functor.cpp | 1 - utils/libmysql_client/libmysql_client.h | 1 - utils/loggingcpp/errorcodes.cpp | 1 - utils/loggingcpp/exceptclasses.h | 1 - utils/messageqcpp/inetstreamsocket.cpp | 12 +----------- utils/messageqcpp/messagequeue.h | 1 - utils/multicast/multicast.cpp | 1 - utils/regr/regrmysql.cpp | 1 - utils/rowgroup/rowgroup.cpp | 1 - utils/rowgroup/rowgroup.h | 1 - utils/rwlock/rwlock.cpp | 1 - utils/rwlock/rwlock.h | 1 - utils/startup/installdir.cpp | 1 - utils/startup/installdir.h | 1 - utils/testbc/iomanager.cpp | 1 - utils/testbc/iomanager.h | 1 - utils/threadpool/prioritythreadpool.cpp | 1 - utils/udfsdk/udfmysql.cpp | 1 - utils/udfsdk/udfsdk.cpp | 1 - utils/udfsdk/udfsdk.h | 1 - utils/windowfunction/framebound.cpp | 1 - utils/windowfunction/framebound.h | 1 - utils/windowfunction/frameboundrange.cpp | 1 - utils/windowfunction/frameboundrange.h | 1 - utils/windowfunction/frameboundrow.cpp | 1 - utils/windowfunction/frameboundrow.h | 1 - utils/windowfunction/idborderby.cpp | 1 - utils/windowfunction/wf_count.cpp | 1 - utils/windowfunction/wf_count.h | 1 - utils/windowfunction/wf_lead_lag.cpp | 1 - utils/windowfunction/wf_lead_lag.h | 1 - utils/windowfunction/wf_min_max.cpp | 1 - utils/windowfunction/wf_min_max.h | 1 - utils/windowfunction/wf_nth_value.cpp | 1 - utils/windowfunction/wf_nth_value.h | 1 - utils/windowfunction/wf_ntile.cpp | 1 - utils/windowfunction/wf_ntile.h | 1 - utils/windowfunction/wf_percentile.cpp | 1 - utils/windowfunction/wf_percentile.h | 1 - utils/windowfunction/wf_ranking.cpp | 1 - utils/windowfunction/wf_ranking.h | 1 - utils/windowfunction/wf_row_number.cpp | 1 - utils/windowfunction/wf_row_number.h | 1 - utils/windowfunction/wf_stats.cpp | 1 - utils/windowfunction/wf_stats.h | 1 - utils/windowfunction/wf_sum_avg.cpp | 1 - utils/windowfunction/wf_sum_avg.h | 1 - utils/windowfunction/wf_udaf.cpp | 1 - utils/windowfunction/wf_udaf.h | 1 - utils/windowfunction/windowframe.cpp | 1 - utils/windowfunction/windowframe.h | 1 - utils/windowfunction/windowfunction.cpp | 1 - utils/windowfunction/windowfunction.h | 1 - utils/windowfunction/windowfunctiontype.cpp | 1 - utils/windowfunction/windowfunctiontype.h | 1 - versioning/BRM/brmshmimpl.cpp | 1 - versioning/BRM/extentmap.cpp | 1 - versioning/BRM/load_brm.cpp | 1 - versioning/BRM/slavecomm.cpp | 1 - versioning/BRM/vbbm.cpp | 5 ++--- writeengine/client/we_clients.cpp | 1 - writeengine/client/we_clients.h | 1 - writeengine/redistribute/we_redistribute.cpp | 1 - writeengine/redistribute/we_redistribute.h | 1 - writeengine/redistribute/we_redistributecontrol.cpp | 1 - writeengine/redistribute/we_redistributecontrol.h | 1 - .../redistribute/we_redistributecontrolthread.cpp | 1 - .../redistribute/we_redistributecontrolthread.h | 1 - writeengine/redistribute/we_redistributedef.h | 1 - .../redistribute/we_redistributeworkerthread.cpp | 1 - .../redistribute/we_redistributeworkerthread.h | 1 - writeengine/server/we_ddlcommandproc.cpp | 1 - writeengine/shared/we_brm.cpp | 1 - writeengine/shared/we_chunkmanager.cpp | 1 - writeengine/splitter/we_splitterapp.cpp | 1 - writeengine/wrapper/we_colop.cpp | 1 - writeengine/wrapper/we_tablemetadata.cpp | 1 - writeengine/wrapper/we_tablemetadata.h | 1 - writeengine/wrapper/writeengine.cpp | 1 - 325 files changed, 20 insertions(+), 353 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 079d9abfc..0b6fbf260 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,7 +223,7 @@ IF (MASK_LONGDOUBLE) ENDIF() -SET (CMAKE_REQUIRED_FLAGS "-Werror -Wall") +MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall") SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") SET (ENGINE_DT_LIB datatypes) SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${ENGINE_DT_LIB}) diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp index 59bc2bf21..2ca95ba2c 100644 --- a/datatypes/mcs_datatype.cpp +++ b/datatypes/mcs_datatype.cpp @@ -1915,4 +1915,3 @@ const uint8_t* TypeHandlerUDecimal128::getEmptyValueForType( } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index a2686b45e..9c58214bb 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -2521,4 +2521,3 @@ class TypeHandlerTimestamp : public TypeHandlerTemporal } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_datatype_basic.h b/datatypes/mcs_datatype_basic.h index 1056c34e9..cd3327997 100644 --- a/datatypes/mcs_datatype_basic.h +++ b/datatypes/mcs_datatype_basic.h @@ -78,4 +78,3 @@ uint64_t xFloatToMCSUInt64Round(SRC value) } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_double.h b/datatypes/mcs_double.h index b87bf4242..ca05ea757 100644 --- a/datatypes/mcs_double.h +++ b/datatypes/mcs_double.h @@ -54,4 +54,3 @@ class TDouble } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_float128.h b/datatypes/mcs_float128.h index 0ea0c521e..2a97dadad 100644 --- a/datatypes/mcs_float128.h +++ b/datatypes/mcs_float128.h @@ -724,4 +724,3 @@ class TFloat128 } // namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_int128.cpp b/datatypes/mcs_int128.cpp index 13329fc41..58dee474c 100644 --- a/datatypes/mcs_int128.cpp +++ b/datatypes/mcs_int128.cpp @@ -113,4 +113,3 @@ std::ostream& operator<<(std::ostream& os, const TSInt128& x) } } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_int128.h b/datatypes/mcs_int128.h index f7949fcda..1dd5d1e35 100644 --- a/datatypes/mcs_int128.h +++ b/datatypes/mcs_int128.h @@ -322,4 +322,3 @@ class TSInt128 } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_int64.h b/datatypes/mcs_int64.h index 3737cff95..76ec1e399 100644 --- a/datatypes/mcs_int64.h +++ b/datatypes/mcs_int64.h @@ -179,4 +179,3 @@ class TSInt64Null : public TSInt64, public TNullFlag } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/datatypes/mcs_longdouble.h b/datatypes/mcs_longdouble.h index 07995d537..f0b68d835 100644 --- a/datatypes/mcs_longdouble.h +++ b/datatypes/mcs_longdouble.h @@ -54,4 +54,3 @@ class TLongDouble } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index d2e0b4d74..5557ed3ff 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -9,7 +9,7 @@ FIND_PACKAGE(FLEX REQUIRED) FLEX_TARGET(ddl_scan ddl.l ${CMAKE_CURRENT_BINARY_DIR}/ddl-scan.cpp COMPILE_FLAGS "-i -L -Pddl") ADD_FLEX_BISON_DEPENDENCY(ddl_scan ddl_gram) -set_source_files_properties(ddl-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-sign-compare -DYY_NO_INPUT") +set_source_files_properties(ddl-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-register -Wno-deprecated-register -Wno-sign-compare -DYY_NO_INPUT") ########### next target ############### diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index aca76bf4b..c09c79b48 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -2539,7 +2539,6 @@ void AlterTableProcessor::renameColumn(uint32_t sessionID, execplan::CalpontSyst } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/ddlpackageproc/createtableprocessor.cpp b/dbcon/ddlpackageproc/createtableprocessor.cpp index f51d03958..f0ff32be7 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.cpp +++ b/dbcon/ddlpackageproc/createtableprocessor.cpp @@ -830,4 +830,3 @@ void CreateTableProcessor::rollBackCreateTable(const string& error, BRM::TxnID t } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 3740035d6..bf27af32f 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -1491,4 +1491,3 @@ int DDLPackageProcessor::commitTransaction(uint64_t uniqueId, BRM::TxnID txnID) } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.h b/dbcon/ddlpackageproc/ddlpackageprocessor.h index 9e25ab1d2..047b99d95 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.h +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.h @@ -890,4 +890,3 @@ bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&) #undef EXPORT -// vim:ts=4 sw=4: diff --git a/dbcon/ddlpackageproc/droptableprocessor.cpp b/dbcon/ddlpackageproc/droptableprocessor.cpp index e1ca785d9..4cfc79467 100644 --- a/dbcon/ddlpackageproc/droptableprocessor.cpp +++ b/dbcon/ddlpackageproc/droptableprocessor.cpp @@ -1367,4 +1367,3 @@ TruncTableProcessor::DDLResult TruncTableProcessor::processPackage( } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackage/CMakeLists.txt b/dbcon/dmlpackage/CMakeLists.txt index 63825be10..e999cdae1 100644 --- a/dbcon/dmlpackage/CMakeLists.txt +++ b/dbcon/dmlpackage/CMakeLists.txt @@ -10,7 +10,7 @@ FIND_PACKAGE(FLEX REQUIRED) FLEX_TARGET(dml_scan dml.l ${CMAKE_CURRENT_BINARY_DIR}/dml-scan.cpp COMPILE_FLAGS "-i -L -Pdml") ADD_FLEX_BISON_DEPENDENCY(dml_scan dml_gram) -set_source_files_properties(dml-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-sign-compare -DYY_NO_INPUT") +set_source_files_properties(dml-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-register -Wno-deprecated-register -Wno-sign-compare -DYY_NO_INPUT") ########### next target ############### diff --git a/dbcon/dmlpackageproc/autoincrementdata.cpp b/dbcon/dmlpackageproc/autoincrementdata.cpp index 3a71efaf6..875e90d7e 100644 --- a/dbcon/dmlpackageproc/autoincrementdata.cpp +++ b/dbcon/dmlpackageproc/autoincrementdata.cpp @@ -97,4 +97,3 @@ AutoincrementData::OIDNextValue& AutoincrementData::getOidNextValueMap() return fOidNextValueMap; } -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/autoincrementdata.h b/dbcon/dmlpackageproc/autoincrementdata.h index 9237a2709..7dcd2a200 100644 --- a/dbcon/dmlpackageproc/autoincrementdata.h +++ b/dbcon/dmlpackageproc/autoincrementdata.h @@ -49,4 +49,3 @@ class AutoincrementData boost::mutex fOIDnextvalLock; }; -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp index b5217b0e6..78b19b392 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp @@ -913,4 +913,3 @@ int DMLPackageProcessor::endTransaction(uint64_t uniqueId, BRM::TxnID txnID, boo return rc; } } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/insertpackageprocessor.cpp b/dbcon/dmlpackageproc/insertpackageprocessor.cpp index fb844b696..3fd6b7ae0 100644 --- a/dbcon/dmlpackageproc/insertpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/insertpackageprocessor.cpp @@ -432,4 +432,3 @@ DMLPackageProcessor::DMLResult InsertPackageProcessor::processPackage(dmlpackage } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/tablelockdata.cpp b/dbcon/dmlpackageproc/tablelockdata.cpp index d5f94ec60..1c1699172 100644 --- a/dbcon/dmlpackageproc/tablelockdata.cpp +++ b/dbcon/dmlpackageproc/tablelockdata.cpp @@ -101,4 +101,3 @@ TablelockData::OIDTablelock& TablelockData::getOidTablelockMap() } } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/dbcon/dmlpackageproc/tablelockdata.h b/dbcon/dmlpackageproc/tablelockdata.h index 0cd7e6634..82dd94a55 100644 --- a/dbcon/dmlpackageproc/tablelockdata.h +++ b/dbcon/dmlpackageproc/tablelockdata.h @@ -61,4 +61,3 @@ class TablelockData #undef EXPORT -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/calpontselectexecutionplan.h b/dbcon/execplan/calpontselectexecutionplan.h index e870cf661..10093f8ce 100644 --- a/dbcon/execplan/calpontselectexecutionplan.h +++ b/dbcon/execplan/calpontselectexecutionplan.h @@ -939,4 +939,3 @@ inline std::ostream& operator<<(std::ostream& os, const CalpontSelectExecutionPl } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index f4c69f803..3492b8d16 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -1279,4 +1279,3 @@ bool ctListSort(const CalpontSystemCatalog::ColType& a, const CalpontSystemCatal } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/clientrotator.cpp b/dbcon/execplan/clientrotator.cpp index 7616f90ed..395d76482 100644 --- a/dbcon/execplan/clientrotator.cpp +++ b/dbcon/execplan/clientrotator.cpp @@ -399,4 +399,3 @@ void ClientRotator::writeToLog(int line, const string& msg, bool critical) const } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/clientrotator.h b/dbcon/execplan/clientrotator.h index d71109441..273863c53 100644 --- a/dbcon/execplan/clientrotator.h +++ b/dbcon/execplan/clientrotator.h @@ -166,4 +166,3 @@ class ClientRotator }; } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index a3094285c..bcdc67676 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -347,4 +347,3 @@ bool ConstantColumn::operator!=(const TreeNode* t) const } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/constantfilter.cpp b/dbcon/execplan/constantfilter.cpp index 4f837d0ad..18f872e03 100644 --- a/dbcon/execplan/constantfilter.cpp +++ b/dbcon/execplan/constantfilter.cpp @@ -308,4 +308,3 @@ void ConstantFilter::setSimpleColumnList() } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/execplan/sessionmanager.h b/dbcon/execplan/sessionmanager.h index be60a5b6c..2e46b0cd6 100644 --- a/dbcon/execplan/sessionmanager.h +++ b/dbcon/execplan/sessionmanager.h @@ -213,4 +213,3 @@ class SessionManager } // namespace execplan -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/anydatalist.cpp b/dbcon/joblist/anydatalist.cpp index 4e5df4713..da1ae05a2 100644 --- a/dbcon/joblist/anydatalist.cpp +++ b/dbcon/joblist/anydatalist.cpp @@ -170,4 +170,3 @@ std::ostream& omitOidInDL(std::ostream& strm) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index 6dbc76016..9cddd8945 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -366,4 +366,3 @@ class BatchPrimitiveProcessorJL } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/columncommand-jl.h b/dbcon/joblist/columncommand-jl.h index 407bef1cc..ad36f72aa 100644 --- a/dbcon/joblist/columncommand-jl.h +++ b/dbcon/joblist/columncommand-jl.h @@ -129,4 +129,3 @@ class ColumnCommandJL : public CommandJL } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index e0475c113..d03f21e0d 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -828,4 +828,3 @@ void CrossEngineStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/crossenginestep.h b/dbcon/joblist/crossenginestep.h index d3d1a4dba..ac657f9b5 100644 --- a/dbcon/joblist/crossenginestep.h +++ b/dbcon/joblist/crossenginestep.h @@ -249,4 +249,3 @@ class CrossEngineStep : public BatchPrimitive, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/diskjoinstep.cpp b/dbcon/joblist/diskjoinstep.cpp index c10f78376..f218d6e8e 100644 --- a/dbcon/joblist/diskjoinstep.cpp +++ b/dbcon/joblist/diskjoinstep.cpp @@ -163,7 +163,7 @@ void DiskJoinStep::smallReader() RGData rgData; bool more = true; int64_t memUsage = 0, combinedMemUsage = 0; - int rowCount = 0; + [[maybe_unused]] int rowCount = 0; RowGroup l_smallRG = smallRG; try @@ -224,7 +224,7 @@ void DiskJoinStep::largeReader() RGData rgData; bool more = true; int64_t largeSize = 0; - int rowCount = 0; + [[maybe_unused]] int rowCount = 0; RowGroup l_largeRG = largeRG; largeIterationCount++; diff --git a/dbcon/joblist/distributedenginecomm.cpp b/dbcon/joblist/distributedenginecomm.cpp index 5d0bcb694..67f40f268 100644 --- a/dbcon/joblist/distributedenginecomm.cpp +++ b/dbcon/joblist/distributedenginecomm.cpp @@ -1146,4 +1146,3 @@ uint32_t DistributedEngineComm::MQE::getNextConnectionId(const size_t pmIndex, } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/elementtype.h b/dbcon/joblist/elementtype.h index 742944ffc..a91a1f062 100644 --- a/dbcon/joblist/elementtype.h +++ b/dbcon/joblist/elementtype.h @@ -638,4 +638,3 @@ extern std::ostream& omitOidInDL(std::ostream& strm); #endif -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/errorinfo.h b/dbcon/joblist/errorinfo.h index 88d1d0128..62b404daa 100644 --- a/dbcon/joblist/errorinfo.h +++ b/dbcon/joblist/errorinfo.h @@ -52,4 +52,3 @@ typedef boost::shared_ptr SErrorInfo; } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 2f5b96afc..b182c7b21 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -790,4 +790,3 @@ const string ExpressionStep::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/fifo.h b/dbcon/joblist/fifo.h index d73ebc937..46ebf9922 100644 --- a/dbcon/joblist/fifo.h +++ b/dbcon/joblist/fifo.h @@ -529,4 +529,3 @@ void FIFO::totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) co } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 7b47799f0..26e8dd023 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -1051,4 +1051,3 @@ const string GroupConcatNoOrder::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 6ae8724e6..47a1ecc60 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -822,4 +822,3 @@ bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_ } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 4ba29aa56..441ee1ea4 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -3446,7 +3446,6 @@ void JLF_ExecPlanToJobList::addJobSteps(JobStepVector& nsv, JobInfo& jobInfo, bo } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/joblist/jlf_graphics.cpp b/dbcon/joblist/jlf_graphics.cpp index 7365e1c4e..348fb85d6 100644 --- a/dbcon/joblist/jlf_graphics.cpp +++ b/dbcon/joblist/jlf_graphics.cpp @@ -418,7 +418,6 @@ ostream& writeDotCmds(ostream& dotFile, const JobStepVector& query, const JobSte } // end namespace jlf_graphics -// vim:ts=4 sw=4 syntax=cpp: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index 91d8a5de3..0ea66ac0f 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -875,4 +875,3 @@ SJSTEP doUnionSub(CalpontExecutionPlan* ep, JobInfo& jobInfo) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 7678d2788..d388ce9f0 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -4559,7 +4559,6 @@ SJSTEP unionQueries(JobStepVector& queries, uint64_t distinctUnionNum, JobInfo& } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/joblist/joblist.cpp b/dbcon/joblist/joblist.cpp index fd58bde04..7adc6a980 100644 --- a/dbcon/joblist/joblist.cpp +++ b/dbcon/joblist/joblist.cpp @@ -1233,4 +1233,3 @@ void TupleJobList::abort() #pragma clang diagnostic pop #endif -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/joblist.h b/dbcon/joblist/joblist.h index 1ccaa6dd7..24df23beb 100644 --- a/dbcon/joblist/joblist.h +++ b/dbcon/joblist/joblist.h @@ -260,4 +260,3 @@ typedef boost::shared_ptr STJLP; #undef EXPORT -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 06bdf5920..60985cc8f 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -2316,7 +2316,6 @@ SJLP JobListFactory::makeJobList(CalpontExecutionPlan* cplan, ResourceManager* r } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/dbcon/joblist/jobstep.cpp b/dbcon/joblist/jobstep.cpp index 36d13f35e..8f42dff10 100644 --- a/dbcon/joblist/jobstep.cpp +++ b/dbcon/joblist/jobstep.cpp @@ -244,4 +244,3 @@ void JobStep::handleException(std::exception_ptr e, const int errorCode, const u } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/jobstep.h b/dbcon/joblist/jobstep.h index ce051dcaf..79d1b350d 100644 --- a/dbcon/joblist/jobstep.h +++ b/dbcon/joblist/jobstep.h @@ -566,4 +566,3 @@ typedef boost::shared_ptr SJSTEP; } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 26080f0c2..e784eb26f 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -920,4 +920,3 @@ template bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 09bf8d5de..95e398d24 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -303,4 +303,3 @@ const string LimitedOrderBy::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/passthrucommand-jl.cpp b/dbcon/joblist/passthrucommand-jl.cpp index b512e37b3..84e63f9c5 100644 --- a/dbcon/joblist/passthrucommand-jl.cpp +++ b/dbcon/joblist/passthrucommand-jl.cpp @@ -110,4 +110,3 @@ uint16_t PassThruCommandJL::getWidth() } }; // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index 53dfbcfb0..6f5d51448 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -1224,4 +1224,3 @@ void pColScanStep::appendFilter(const std::vector& fs) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index eefaf02e1..125491ebd 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -1511,4 +1511,3 @@ void pColStep::appendFilter(const std::vector& fs) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/pdictionaryscan.cpp b/dbcon/joblist/pdictionaryscan.cpp index df795b862..94f70ba2e 100644 --- a/dbcon/joblist/pdictionaryscan.cpp +++ b/dbcon/joblist/pdictionaryscan.cpp @@ -916,4 +916,3 @@ void pDictionaryScan::abort() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index 0d495b891..fe0c52ec2 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -884,4 +884,3 @@ struct LbidAtVer #pragma pack(pop) -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index 4683abd69..3bcbd8582 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -1835,4 +1835,3 @@ class PseudoColStep : public pColStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index 9babae7c3..fa7e8920c 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -269,8 +269,8 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, const m float tempFactor = 1.0; uint64_t adjustedMin = 0, adjustedMax = 0; - uint128_t adjustedBigMin, adjustedBigMax; - uint32_t distinctValuesEstimate; + uint128_t adjustedBigMin = 0, adjustedBigMax = 0; + uint32_t distinctValuesEstimate = 0; // Adjust values based on column type and estimate the if (!ct.isWideDecimalType()) diff --git a/dbcon/joblist/subquerystep.cpp b/dbcon/joblist/subquerystep.cpp index a019c24f8..fbc46912e 100644 --- a/dbcon/joblist/subquerystep.cpp +++ b/dbcon/joblist/subquerystep.cpp @@ -533,4 +533,3 @@ void SubAdapterStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/subquerystep.h b/dbcon/joblist/subquerystep.h index 320e4a438..3c587d170 100644 --- a/dbcon/joblist/subquerystep.h +++ b/dbcon/joblist/subquerystep.h @@ -273,4 +273,3 @@ class SubAdapterStep : public JobStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index 816b6aa72..48f1cc2d5 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -622,4 +622,3 @@ void SimpleScalarTransformer::getScalarResult() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/subquerytransformer.h b/dbcon/joblist/subquerytransformer.h index 26e6e0f29..5f9ea4e86 100644 --- a/dbcon/joblist/subquerytransformer.h +++ b/dbcon/joblist/subquerytransformer.h @@ -223,4 +223,3 @@ class SimpleScalarTransformer : public SubQueryTransformer } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index e722f92af..532ce54ba 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -3356,4 +3356,3 @@ template bool TupleBPS::compareSingleValue(uint8_t COP, int64_t val1, i template bool TupleBPS::compareSingleValue(uint8_t COP, int128_t val1, int128_t val2) const; } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 3b64e6ca1..e7a7791a8 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5962,4 +5962,3 @@ void TupleAggregateStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleaggregatestep.h b/dbcon/joblist/tupleaggregatestep.h index 92ff9a1eb..b81c8497d 100644 --- a/dbcon/joblist/tupleaggregatestep.h +++ b/dbcon/joblist/tupleaggregatestep.h @@ -224,4 +224,3 @@ class TupleAggregateStep : public JobStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleannexstep.cpp b/dbcon/joblist/tupleannexstep.cpp index f6a7d70c2..b93e41378 100644 --- a/dbcon/joblist/tupleannexstep.cpp +++ b/dbcon/joblist/tupleannexstep.cpp @@ -1258,4 +1258,3 @@ void TupleAnnexStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleannexstep.h b/dbcon/joblist/tupleannexstep.h index 3d043afce..dcdc064c3 100644 --- a/dbcon/joblist/tupleannexstep.h +++ b/dbcon/joblist/tupleannexstep.h @@ -193,4 +193,3 @@ class reservablePQ : private std::priority_queue } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index ee252ec0f..065b087df 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -842,4 +842,3 @@ const string TupleConstantBooleanStep::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleconstantstep.h b/dbcon/joblist/tupleconstantstep.h index be3480594..ad36b5c9a 100644 --- a/dbcon/joblist/tupleconstantstep.h +++ b/dbcon/joblist/tupleconstantstep.h @@ -189,4 +189,3 @@ class TupleConstantBooleanStep : public TupleConstantStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 930d1c82f..a68b4641c 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -1997,4 +1997,3 @@ void TupleHashJoinStep::abort() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehashjoin.h b/dbcon/joblist/tuplehashjoin.h index 9191de337..aca8a068c 100644 --- a/dbcon/joblist/tuplehashjoin.h +++ b/dbcon/joblist/tuplehashjoin.h @@ -633,4 +633,3 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehavingstep.cpp b/dbcon/joblist/tuplehavingstep.cpp index 701e77b39..60496895a 100644 --- a/dbcon/joblist/tuplehavingstep.cpp +++ b/dbcon/joblist/tuplehavingstep.cpp @@ -408,4 +408,3 @@ void TupleHavingStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehavingstep.h b/dbcon/joblist/tuplehavingstep.h index 1d456b609..b4c185e31 100644 --- a/dbcon/joblist/tuplehavingstep.h +++ b/dbcon/joblist/tuplehavingstep.h @@ -115,4 +115,3 @@ class TupleHavingStep : public ExpressionStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/virtualtable.cpp b/dbcon/joblist/virtualtable.cpp index dc9bf275c..8fe1ec674 100644 --- a/dbcon/joblist/virtualtable.cpp +++ b/dbcon/joblist/virtualtable.cpp @@ -158,4 +158,3 @@ const CalpontSystemCatalog::ColType& VirtualTable::columnType(uint32_t i) const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index e4c56ecd9..2883d1caa 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1597,4 +1597,3 @@ void WindowFunctionStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/joblist/windowfunctionstep.h b/dbcon/joblist/windowfunctionstep.h index 2c839a5e6..d1da2aafd 100644 --- a/dbcon/joblist/windowfunctionstep.h +++ b/dbcon/joblist/windowfunctionstep.h @@ -225,4 +225,3 @@ class WindowFunctionStep : public JobStep, public TupleDeliveryStep } // namespace joblist -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_autoi.cpp b/dbcon/mysql/ha_autoi.cpp index ddfbcdc22..4b9a2ae3b 100644 --- a/dbcon/mysql/ha_autoi.cpp +++ b/dbcon/mysql/ha_autoi.cpp @@ -176,4 +176,3 @@ bool parseAutoincrementColumnComment(std::string comment, uint64_t& startValue) return autoincrement; } -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h index c0390877e..17aa7d17e 100644 --- a/dbcon/mysql/ha_mcs_datatype.h +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -841,4 +841,3 @@ class WriteBatchFieldMariaDB : public WriteBatchField } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 54ee97a17..2efba1efd 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -2807,4 +2807,3 @@ extern "C" } } -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_mcs_dml.cpp b/dbcon/mysql/ha_mcs_dml.cpp index 554e03bba..48ce3b33d 100644 --- a/dbcon/mysql/ha_mcs_dml.cpp +++ b/dbcon/mysql/ha_mcs_dml.cpp @@ -1001,4 +1001,3 @@ int ha_mcs_impl_close_connection_(handlerton* hton, THD* thd, cal_connection_inf return rc; } -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 7b396360e..9651413f8 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -10459,4 +10459,3 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro } } // namespace cal_impl_if -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_pseudocolumn.cpp b/dbcon/mysql/ha_pseudocolumn.cpp index 26fb82264..a9341049a 100644 --- a/dbcon/mysql/ha_pseudocolumn.cpp +++ b/dbcon/mysql/ha_pseudocolumn.cpp @@ -579,4 +579,3 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& } } // namespace cal_impl_if -// vim:ts=4 sw=4: diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index 47f74f89a..6798e6fca 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -118,4 +118,3 @@ inline char* idb_mysql_query_str(THD* thd) } } // namespace -// vim:ts=4 sw=4: diff --git a/ddlproc/ddlproc.cpp b/ddlproc/ddlproc.cpp index e479fbf49..f203f85d3 100644 --- a/ddlproc/ddlproc.cpp +++ b/ddlproc/ddlproc.cpp @@ -248,4 +248,3 @@ int main(int argc, char** argv) return ServiceDDLProc(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/ddlproc/ddlprocessor.cpp b/ddlproc/ddlprocessor.cpp index eb6f3d09b..cca42af18 100644 --- a/ddlproc/ddlprocessor.cpp +++ b/ddlproc/ddlprocessor.cpp @@ -876,4 +876,3 @@ int DDLProcessor::commitTransaction(uint32_t txnID, std::string& errorMsg) return rc; } } // namespace ddlprocessor -// vim:ts=4 sw=4: diff --git a/dmlproc/batchinsertprocessor.cpp b/dmlproc/batchinsertprocessor.cpp index 1a61e1eb6..483074179 100644 --- a/dmlproc/batchinsertprocessor.cpp +++ b/dmlproc/batchinsertprocessor.cpp @@ -547,4 +547,3 @@ void BatchInsertProc::getError(int& errorCode, std::string& errMsg) errMsg = fErrMsg; } } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/dmlproc/batchinsertprocessor.h b/dmlproc/batchinsertprocessor.h index 270bd592a..725581d20 100644 --- a/dmlproc/batchinsertprocessor.h +++ b/dmlproc/batchinsertprocessor.h @@ -93,4 +93,3 @@ class BatchInsertProc }; } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index f1fa9cc4c..fb79ae59f 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -639,14 +639,7 @@ int ServiceDMLProc::Child() // Couldn't check the return code b/c // fuser returns 1 for unused port. -#if defined(__GNUC__) && __GNUC__ >= 5 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" - (void)::system(cmd.c_str()); -#pragma GCC diagnostic pop -#else - (void)::system(cmd.c_str()); -#endif + std::ignore = ::system(cmd.c_str()); } catch (...) { @@ -698,4 +691,3 @@ int main(int argc, char** argv) return ServiceDMLProc(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/dmlproc/dmlprocessor.cpp b/dmlproc/dmlprocessor.cpp index 33e9b3d14..0cc5337a5 100644 --- a/dmlproc/dmlprocessor.cpp +++ b/dmlproc/dmlprocessor.cpp @@ -2001,4 +2001,3 @@ void DMLProcessor::log(const std::string& msg, logging::LOG_TYPE level) } } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/dmlproc/dmlprocessor.h b/dmlproc/dmlprocessor.h index 43b1850ba..fbc7c32d6 100644 --- a/dmlproc/dmlprocessor.h +++ b/dmlproc/dmlprocessor.h @@ -330,4 +330,3 @@ class RollbackTransactionProcessor : public dmlpackageprocessor::DMLPackageProce } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/exemgr/activestatementcounter.cpp b/exemgr/activestatementcounter.cpp index 27a3df1de..b25e61549 100644 --- a/exemgr/activestatementcounter.cpp +++ b/exemgr/activestatementcounter.cpp @@ -57,4 +57,3 @@ void ActiveStatementCounter::decr(bool& counted) --fStatementCount; condvar.notify_one(); } -// vim:ts=4 sw=4: diff --git a/exemgr/activestatementcounter.h b/exemgr/activestatementcounter.h index 149ad23e9..87e7163aa 100644 --- a/exemgr/activestatementcounter.h +++ b/exemgr/activestatementcounter.h @@ -62,4 +62,3 @@ class ActiveStatementCounter BRM::VSS fVss; }; -// vim:ts=4 sw=4: diff --git a/exemgr/main.cpp b/exemgr/main.cpp index b1b31b9c6..8b84443f1 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1825,4 +1825,3 @@ int main(int argc, char* argv[]) return ServiceExeMgr(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index cfbd00886..015f09d83 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -1066,4 +1066,3 @@ std::string Oam::itoa(const int i) return ss.str(); } } // namespace oam -// vim:ts=4 sw=4: diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index cf5e03248..c2df5bf24 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -433,4 +433,3 @@ class Oam #undef EXPORT -// vim:ts=4 sw=4: diff --git a/oamapps/columnstoreDB/columnstoreDB.cpp b/oamapps/columnstoreDB/columnstoreDB.cpp index f716f9bd8..3bdca8b64 100644 --- a/oamapps/columnstoreDB/columnstoreDB.cpp +++ b/oamapps/columnstoreDB/columnstoreDB.cpp @@ -271,7 +271,7 @@ int main(int argc, char** argv) Oam oam; BRM::DBRM dbrm; - char c; + signed char c; // Invokes member function `int operator ()(void);' while ((c = getopt(argc, argv, "c:h")) != -1) diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index d46bd773a..a5b51dec0 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -284,4 +284,3 @@ int main(int argc, char* argv[]) exit(0); } -// vim:ts=4 sw=4: diff --git a/oamapps/sessionWalker/sessionwalker.cpp b/oamapps/sessionWalker/sessionwalker.cpp index 5f0960128..06cf08bd0 100644 --- a/oamapps/sessionWalker/sessionwalker.cpp +++ b/oamapps/sessionWalker/sessionwalker.cpp @@ -133,4 +133,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/blockcacheclient.h b/primitives/blockcache/blockcacheclient.h index 307814898..e83c04c19 100644 --- a/primitives/blockcache/blockcacheclient.h +++ b/primitives/blockcache/blockcacheclient.h @@ -176,4 +176,3 @@ class blockCacheClient } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/blockrequestprocessor.h b/primitives/blockcache/blockrequestprocessor.h index c0257ef41..0970c40d5 100644 --- a/primitives/blockcache/blockrequestprocessor.h +++ b/primitives/blockcache/blockrequestprocessor.h @@ -190,4 +190,3 @@ class BlockRequestProcessor }; } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/filerequest.h b/primitives/blockcache/filerequest.h index cdca79f48..8cee46733 100644 --- a/primitives/blockcache/filerequest.h +++ b/primitives/blockcache/filerequest.h @@ -320,4 +320,3 @@ class fileRequest } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/iomanager.cpp b/primitives/blockcache/iomanager.cpp index dbfa22a84..5473cf3d5 100644 --- a/primitives/blockcache/iomanager.cpp +++ b/primitives/blockcache/iomanager.cpp @@ -1440,4 +1440,3 @@ void ioManager::handleBlockReadError(fileRequest* fr, const string& errMsg, bool } } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/blockcache/iomanager.h b/primitives/blockcache/iomanager.h index 70b4a09b5..cc11b61e7 100644 --- a/primitives/blockcache/iomanager.h +++ b/primitives/blockcache/iomanager.h @@ -147,4 +147,3 @@ void dropFDCache(); void purgeFDCache(std::vector& files); } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 2432cc690..5ca4c3f6f 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1764,4 +1764,3 @@ template void primitives::PrimitiveProcessor::columnScanAndFilter(NewC ColResultHeader*); } // namespace primitives -// vim:ts=2 sw=2: diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index 2942d4fda..360d8b607 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -620,4 +620,3 @@ void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector* out, } } // namespace primitives -// vim:ts=4 sw=4: diff --git a/primitives/linux-port/index.cpp b/primitives/linux-port/index.cpp index bbfc949ef..2d86d35e2 100644 --- a/primitives/linux-port/index.cpp +++ b/primitives/linux-port/index.cpp @@ -1130,4 +1130,3 @@ void PrimitiveProcessor::p_IdxList(const IndexListHeader* rqst, IndexListHeader* } } // namespace primitives -// vim:ts=4 sw=4: diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index bbd555b48..73e6e90fb 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -562,4 +562,3 @@ boost::shared_ptr _parseColumnFilter( } // namespace primitives -// vim:ts=2 sw=2: diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index ca52daf22..dab8c6693 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -2746,4 +2746,3 @@ void BatchPrimitiveProcessor::buildVSSCache(uint32_t loopCount) } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 16426f49d..a7069359b 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -1226,4 +1226,3 @@ void ColumnCommandInt128::issuePrimitive() } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index 57dda7142..518374ff0 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -306,4 +306,3 @@ inline void ColumnCommand::fillEmptyBlock(uint } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index 5581ceee9..ad073ddc7 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -2575,4 +2575,3 @@ bool BPPV::aborted() // end workaround } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 277065e91..943e65595 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -784,4 +784,3 @@ int main(int argc, char** argv) return ServicePrimProc(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/primitives/primproc/udf.cpp b/primitives/primproc/udf.cpp index 7a836590d..188e5128a 100644 --- a/primitives/primproc/udf.cpp +++ b/primitives/primproc/udf.cpp @@ -69,4 +69,3 @@ void loadUDFs() } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/storage-manager/src/smcat.cpp b/storage-manager/src/smcat.cpp index bc1b1e1a9..101ab41a7 100644 --- a/storage-manager/src/smcat.cpp +++ b/storage-manager/src/smcat.cpp @@ -93,7 +93,7 @@ void catFileOffline(const char* filename, int prefixlen) void catFileOnline(const char* filename, int prefixlen) { uint8_t data[8192]; - off_t offset = 0; + [[maybe_unused]] off_t offset = 0; int read_err, write_err, count; idbdatafile::SMDataFile df(filename, O_RDONLY, 0); diff --git a/storage-manager/src/unit_tests.cpp b/storage-manager/src/unit_tests.cpp index 1864939f4..0a9dd8d2a 100644 --- a/storage-manager/src/unit_tests.cpp +++ b/storage-manager/src/unit_tests.cpp @@ -1056,10 +1056,10 @@ bool copytask(bool connectionTest = false) copy_cmd* cmd = (copy_cmd*)buf; cmd->opcode = COPY; cmd->file1.flen = strlen(source); - strncpy(cmd->file1.filename, source, cmd->file1.flen); + memcpy(cmd->file1.filename, source, cmd->file1.flen); f_name* file2 = (f_name*)&cmd->file1.filename[cmd->file1.flen]; file2->flen = strlen(dest); - strncpy(file2->filename, dest, file2->flen); + memcpy(file2->filename, dest, file2->flen); uint len = (uint64_t)&file2->filename[file2->flen] - (uint64_t)buf; diff --git a/tests/primitives_column_scan_and_filter.cpp b/tests/primitives_column_scan_and_filter.cpp index 70f921b38..eeed651ee 100644 --- a/tests/primitives_column_scan_and_filter.cpp +++ b/tests/primitives_column_scan_and_filter.cpp @@ -965,4 +965,3 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes2CompFilters) EXPECT_EQ(out->Max, __col16block_cdf_umax); EXPECT_EQ(out->Min, __col16block_cdf_umin); } -// vim:ts=2 sw=2: diff --git a/tests/primitives_scan_bench.cpp b/tests/primitives_scan_bench.cpp index 2d6b77c9a..c2431b8d9 100644 --- a/tests/primitives_scan_bench.cpp +++ b/tests/primitives_scan_bench.cpp @@ -427,4 +427,3 @@ BENCHMARK_DEFINE_F(FilterBenchFixture, BM_ColumnScan8Byte1FilterVectorizedCode)( BENCHMARK_REGISTER_F(FilterBenchFixture, BM_ColumnScan8Byte1FilterVectorizedCode); BENCHMARK_MAIN(); -// vim:ts=2 sw=2: diff --git a/tools/bincvt/li2bin.cpp b/tools/bincvt/li2bin.cpp index 51b1d8b64..91bf9c4c0 100644 --- a/tools/bincvt/li2bin.cpp +++ b/tools/bincvt/li2bin.cpp @@ -278,4 +278,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/clearShm/main.cpp b/tools/clearShm/main.cpp index 31e07b9a8..79e38dea5 100644 --- a/tools/clearShm/main.cpp +++ b/tools/clearShm/main.cpp @@ -239,4 +239,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index 38b9ffd54..79d2a4beb 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -324,4 +324,3 @@ int main(int argc, char* argv[]) return 1; } -// vim:ts=4 sw=4: diff --git a/tools/ddlcleanup/ddlcleanup.cpp b/tools/ddlcleanup/ddlcleanup.cpp index bbbbac437..6d1ca008a 100644 --- a/tools/ddlcleanup/ddlcleanup.cpp +++ b/tools/ddlcleanup/ddlcleanup.cpp @@ -80,4 +80,3 @@ int main(int argc, char** argv) return ddlcleanuputil::ddl_cleanup(); } -// vim:ts=4 sw=4: diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index 2dcb6ce81..bce63ed0e 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -1040,4 +1040,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/qfe/server.cpp b/tools/qfe/server.cpp index 018cf470a..7b3276ded 100644 --- a/tools/qfe/server.cpp +++ b/tools/qfe/server.cpp @@ -513,4 +513,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/sendPlan/sendplan.cpp b/tools/sendPlan/sendplan.cpp index 5374d8676..bd8f4bb4e 100644 --- a/tools/sendPlan/sendplan.cpp +++ b/tools/sendPlan/sendplan.cpp @@ -390,4 +390,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/tools/setConfig/main.cpp b/tools/setConfig/main.cpp index 1c536167f..7d7fe396e 100644 --- a/tools/setConfig/main.cpp +++ b/tools/setConfig/main.cpp @@ -148,4 +148,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/utils/cacheutils/cacheutils.cpp b/utils/cacheutils/cacheutils.cpp index ead2483ad..a0042e155 100644 --- a/utils/cacheutils/cacheutils.cpp +++ b/utils/cacheutils/cacheutils.cpp @@ -354,4 +354,3 @@ int purgePrimProcFdCache(const std::vector files, const int pmId) } } // namespace cacheutils -// vim:ts=4 sw=4: diff --git a/utils/cacheutils/cacheutils.h b/utils/cacheutils/cacheutils.h index 013ef93c3..23624e521 100644 --- a/utils/cacheutils/cacheutils.h +++ b/utils/cacheutils/cacheutils.h @@ -72,4 +72,3 @@ int dropPrimProcFdCache(); int purgePrimProcFdCache(const std::vector files, const int pmId); } // namespace cacheutils -// vim:ts=4 sw=4: diff --git a/utils/common/hashfamily.h b/utils/common/hashfamily.h index fa86b7a15..4b09e01a8 100644 --- a/utils/common/hashfamily.h +++ b/utils/common/hashfamily.h @@ -50,4 +50,3 @@ class HashFamily }; } // namespace utils -// vim:ts=2 sw=2: diff --git a/utils/common/mcs_basic_types.h b/utils/common/mcs_basic_types.h index ff64c6a00..3e877ed24 100644 --- a/utils/common/mcs_basic_types.h +++ b/utils/common/mcs_basic_types.h @@ -21,4 +21,3 @@ using int128_t = __int128; using uint128_t = unsigned __int128; -// vim:ts=2 sw=2: diff --git a/utils/common/simd_sse.h b/utils/common/simd_sse.h index 9f7e8ffb9..ccf2ff2b7 100644 --- a/utils/common/simd_sse.h +++ b/utils/common/simd_sse.h @@ -896,4 +896,3 @@ class SimdFilterProcessor< } // namespace simd #endif // if defined(__x86_64__ ) -// vim:ts=2 sw=2: diff --git a/utils/compress/idbcompress.cpp b/utils/compress/idbcompress.cpp index cffe8d268..2c64c3387 100644 --- a/utils/compress/idbcompress.cpp +++ b/utils/compress/idbcompress.cpp @@ -672,4 +672,3 @@ std::shared_ptr getCompressorByType( #endif } // namespace compress -// vim:ts=4 sw=4: diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index 6e2101f0c..57c995964 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -684,4 +684,3 @@ std::string Config::getTempFileDir(Config::TempDirPurpose what) } } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/configcpp/configcpp.h b/utils/configcpp/configcpp.h index 41bb1ad7e..a413c3f52 100644 --- a/utils/configcpp/configcpp.h +++ b/utils/configcpp/configcpp.h @@ -254,4 +254,3 @@ class Config #undef EXPORT -// vim:ts=4 sw=4: diff --git a/utils/configcpp/configstream.cpp b/utils/configcpp/configstream.cpp index 059f690ed..cfb1a4acf 100644 --- a/utils/configcpp/configstream.cpp +++ b/utils/configcpp/configstream.cpp @@ -64,4 +64,3 @@ void ConfigStream::init(const xmlChar* xp) } } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/configcpp/configstream.h b/utils/configcpp/configstream.h index 25ed12f67..deefb02f7 100644 --- a/utils/configcpp/configstream.h +++ b/utils/configcpp/configstream.h @@ -61,4 +61,3 @@ class ConfigStream } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/configcpp/xmlparser.cpp b/utils/configcpp/xmlparser.cpp index 03da1f959..37b6bddea 100644 --- a/utils/configcpp/xmlparser.cpp +++ b/utils/configcpp/xmlparser.cpp @@ -285,4 +285,3 @@ const vector XMLParser::enumSection(const xmlDocPtr doc, const string& s } } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/configcpp/xmlparser.h b/utils/configcpp/xmlparser.h index c7576890b..e31659fea 100644 --- a/utils/configcpp/xmlparser.h +++ b/utils/configcpp/xmlparser.h @@ -63,4 +63,3 @@ class XMLParser } // namespace config -// vim:ts=4 sw=4: diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 141954333..37ff38c4e 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1418,7 +1418,7 @@ boost::any DataConvert::StringToUDecimal(const datatypes::SystemCatalog::TypeAtt { int64_t val64; number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - char ival = (char)val64; + signed char ival = (signed char)val64; if (ival < 0 && ival != static_cast(joblist::TINYINTEMPTYROW) && ival != static_cast(joblist::TINYINTNULL)) @@ -3388,4 +3388,3 @@ void DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd& u } } // namespace dataconvert -// vim:ts=4 sw=4: diff --git a/utils/ddlcleanup/ddlcleanuputil.cpp b/utils/ddlcleanup/ddlcleanuputil.cpp index a9defdf45..ca196c295 100644 --- a/utils/ddlcleanup/ddlcleanuputil.cpp +++ b/utils/ddlcleanup/ddlcleanuputil.cpp @@ -270,4 +270,3 @@ int ddl_cleanup() } } // namespace ddlcleanuputil -// vim:ts=4 sw=4: diff --git a/utils/ddlcleanup/ddlcleanuputil.h b/utils/ddlcleanup/ddlcleanuputil.h index f7523d43a..bfd8e87cd 100644 --- a/utils/ddlcleanup/ddlcleanuputil.h +++ b/utils/ddlcleanup/ddlcleanuputil.h @@ -28,4 +28,3 @@ namespace ddlcleanuputil { int ddl_cleanup(); } -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_abs.cpp b/utils/funcexp/func_abs.cpp index 64ac5d014..cd05ade7a 100644 --- a/utils/funcexp/func_abs.cpp +++ b/utils/funcexp/func_abs.cpp @@ -79,4 +79,3 @@ long double Func_abs::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index 5a2506b3f..82c173edc 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -303,4 +303,3 @@ int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, boo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ascii.cpp b/utils/funcexp/func_ascii.cpp index eb93f6011..5cf8653d2 100644 --- a/utils/funcexp/func_ascii.cpp +++ b/utils/funcexp/func_ascii.cpp @@ -56,4 +56,3 @@ int64_t Func_ascii::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index 3ce66eb60..657123635 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -376,4 +376,3 @@ bool Func_notbetween::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index c10030b98..998696ae0 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -494,4 +494,3 @@ bool Func_bit_count::fix(execplan::FunctionColumn& col) const } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index e8bfbd0f4..fba7691e0 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -723,4 +723,3 @@ int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 4b224c6d9..ef0e2f515 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -1549,4 +1549,3 @@ double Func_cast_double::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index dc559a5db..e5c1f8728 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -598,4 +598,3 @@ IDB_Decimal Func_ceil::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index 7ead2b947..0c4adeaf9 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -178,4 +178,3 @@ string Func_char::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontS } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index 553d54399..5270d1d8a 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -121,4 +121,3 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index ebabde050..bbb5a279e 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -244,4 +244,3 @@ execplan::IDB_Decimal Func_coalesce::getDecimalVal(rowgroup::Row& row, FunctionP } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat.cpp b/utils/funcexp/func_concat.cpp index 73b2a2b2d..05835d23a 100644 --- a/utils/funcexp/func_concat.cpp +++ b/utils/funcexp/func_concat.cpp @@ -65,4 +65,3 @@ string Func_concat::getStrVal(Row& row, FunctionParm& parm, bool& isNull, Calpon } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat_oracle.cpp b/utils/funcexp/func_concat_oracle.cpp index 28713a107..46631e7a4 100644 --- a/utils/funcexp/func_concat_oracle.cpp +++ b/utils/funcexp/func_concat_oracle.cpp @@ -71,4 +71,3 @@ string Func_concat_oracle::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_concat_ws.cpp b/utils/funcexp/func_concat_ws.cpp index 2c07c5b0f..ea0cf22d4 100644 --- a/utils/funcexp/func_concat_ws.cpp +++ b/utils/funcexp/func_concat_ws.cpp @@ -121,4 +121,3 @@ string Func_concat_ws::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_conv.cpp b/utils/funcexp/func_conv.cpp index 5f73f5ea4..037ce83e8 100644 --- a/utils/funcexp/func_conv.cpp +++ b/utils/funcexp/func_conv.cpp @@ -304,4 +304,3 @@ string Func_conv::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_convert_tz.cpp b/utils/funcexp/func_convert_tz.cpp index 03efdad88..363392796 100644 --- a/utils/funcexp/func_convert_tz.cpp +++ b/utils/funcexp/func_convert_tz.cpp @@ -220,4 +220,3 @@ string Func_convert_tz::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_crc32.cpp b/utils/funcexp/func_crc32.cpp index a40717378..786c3f33e 100644 --- a/utils/funcexp/func_crc32.cpp +++ b/utils/funcexp/func_crc32.cpp @@ -64,4 +64,3 @@ int64_t Func_crc32::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date.cpp b/utils/funcexp/func_date.cpp index 5544e8b31..b6e554e5f 100644 --- a/utils/funcexp/func_date.cpp +++ b/utils/funcexp/func_date.cpp @@ -149,4 +149,3 @@ string Func_date::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index 2e715acc7..6cc47fdae 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -816,4 +816,3 @@ string Func_date_add::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index 962fc811d..8a3ba9749 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -416,4 +416,3 @@ int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, FunctionParm& p } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index bd4a484d8..e7954e3d0 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -143,4 +143,3 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index 07078a25f..e8e069db5 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -179,4 +179,3 @@ string Func_dayname::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index 441bfeebc..c8fb40ab2 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -166,4 +166,3 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index fb9d97428..215ec2150 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -160,4 +160,3 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_div.cpp b/utils/funcexp/func_div.cpp index 7169992bb..d36e75c9e 100644 --- a/utils/funcexp/func_div.cpp +++ b/utils/funcexp/func_div.cpp @@ -108,4 +108,3 @@ string Func_div::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_elt.cpp b/utils/funcexp/func_elt.cpp index 22446095f..860dd1ae3 100644 --- a/utils/funcexp/func_elt.cpp +++ b/utils/funcexp/func_elt.cpp @@ -96,4 +96,3 @@ string Func_elt::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_exp.cpp b/utils/funcexp/func_exp.cpp index bab2fc926..d57b28556 100644 --- a/utils/funcexp/func_exp.cpp +++ b/utils/funcexp/func_exp.cpp @@ -110,4 +110,3 @@ long double Func_exp::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index a6f8d6353..b051f2676 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -254,4 +254,3 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_find_in_set.cpp b/utils/funcexp/func_find_in_set.cpp index 94c1243dc..85b0c7e25 100644 --- a/utils/funcexp/func_find_in_set.cpp +++ b/utils/funcexp/func_find_in_set.cpp @@ -127,4 +127,3 @@ execplan::IDB_Decimal Func_find_in_set::getDecimalVal(rowgroup::Row& row, Functi } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index 7c0dacc82..f497400b5 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -555,4 +555,3 @@ IDB_Decimal Func_floor::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_from_days.cpp b/utils/funcexp/func_from_days.cpp index e9f45b421..a3e964ee8 100644 --- a/utils/funcexp/func_from_days.cpp +++ b/utils/funcexp/func_from_days.cpp @@ -80,4 +80,3 @@ int64_t Func_from_days::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index cca4083be..b01bfab34 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -212,4 +212,3 @@ long double Func_from_unixtime::getLongDoubleVal(rowgroup::Row& row, FunctionPar } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_get_format.cpp b/utils/funcexp/func_get_format.cpp index d902c0873..00f513643 100644 --- a/utils/funcexp/func_get_format.cpp +++ b/utils/funcexp/func_get_format.cpp @@ -107,4 +107,3 @@ string Func_get_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index e8562be20..b3dea5beb 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -240,4 +240,3 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index 133eca923..0e550d0e7 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -140,4 +140,3 @@ string Func_hex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 7d2da7411..0550ef3ef 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -156,4 +156,3 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_idbpartition.cpp b/utils/funcexp/func_idbpartition.cpp index 3b71cdd97..b7b5e6971 100644 --- a/utils/funcexp/func_idbpartition.cpp +++ b/utils/funcexp/func_idbpartition.cpp @@ -63,4 +63,3 @@ string Func_idbpartition::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 98fe8a334..d5941df8d 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -234,4 +234,3 @@ int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, Calpo } } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 52dae2556..a2450a5a9 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -205,4 +205,3 @@ bool Func_ifnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, Calpont } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_in.cpp b/utils/funcexp/func_in.cpp index 89ed89593..d452fc242 100644 --- a/utils/funcexp/func_in.cpp +++ b/utils/funcexp/func_in.cpp @@ -386,4 +386,3 @@ bool Func_notin::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_insert.cpp b/utils/funcexp/func_insert.cpp index 236752e96..72fd06e46 100644 --- a/utils/funcexp/func_insert.cpp +++ b/utils/funcexp/func_insert.cpp @@ -102,4 +102,3 @@ std::string Func_insert::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_instr.cpp b/utils/funcexp/func_instr.cpp index ab0379cce..71eb81d0a 100644 --- a/utils/funcexp/func_instr.cpp +++ b/utils/funcexp/func_instr.cpp @@ -85,4 +85,3 @@ int64_t Func_instr::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_isnull.cpp b/utils/funcexp/func_isnull.cpp index 48337b3ec..4ff7e643a 100644 --- a/utils/funcexp/func_isnull.cpp +++ b/utils/funcexp/func_isnull.cpp @@ -79,4 +79,3 @@ bool Func_isnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, Calpont } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index 89f173a77..e74a2d514 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -195,4 +195,3 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_lcase.cpp b/utils/funcexp/func_lcase.cpp index 7de396e47..ccc2dccb5 100644 --- a/utils/funcexp/func_lcase.cpp +++ b/utils/funcexp/func_lcase.cpp @@ -64,4 +64,3 @@ std::string Func_lcase::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index add78cfa7..d488b7516 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -218,4 +218,3 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_left.cpp b/utils/funcexp/func_left.cpp index a90d5f46c..de31d1770 100644 --- a/utils/funcexp/func_left.cpp +++ b/utils/funcexp/func_left.cpp @@ -73,4 +73,3 @@ std::string Func_left::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_length.cpp b/utils/funcexp/func_length.cpp index f63e2f1bb..a6f7236f5 100644 --- a/utils/funcexp/func_length.cpp +++ b/utils/funcexp/func_length.cpp @@ -55,4 +55,3 @@ int64_t Func_length::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_lpad.cpp b/utils/funcexp/func_lpad.cpp index 0ad0bdc25..8a5544a6f 100644 --- a/utils/funcexp/func_lpad.cpp +++ b/utils/funcexp/func_lpad.cpp @@ -127,4 +127,3 @@ std::string Func_lpad::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ltrim.cpp b/utils/funcexp/func_ltrim.cpp index 39fd7165f..3d7e96918 100644 --- a/utils/funcexp/func_ltrim.cpp +++ b/utils/funcexp/func_ltrim.cpp @@ -93,4 +93,3 @@ std::string Func_ltrim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ltrim_oracle.cpp b/utils/funcexp/func_ltrim_oracle.cpp index 234808e74..70fc1dc8b 100644 --- a/utils/funcexp/func_ltrim_oracle.cpp +++ b/utils/funcexp/func_ltrim_oracle.cpp @@ -97,4 +97,3 @@ std::string Func_ltrim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, b } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_makedate.cpp b/utils/funcexp/func_makedate.cpp index 0418508d3..fb49f54cc 100644 --- a/utils/funcexp/func_makedate.cpp +++ b/utils/funcexp/func_makedate.cpp @@ -190,4 +190,3 @@ string Func_makedate::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_maketime.cpp b/utils/funcexp/func_maketime.cpp index 6d0a9d903..ad918ccce 100644 --- a/utils/funcexp/func_maketime.cpp +++ b/utils/funcexp/func_maketime.cpp @@ -171,4 +171,3 @@ string Func_maketime::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 7d60ae507..74a12aa7d 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -2276,4 +2276,3 @@ double Func_degrees::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, Ca } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_md5.cpp b/utils/funcexp/func_md5.cpp index 6ef058e84..0ce6b0a05 100644 --- a/utils/funcexp/func_md5.cpp +++ b/utils/funcexp/func_md5.cpp @@ -512,4 +512,3 @@ string Func_md5::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index aca1e3aea..8c94647bf 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -137,4 +137,3 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index af310c876..8e5724547 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -137,4 +137,3 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_mod.cpp b/utils/funcexp/func_mod.cpp index bf33a6491..dff1a4e07 100644 --- a/utils/funcexp/func_mod.cpp +++ b/utils/funcexp/func_mod.cpp @@ -531,4 +531,3 @@ std::string Func_mod::getStrVal(Row& row, FunctionParm& fp, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index 50ee3600e..ea66abf32 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -142,4 +142,3 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 8ea7c3e37..c674f454e 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -191,4 +191,3 @@ execplan::IDB_Decimal Func_monthname::getDecimalVal(rowgroup::Row& row, Function } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index ba4ab740f..3821151e4 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -964,7 +964,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionPar { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -1022,4 +1022,3 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionPar } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_period_add.cpp b/utils/funcexp/func_period_add.cpp index 23b964151..4038651ed 100644 --- a/utils/funcexp/func_period_add.cpp +++ b/utils/funcexp/func_period_add.cpp @@ -94,4 +94,3 @@ int64_t Func_period_add::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_period_diff.cpp b/utils/funcexp/func_period_diff.cpp index 470ab60c9..080d15a42 100644 --- a/utils/funcexp/func_period_diff.cpp +++ b/utils/funcexp/func_period_diff.cpp @@ -113,4 +113,3 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_pow.cpp b/utils/funcexp/func_pow.cpp index 734b50a22..236e7e2c5 100644 --- a/utils/funcexp/func_pow.cpp +++ b/utils/funcexp/func_pow.cpp @@ -121,4 +121,3 @@ long double Func_pow::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index 04c4babb5..5f1fd0033 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -139,4 +139,3 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_quote.cpp b/utils/funcexp/func_quote.cpp index 05ce4cecf..b82cf7baa 100644 --- a/utils/funcexp/func_quote.cpp +++ b/utils/funcexp/func_quote.cpp @@ -80,4 +80,3 @@ std::string Func_quote::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rand.cpp b/utils/funcexp/func_rand.cpp index 223949bbf..b7da263d9 100644 --- a/utils/funcexp/func_rand.cpp +++ b/utils/funcexp/func_rand.cpp @@ -98,4 +98,3 @@ double Func_rand::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index cb495c8c1..60e1459de 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -248,4 +248,3 @@ bool Func_regexp::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_repeat.cpp b/utils/funcexp/func_repeat.cpp index 6b0e6ace0..ebbfc355a 100644 --- a/utils/funcexp/func_repeat.cpp +++ b/utils/funcexp/func_repeat.cpp @@ -90,4 +90,3 @@ std::string Func_repeat::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_replace.cpp b/utils/funcexp/func_replace.cpp index 05e0fe359..da18fb3f1 100644 --- a/utils/funcexp/func_replace.cpp +++ b/utils/funcexp/func_replace.cpp @@ -173,4 +173,3 @@ std::string Func_replace::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_replace_oracle.cpp b/utils/funcexp/func_replace_oracle.cpp index b8eb7f6ef..113b49d96 100644 --- a/utils/funcexp/func_replace_oracle.cpp +++ b/utils/funcexp/func_replace_oracle.cpp @@ -167,4 +167,3 @@ std::string Func_replace_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_reverse.cpp b/utils/funcexp/func_reverse.cpp index 2b383ee7f..db57d0953 100644 --- a/utils/funcexp/func_reverse.cpp +++ b/utils/funcexp/func_reverse.cpp @@ -93,4 +93,3 @@ std::string Func_reverse::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_right.cpp b/utils/funcexp/func_right.cpp index 4615256e5..2c1ec612a 100644 --- a/utils/funcexp/func_right.cpp +++ b/utils/funcexp/func_right.cpp @@ -73,4 +73,3 @@ std::string Func_right::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 16855324e..c6e673361 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -645,7 +645,7 @@ string Func_round::getStrVal(Row& row, FunctionParm& parm, bool& isNull, Calpont { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); int64_t e = (x.scale < 0) ? (-x.scale) : x.scale; - int64_t p = 1; + [[maybe_unused]] int64_t p = 1; while (e-- > 0) p *= 10; @@ -716,4 +716,3 @@ int64_t Func_round::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, b } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rpad.cpp b/utils/funcexp/func_rpad.cpp index fba47bfae..a2a538ff3 100644 --- a/utils/funcexp/func_rpad.cpp +++ b/utils/funcexp/func_rpad.cpp @@ -127,4 +127,3 @@ std::string Func_rpad::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rtrim.cpp b/utils/funcexp/func_rtrim.cpp index 426ec8712..ee61906b4 100644 --- a/utils/funcexp/func_rtrim.cpp +++ b/utils/funcexp/func_rtrim.cpp @@ -157,4 +157,3 @@ std::string Func_rtrim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_rtrim_oracle.cpp b/utils/funcexp/func_rtrim_oracle.cpp index 1b47d7ddf..93713e113 100644 --- a/utils/funcexp/func_rtrim_oracle.cpp +++ b/utils/funcexp/func_rtrim_oracle.cpp @@ -161,4 +161,3 @@ std::string Func_rtrim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, b } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sec_to_time.cpp b/utils/funcexp/func_sec_to_time.cpp index d3b4af675..9a2fef4c4 100644 --- a/utils/funcexp/func_sec_to_time.cpp +++ b/utils/funcexp/func_sec_to_time.cpp @@ -231,4 +231,3 @@ execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, Functi } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 7731ea66c..6136c3ede 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -144,4 +144,3 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sha.cpp b/utils/funcexp/func_sha.cpp index bc74a0a2a..ec3fac1f4 100644 --- a/utils/funcexp/func_sha.cpp +++ b/utils/funcexp/func_sha.cpp @@ -655,4 +655,3 @@ string Func_sha::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sign.cpp b/utils/funcexp/func_sign.cpp index a0cece67a..426fbb0c3 100644 --- a/utils/funcexp/func_sign.cpp +++ b/utils/funcexp/func_sign.cpp @@ -71,4 +71,3 @@ string Func_sign::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_space.cpp b/utils/funcexp/func_space.cpp index 93e765a56..9a9a85789 100644 --- a/utils/funcexp/func_space.cpp +++ b/utils/funcexp/func_space.cpp @@ -57,4 +57,3 @@ std::string Func_space::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index 0580021ff..ab0fb4f61 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -264,4 +264,3 @@ int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_strcmp.cpp b/utils/funcexp/func_strcmp.cpp index a16c68cc4..63b3b0b22 100644 --- a/utils/funcexp/func_strcmp.cpp +++ b/utils/funcexp/func_strcmp.cpp @@ -74,4 +74,3 @@ std::string Func_strcmp::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_substr.cpp b/utils/funcexp/func_substr.cpp index cb45acc0b..9a39a4f10 100644 --- a/utils/funcexp/func_substr.cpp +++ b/utils/funcexp/func_substr.cpp @@ -98,4 +98,3 @@ std::string Func_substr::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_substring_index.cpp b/utils/funcexp/func_substring_index.cpp index ffae25e5e..82f94a7ee 100644 --- a/utils/funcexp/func_substring_index.cpp +++ b/utils/funcexp/func_substring_index.cpp @@ -205,4 +205,3 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row, FunctionParm& fp } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index b30e66c53..780492f37 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -105,4 +105,3 @@ int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 14d2246ae..e898f7820 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -169,4 +169,3 @@ double Func_time::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index d075658b0..1205e21ba 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -229,4 +229,3 @@ string Func_time_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index 388ef2ca6..3cf718aa2 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -195,4 +195,3 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index d3f891397..030004761 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -349,4 +349,3 @@ double Func_timediff::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index 07e7eda73..6ad2f3952 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -196,4 +196,3 @@ int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_to_days.cpp b/utils/funcexp/func_to_days.cpp index f597c5fd7..d6f831be1 100644 --- a/utils/funcexp/func_to_days.cpp +++ b/utils/funcexp/func_to_days.cpp @@ -151,4 +151,3 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_trim.cpp b/utils/funcexp/func_trim.cpp index 636afe44e..d47823563 100644 --- a/utils/funcexp/func_trim.cpp +++ b/utils/funcexp/func_trim.cpp @@ -168,4 +168,3 @@ std::string Func_trim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_trim_oracle.cpp b/utils/funcexp/func_trim_oracle.cpp index 5e1226c58..4daec213f 100644 --- a/utils/funcexp/func_trim_oracle.cpp +++ b/utils/funcexp/func_trim_oracle.cpp @@ -166,4 +166,3 @@ std::string Func_trim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, bo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index 87f045af0..7a529c00f 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -522,7 +522,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -613,7 +613,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -721,4 +721,3 @@ int64_t Func_truncate::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ucase.cpp b/utils/funcexp/func_ucase.cpp index 4d78fb8b2..578ad5e07 100644 --- a/utils/funcexp/func_ucase.cpp +++ b/utils/funcexp/func_ucase.cpp @@ -73,4 +73,3 @@ std::string Func_ucase::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_unhex.cpp b/utils/funcexp/func_unhex.cpp index 38da2cb2d..92ef5fa3e 100644 --- a/utils/funcexp/func_unhex.cpp +++ b/utils/funcexp/func_unhex.cpp @@ -112,4 +112,3 @@ string Func_unhex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_unix_timestamp.cpp b/utils/funcexp/func_unix_timestamp.cpp index d5b65fe25..f9c81e6ba 100644 --- a/utils/funcexp/func_unix_timestamp.cpp +++ b/utils/funcexp/func_unix_timestamp.cpp @@ -223,4 +223,3 @@ string Func_unix_timestamp::getStrVal(rowgroup::Row& row, FunctionParm& parm, bo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index 8f7a3af08..e6ab52f35 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -169,4 +169,3 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index 302293b8e..e78d12c66 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -164,4 +164,3 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 2f4fc827b..9ea12fbdf 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -140,4 +140,3 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 3b85692bf..2ad941396 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -173,4 +173,3 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 4eed16886..3da34944c 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -361,4 +361,3 @@ string Func::longDoubleToString(long double ld) } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/utils/libmysql_client/libmysql_client.h b/utils/libmysql_client/libmysql_client.h index 4207a7054..931886b78 100644 --- a/utils/libmysql_client/libmysql_client.h +++ b/utils/libmysql_client/libmysql_client.h @@ -91,4 +91,3 @@ class LibMySQL #endif // UTILS_LIBMYSQL_CL_H -// vim:ts=4 sw=4: diff --git a/utils/loggingcpp/errorcodes.cpp b/utils/loggingcpp/errorcodes.cpp index e52a5e32d..c8e879c02 100644 --- a/utils/loggingcpp/errorcodes.cpp +++ b/utils/loggingcpp/errorcodes.cpp @@ -103,4 +103,3 @@ string ErrorCodes::errorString(uint16_t code) const return (fPreamble + msg); } } // namespace logging -// vim:ts=4 sw=4: diff --git a/utils/loggingcpp/exceptclasses.h b/utils/loggingcpp/exceptclasses.h index d1139e89b..477a9c35e 100644 --- a/utils/loggingcpp/exceptclasses.h +++ b/utils/loggingcpp/exceptclasses.h @@ -306,4 +306,3 @@ class ProtocolError : public std::logic_error } // namespace logging -// vim:ts=4 sw=4: diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index 81ed82f6d..45f605bcb 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -101,8 +101,6 @@ using boost::scoped_array; // some static functions namespace { -using messageqcpp::ByteStream; - // @bug 2441 - Retry after 512 read() error. // ERESTARTSYS (512) is a kernal I/O errno that is similar to a EINTR, except // that it is not supposed to "leak" out into the user space. But we are @@ -937,16 +935,8 @@ void InetStreamSocket::connect(const sockaddr* serv_addr) char buf = '\0'; (void)::recv(socketParms().sd(), &buf, 1, 0); #else -#if defined(__GNUC__) && __GNUC__ >= 5 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" char buf = '\0'; - ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer -#pragma GCC diagnostic pop -#else - char buf = '\0'; - ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer -#endif // pragma + std::ignore = ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer #endif return; } diff --git a/utils/messageqcpp/messagequeue.h b/utils/messageqcpp/messagequeue.h index 5dfd02f94..99a63cbdf 100644 --- a/utils/messageqcpp/messagequeue.h +++ b/utils/messageqcpp/messagequeue.h @@ -336,4 +336,3 @@ inline void MessageQueueClient::syncProto(bool use) #undef EXPORT -// vim:ts=4 sw=4: diff --git a/utils/multicast/multicast.cpp b/utils/multicast/multicast.cpp index 116f0382c..b471c170f 100644 --- a/utils/multicast/multicast.cpp +++ b/utils/multicast/multicast.cpp @@ -93,4 +93,3 @@ void MulticastSender::send(const ByteStream& msg) } // namespace multicast -// vim:ts=4 sw=4: diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 1b480aa84..d9e9a51b6 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -1405,4 +1405,3 @@ extern "C" return valOut; } } -// vim:ts=4 sw=4: diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 4e767e1d8..7c297f8fb 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1751,4 +1751,3 @@ RowGroup RowGroup::truncate(uint32_t cols) } // namespace rowgroup -// vim:ts=4 sw=4: diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 087371214..c3ecab709 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -2183,4 +2183,3 @@ inline void RGData::getRow(uint32_t num, Row* row) } // namespace rowgroup -// vim:ts=4 sw=4: diff --git a/utils/rwlock/rwlock.cpp b/utils/rwlock/rwlock.cpp index 73a9c5db2..ca7b82e9d 100644 --- a/utils/rwlock/rwlock.cpp +++ b/utils/rwlock/rwlock.cpp @@ -746,4 +746,3 @@ LockState RWLock::getLockState() } } // namespace rwlock -// vim:ts=4 sw=4: diff --git a/utils/rwlock/rwlock.h b/utils/rwlock/rwlock.h index ec47a0c70..08711bddc 100644 --- a/utils/rwlock/rwlock.h +++ b/utils/rwlock/rwlock.h @@ -275,4 +275,3 @@ class RWLock #undef EXPORT -// vim:ts=4 sw=4: diff --git a/utils/startup/installdir.cpp b/utils/startup/installdir.cpp index 65437a9c1..2db2128db 100644 --- a/utils/startup/installdir.cpp +++ b/utils/startup/installdir.cpp @@ -112,4 +112,3 @@ const string StartUp::tmpDir() } } // namespace startup -// vim:ts=4 sw=4: diff --git a/utils/startup/installdir.h b/utils/startup/installdir.h index bee02fdb4..8195601a9 100644 --- a/utils/startup/installdir.h +++ b/utils/startup/installdir.h @@ -52,4 +52,3 @@ class StartUp } // namespace startup -// vim:ts=4 sw=4: diff --git a/utils/testbc/iomanager.cpp b/utils/testbc/iomanager.cpp index 7c3cc4d6c..b2944e5b7 100644 --- a/utils/testbc/iomanager.cpp +++ b/utils/testbc/iomanager.cpp @@ -325,4 +325,3 @@ fileRequest* ioManager::getNextRequest() } } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/utils/testbc/iomanager.h b/utils/testbc/iomanager.h index 3ad953ded..f9dae5b5e 100644 --- a/utils/testbc/iomanager.h +++ b/utils/testbc/iomanager.h @@ -98,4 +98,3 @@ class ioManager }; } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/utils/threadpool/prioritythreadpool.cpp b/utils/threadpool/prioritythreadpool.cpp index e71c3bbb4..da7190f32 100644 --- a/utils/threadpool/prioritythreadpool.cpp +++ b/utils/threadpool/prioritythreadpool.cpp @@ -290,4 +290,3 @@ void PriorityThreadPool::stop() } } // namespace threadpool -// vim:ts=4 sw=4: diff --git a/utils/udfsdk/udfmysql.cpp b/utils/udfsdk/udfmysql.cpp index 552729421..f32fc6b3a 100644 --- a/utils/udfsdk/udfmysql.cpp +++ b/utils/udfsdk/udfmysql.cpp @@ -535,4 +535,3 @@ extern "C" return 0; } } -// vim:ts=4 sw=4: diff --git a/utils/udfsdk/udfsdk.cpp b/utils/udfsdk/udfsdk.cpp index 0cf806181..dc5e4fa86 100644 --- a/utils/udfsdk/udfsdk.cpp +++ b/utils/udfsdk/udfsdk.cpp @@ -375,4 +375,3 @@ int64_t MCS_isnull::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace udfsdk -// vim:ts=4 sw=4: diff --git a/utils/udfsdk/udfsdk.h b/utils/udfsdk/udfsdk.h index 9e53601b3..6c0ee013b 100644 --- a/utils/udfsdk/udfsdk.h +++ b/utils/udfsdk/udfsdk.h @@ -314,4 +314,3 @@ class MCS_isnull : public funcexp::Func #undef EXPORT -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/framebound.cpp b/utils/windowfunction/framebound.cpp index 043620f53..56b50d2e0 100644 --- a/utils/windowfunction/framebound.cpp +++ b/utils/windowfunction/framebound.cpp @@ -72,4 +72,3 @@ const string FrameBound::toString() const } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/framebound.h b/utils/windowfunction/framebound.h index 1664f9c8b..7dd4ad179 100644 --- a/utils/windowfunction/framebound.h +++ b/utils/windowfunction/framebound.h @@ -145,4 +145,3 @@ extern std::map colType2String; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/frameboundrange.cpp b/utils/windowfunction/frameboundrange.cpp index 0b356bce7..e884d9303 100644 --- a/utils/windowfunction/frameboundrange.cpp +++ b/utils/windowfunction/frameboundrange.cpp @@ -411,4 +411,3 @@ template class FrameBoundExpressionRange; template class FrameBoundExpressionRange; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/frameboundrange.h b/utils/windowfunction/frameboundrange.h index d221e59fd..5526a0b04 100644 --- a/utils/windowfunction/frameboundrange.h +++ b/utils/windowfunction/frameboundrange.h @@ -213,4 +213,3 @@ class FrameBoundExpressionRange : public FrameBoundConstantRange } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/frameboundrow.cpp b/utils/windowfunction/frameboundrow.cpp index 616b1e0eb..3d4427413 100644 --- a/utils/windowfunction/frameboundrow.cpp +++ b/utils/windowfunction/frameboundrow.cpp @@ -139,4 +139,3 @@ template class FrameBoundExpressionRow; template class FrameBoundExpressionRow; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/frameboundrow.h b/utils/windowfunction/frameboundrow.h index f7d4ec26c..79c071e49 100644 --- a/utils/windowfunction/frameboundrow.h +++ b/utils/windowfunction/frameboundrow.h @@ -146,4 +146,3 @@ class FrameBoundExpressionRow : public FrameBoundConstantRow } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 073fc4459..1de465d6f 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -900,4 +900,3 @@ bool IdbOrderBy::Eq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) } } // namespace ordering -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index c45cd2978..bc4694762 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -184,4 +184,3 @@ template boost::shared_ptr WF_count::makeFunction(i WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_count.h b/utils/windowfunction/wf_count.h index 34b880589..cb0844788 100644 --- a/utils/windowfunction/wf_count.h +++ b/utils/windowfunction/wf_count.h @@ -47,4 +47,3 @@ class WF_count : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 4e7de53ff..b3babe8b8 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -306,4 +306,3 @@ template void WF_lead_lag::parseParms(const std::vector& template void WF_lead_lag::parseParms(const std::vector&); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_lead_lag.h b/utils/windowfunction/wf_lead_lag.h index 2aac7265d..7ccb41062 100644 --- a/utils/windowfunction/wf_lead_lag.h +++ b/utils/windowfunction/wf_lead_lag.h @@ -52,4 +52,3 @@ class WF_lead_lag : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 4120fe32a..b5b63bde7 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -185,4 +185,3 @@ template boost::shared_ptr WF_min_max::makeFunction WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_min_max.h b/utils/windowfunction/wf_min_max.h index a296b8f19..caf615191 100644 --- a/utils/windowfunction/wf_min_max.h +++ b/utils/windowfunction/wf_min_max.h @@ -46,4 +46,3 @@ class WF_min_max : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 338984a04..f3c9e2a1b 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -271,4 +271,3 @@ template boost::shared_ptr WF_nth_value::makeFuncti WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_nth_value.h b/utils/windowfunction/wf_nth_value.h index a4bf2c5f3..d4a2247ff 100644 --- a/utils/windowfunction/wf_nth_value.h +++ b/utils/windowfunction/wf_nth_value.h @@ -50,4 +50,3 @@ class WF_nth_value : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_ntile.cpp b/utils/windowfunction/wf_ntile.cpp index 375fd81d2..4b6634603 100644 --- a/utils/windowfunction/wf_ntile.cpp +++ b/utils/windowfunction/wf_ntile.cpp @@ -154,4 +154,3 @@ void WF_ntile::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_ntile.h b/utils/windowfunction/wf_ntile.h index e5c7278b0..ed5ee6b77 100644 --- a/utils/windowfunction/wf_ntile.h +++ b/utils/windowfunction/wf_ntile.h @@ -47,4 +47,3 @@ class WF_ntile : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 4d0dfce07..24d6b9f28 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -382,4 +382,3 @@ template boost::shared_ptr WF_percentile::makeFunct WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_percentile.h b/utils/windowfunction/wf_percentile.h index da50a04e9..db08c08d2 100644 --- a/utils/windowfunction/wf_percentile.h +++ b/utils/windowfunction/wf_percentile.h @@ -48,4 +48,3 @@ class WF_percentile : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_ranking.cpp b/utils/windowfunction/wf_ranking.cpp index f0459b7fe..540ca8082 100644 --- a/utils/windowfunction/wf_ranking.cpp +++ b/utils/windowfunction/wf_ranking.cpp @@ -145,4 +145,3 @@ void WF_ranking::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_ranking.h b/utils/windowfunction/wf_ranking.h index 748c9607e..ecfdae97a 100644 --- a/utils/windowfunction/wf_ranking.h +++ b/utils/windowfunction/wf_ranking.h @@ -46,4 +46,3 @@ class WF_ranking : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_row_number.cpp b/utils/windowfunction/wf_row_number.cpp index 6453c0a08..6dd58ce31 100644 --- a/utils/windowfunction/wf_row_number.cpp +++ b/utils/windowfunction/wf_row_number.cpp @@ -86,4 +86,3 @@ void WF_row_number::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_row_number.h b/utils/windowfunction/wf_row_number.h index f7c4e4ca3..953a68c35 100644 --- a/utils/windowfunction/wf_row_number.h +++ b/utils/windowfunction/wf_row_number.h @@ -45,4 +45,3 @@ class WF_row_number : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index b7a01de79..d4df1b995 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -239,4 +239,3 @@ template boost::shared_ptr WF_stats::makeFunction(i WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_stats.h b/utils/windowfunction/wf_stats.h index fd72c7b3d..a443ad322 100644 --- a/utils/windowfunction/wf_stats.h +++ b/utils/windowfunction/wf_stats.h @@ -48,4 +48,3 @@ class WF_stats : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index d988f0a14..573b6e097 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -306,4 +306,3 @@ template boost::shared_ptr WF_sum_avg: int, const string&, int, WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_sum_avg.h b/utils/windowfunction/wf_sum_avg.h index 2ad329b9b..c423e7617 100644 --- a/utils/windowfunction/wf_sum_avg.h +++ b/utils/windowfunction/wf_sum_avg.h @@ -60,4 +60,3 @@ class WF_sum_avg : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 274c97018..0894c6ad1 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -1191,4 +1191,3 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) fPrev = c; } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_udaf.h b/utils/windowfunction/wf_udaf.h index e3c28e317..54b2085eb 100644 --- a/utils/windowfunction/wf_udaf.h +++ b/utils/windowfunction/wf_udaf.h @@ -112,4 +112,3 @@ class WF_udaf : public WindowFunctionType } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowframe.cpp b/utils/windowfunction/windowframe.cpp index cbc9352c6..491e0a0fb 100644 --- a/utils/windowfunction/windowframe.cpp +++ b/utils/windowfunction/windowframe.cpp @@ -81,4 +81,3 @@ const string WindowFrame::toString() const } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowframe.h b/utils/windowfunction/windowframe.h index 6b3356538..2dd3c23d7 100644 --- a/utils/windowfunction/windowframe.h +++ b/utils/windowfunction/windowframe.h @@ -119,4 +119,3 @@ class WindowFrame } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowfunction.cpp b/utils/windowfunction/windowfunction.cpp index a0e844933..941fff911 100644 --- a/utils/windowfunction/windowfunction.cpp +++ b/utils/windowfunction/windowfunction.cpp @@ -259,4 +259,3 @@ void WindowFunction::sort(std::vector::iterator v, uint64_t n) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowfunction.h b/utils/windowfunction/windowfunction.h index fc6569b27..6bdd41dde 100644 --- a/utils/windowfunction/windowfunction.h +++ b/utils/windowfunction/windowfunction.h @@ -111,4 +111,3 @@ class WindowFunction } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index ccf74609d..43f05f4bc 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -797,4 +797,3 @@ void WindowFunctionType::constParms(const std::vector& functionParms) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index 40b7a54a8..ef59444cc 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -303,4 +303,3 @@ extern std::map colType2String; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index 2c25caf8b..c5504428c 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -240,4 +240,3 @@ void BRMShmImpl::destroy() } // namespace BRM -// vim:ts=4 sw=4: diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 6235f3017..95d280e18 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -5953,4 +5953,3 @@ template int ExtentMap::getMaxMin(const LBID_t lbidRange, int64_t& max, int32_t& seqNum); } // namespace BRM -// vim:ts=4 sw=4: diff --git a/versioning/BRM/load_brm.cpp b/versioning/BRM/load_brm.cpp index 4378fb766..b72bd3c2a 100644 --- a/versioning/BRM/load_brm.cpp +++ b/versioning/BRM/load_brm.cpp @@ -127,4 +127,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index 7b665a1a6..1f490dd8c 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -2432,4 +2432,3 @@ void SlaveComm::do_dmlReleaseLBIDRanges(ByteStream& msg) } // namespace BRM -// vim:ts=4 sw=4: diff --git a/versioning/BRM/vbbm.cpp b/versioning/BRM/vbbm.cpp index 253bd0512..2ed2454d1 100644 --- a/versioning/BRM/vbbm.cpp +++ b/versioning/BRM/vbbm.cpp @@ -40,7 +40,6 @@ #include #include -namespace bi = boost::interprocess; #include #include @@ -1076,8 +1075,8 @@ void VBBM::save(string filename) } var = VBBM_MAGIC_V2; - int bytesWritten = 0; - int bytesToWrite = 12; + [[maybe_unused]] int bytesWritten = 0; + [[maybe_unused]] int bytesToWrite = 12; bytesWritten += out->write((char*)&var, 4); bytesWritten += out->write((char*)&vbbm->vbCurrentSize, 4); bytesWritten += out->write((char*)&vbbm->nFiles, 4); diff --git a/writeengine/client/we_clients.cpp b/writeengine/client/we_clients.cpp index c696dbb6d..b6d66e2e9 100644 --- a/writeengine/client/we_clients.cpp +++ b/writeengine/client/we_clients.cpp @@ -545,4 +545,3 @@ void WEClients::addDataToOutput(SBS sbs, uint32_t connIndex) } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/client/we_clients.h b/writeengine/client/we_clients.h index 82c40a0e8..6b1a59d7f 100644 --- a/writeengine/client/we_clients.h +++ b/writeengine/client/we_clients.h @@ -180,4 +180,3 @@ class WEClients #undef EXPORT -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistribute.cpp b/writeengine/redistribute/we_redistribute.cpp index d155715b5..9f9101935 100644 --- a/writeengine/redistribute/we_redistribute.cpp +++ b/writeengine/redistribute/we_redistribute.cpp @@ -52,4 +52,3 @@ void Redistribute::handleRedistributeMessage(ByteStream& bs, IOSocket& ios) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistribute.h b/writeengine/redistribute/we_redistribute.h index 4af403484..121efb11e 100644 --- a/writeengine/redistribute/we_redistribute.h +++ b/writeengine/redistribute/we_redistribute.h @@ -44,4 +44,3 @@ class Redistribute } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrol.cpp b/writeengine/redistribute/we_redistributecontrol.cpp index e84bc9478..76a4892e5 100644 --- a/writeengine/redistribute/we_redistributecontrol.cpp +++ b/writeengine/redistribute/we_redistributecontrol.cpp @@ -698,4 +698,3 @@ void RedistributeControl::logMessage(const string& msg) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrol.h b/writeengine/redistribute/we_redistributecontrol.h index 3dcf350e4..13359344e 100644 --- a/writeengine/redistribute/we_redistributecontrol.h +++ b/writeengine/redistribute/we_redistributecontrol.h @@ -126,4 +126,3 @@ class RedistributeControl } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index ce808eb20..774f40118 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -838,4 +838,3 @@ void RedistributeControlThread::doStopAction() } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributecontrolthread.h b/writeengine/redistribute/we_redistributecontrolthread.h index ea32f3f5c..9b4c88ac8 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.h +++ b/writeengine/redistribute/we_redistributecontrolthread.h @@ -125,4 +125,3 @@ class RedistributeControlThread } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributedef.h b/writeengine/redistribute/we_redistributedef.h index 7ddcf4e5e..4b1d7a895 100644 --- a/writeengine/redistribute/we_redistributedef.h +++ b/writeengine/redistribute/we_redistributedef.h @@ -203,4 +203,3 @@ struct RedistributeInfo } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributeworkerthread.cpp b/writeengine/redistribute/we_redistributeworkerthread.cpp index 6f4509839..822caff24 100644 --- a/writeengine/redistribute/we_redistributeworkerthread.cpp +++ b/writeengine/redistribute/we_redistributeworkerthread.cpp @@ -1520,4 +1520,3 @@ void RedistributeWorkerThread::logMessage(const string& msg, int line) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/redistribute/we_redistributeworkerthread.h b/writeengine/redistribute/we_redistributeworkerthread.h index c989cd286..9dc5fa417 100644 --- a/writeengine/redistribute/we_redistributeworkerthread.h +++ b/writeengine/redistribute/we_redistributeworkerthread.h @@ -141,4 +141,3 @@ class RedistributeWorkerThread } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 1d832f9e0..3970e6c1e 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -4952,4 +4952,3 @@ void WE_DDLCommandProc::purgeFDCache() } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/shared/we_brm.cpp b/writeengine/shared/we_brm.cpp index a049806b1..fb5d88fac 100644 --- a/writeengine/shared/we_brm.cpp +++ b/writeengine/shared/we_brm.cpp @@ -1741,4 +1741,3 @@ int BRMWrapper::getExtentCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMi } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index d2ab6d324..1254176ef 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -2640,4 +2640,3 @@ int ChunkManager::checkFixLastDictChunk(const FID& fid, uint16_t root, uint32_t } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index 366c64ad7..22a29100a 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -572,4 +572,3 @@ int main(int argc, char** argv) return SPLTR_EXIT_STATUS; } -// vim:ts=4 sw=4: diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 25bed1ebe..bbe3dc42a 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1910,4 +1910,3 @@ int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/wrapper/we_tablemetadata.cpp b/writeengine/wrapper/we_tablemetadata.cpp index 64bfff06d..421ecbcb8 100644 --- a/writeengine/wrapper/we_tablemetadata.cpp +++ b/writeengine/wrapper/we_tablemetadata.cpp @@ -109,4 +109,3 @@ ColsExtsInfoMap& TableMetaData::getColsExtsInfoMap() return fColsExtsInfoMap; } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/writeengine/wrapper/we_tablemetadata.h b/writeengine/wrapper/we_tablemetadata.h index 1202d6afd..4a753c960 100644 --- a/writeengine/wrapper/we_tablemetadata.h +++ b/writeengine/wrapper/we_tablemetadata.h @@ -98,4 +98,3 @@ class TableMetaData #undef EXPORT -// vim:ts=4 sw=4: diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 815204120..414714725 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -6628,4 +6628,3 @@ int WriteEngineWrapper::RemoveTxnFromLBIDMap(const TxnID txnid) } } // end of namespace -// vim:ts=4 sw=4: From 6b1c69699174b0583c08be17f5aa082f8ccd07e9 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Thu, 17 Feb 2022 22:23:52 +0000 Subject: [PATCH 08/55] chars are unsigned on arm y default --- writeengine/splitter/we_filereadthread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index 4faab3fa6..a21d5ef1f 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -580,7 +580,7 @@ int WEFileReadThread::getNextRow(istream& ifs, char* pBuf, int MaxLen) const char ESC = fEsc; bool aTrailEsc = false; char* pEnd = pBuf; - char aCh = ifs.get(); + int aCh = ifs.get(); while (ifs.good()) { From b5ccd52c0933fd7556da601cdc1a6e5254f90b11 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Thu, 17 Feb 2022 14:22:49 +0000 Subject: [PATCH 09/55] OpenSSL 3 support for Columnstore --- dbcon/joblist/CMakeLists.txt | 2 +- tools/passwd/CMakeLists.txt | 7 ++++--- tools/passwd/cskeys.cpp | 7 +++++++ tools/passwd/secrets.cpp | 13 +++++++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 77fca4a5e..1bbff02b4 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -59,7 +59,7 @@ set(joblist_LIB_SRCS ${ENGINE_SRC_DIR}/tools/passwd/secrets.cpp) add_library(joblist SHARED ${joblist_LIB_SRCS}) - +target_include_directories(joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) add_dependencies(joblist loggingcpp) install(TARGETS joblist DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/tools/passwd/CMakeLists.txt b/tools/passwd/CMakeLists.txt index cc9e7e330..836b2219a 100644 --- a/tools/passwd/CMakeLists.txt +++ b/tools/passwd/CMakeLists.txt @@ -2,15 +2,16 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### - set(cspasswd_SRCS cspasswd.cpp secrets.cpp) set(cskeys_SRCS cskeys.cpp secrets.cpp) add_executable(cspasswd ${cspasswd_SRCS}) add_executable(cskeys ${cskeys_SRCS}) -target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +target_include_directories(cspasswd BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) +target_include_directories(cskeys BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) +target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES}) +target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES}) install(TARGETS cspasswd DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) install(TARGETS cskeys DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/tools/passwd/cskeys.cpp b/tools/passwd/cskeys.cpp index 181503605..f0d61959c 100644 --- a/tools/passwd/cskeys.cpp +++ b/tools/passwd/cskeys.cpp @@ -18,10 +18,17 @@ #include #include #include +#include #include "secrets.h" #include "mcsconfig.h" +#ifdef OPENSSL_VERSION_PREREQ +#if OPENSSL_VERSION_PREREQ(3,0) + #define EVP_CIPHER_key_length EVP_CIPHER_get_key_length +#endif +#endif + using std::string; using ByteVec = std::vector; diff --git a/tools/passwd/secrets.cpp b/tools/passwd/secrets.cpp index a7635e880..aac32ffa5 100644 --- a/tools/passwd/secrets.cpp +++ b/tools/passwd/secrets.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #define BOOST_SPIRIT_THREADSAFE #include @@ -34,6 +35,15 @@ using std::string; + +#ifdef OPENSSL_VERSION_PREREQ +#if OPENSSL_VERSION_PREREQ(3,0) + #define EVP_CIPHER_key_length EVP_CIPHER_get_key_length + #define EVP_CIPHER_iv_length EVP_CIPHER_get_iv_length + #define EVP_CIPHER_blocksize EVP_CIPHER_get_blocksize +#endif +#endif + const char* const SECRETS_FILENAME = ".secrets"; namespace @@ -469,6 +479,8 @@ string decrypt_password(const string& input) * @param input Encrypted password in hex form * @return Decrypted password or empty on error */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" string decrypt_password_old(const ByteVec& key, const ByteVec& iv, const std::string& input) { string rval; @@ -501,6 +513,7 @@ string decrypt_password_old(const ByteVec& key, const ByteVec& iv, const std::st } return rval; } +#pragma GCC diagnostic pop string decrypt_password(const ByteVec& key, const std::string& input) { From 0b0efe338633261b652d5d5ca5e097815da12b5a Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 21 Feb 2022 14:19:19 +0300 Subject: [PATCH 10/55] Centos Bootstrap fixes --- build/bootstrap_mcs.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index ccfde5e0e..f9f42cfdd 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -20,12 +20,13 @@ fi message "Building Mariadb Server from $MDB_SOURCE_PATH" BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") -DISTRO_OPTIONS=("Ubuntu" "Centos" "Debian" "openSUSE") +DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "openSUSE") optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true +optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true source $( optparse.build ) @@ -50,11 +51,12 @@ install_deps() libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev - elif [ $OS = 'CentOS Linux' ]; then + elif [ $OS = 'CentOS' ]; then yum -y install epel-release \ && yum -y groupinstall "Development Tools" \ - && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu \ - && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel gtest-devel cppunit-devel + && yum config-manager --set-enabled powertools \ + && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ + && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel elif [ $OS = 'OpenSuse' ]; then zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel \ @@ -126,6 +128,15 @@ build() cd $MDB_SOURCE_PATH + if [[ $SKIP_SUBMODULES = true ]] ; then + warn "Skipping initialization of columnstore submodules" + else + message "Initialization of columnstore submodules" + cd storage/columnstore/columnstore + git submodule update --init + cd - + fi + if [[ $FORCE_CMAKE_CONFIG = true ]] ; then warn "Erasing cmake cache" rm -f "$MDB_SOURCE_PATH/CMakeCache.txt" @@ -134,7 +145,7 @@ build() if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" - elif [ $OS = 'Centos' ]; then + elif [ $OS = 'CentOS' ]; then MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" fi From 45733c421a88e3fda8d38c8189c5cb7e41021514 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 21 Feb 2022 15:09:44 +0300 Subject: [PATCH 11/55] openSUSE bootstrap fix --- build/bootstrap_mcs.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index f9f42cfdd..a8fffec98 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -57,11 +57,12 @@ install_deps() && yum config-manager --set-enabled powertools \ && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel - elif [ $OS = 'OpenSuse' ]; then + elif [ $OS = 'openSUSE' ]; then zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ - && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel \ - && zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel \ - && zypper install -y gcc gcc-c++ git automake libtool gtest cppunit-devel + && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel \ + && zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel liblz4-devel lz4 \ + && zypper install -y gcc gcc-c++ git automake libtool gtest cppunit-devel pcre2-devel systemd-devel libaio-devel snappy-devel cracklib-devel policycoreutils-devel ncurses-devel \ + && zypper install -y make flex libcurl-devel automake libtool rpm-build lsof iproute pam-devel perl-DBI expect createrepo fi } @@ -131,10 +132,10 @@ build() if [[ $SKIP_SUBMODULES = true ]] ; then warn "Skipping initialization of columnstore submodules" else - message "Initialization of columnstore submodules" - cd storage/columnstore/columnstore - git submodule update --init - cd - + message "Initialization of columnstore submodules" + cd storage/columnstore/columnstore + git submodule update --init + cd - fi if [[ $FORCE_CMAKE_CONFIG = true ]] ; then @@ -147,6 +148,8 @@ build() MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" elif [ $OS = 'CentOS' ]; then MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" + elif [ $OS = 'openSUSE' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15" fi message "building with flags $MDB_CMAKE_FLAGS" From 4b412d4e09b1445288801567c84548561635641a Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Mon, 21 Feb 2022 16:09:31 -0600 Subject: [PATCH 12/55] MCOL-4940: test case for ROUND fix (#2271) --- mysql-test/columnstore/bugfixes/mcol-4940.result | 11 +++++++++++ mysql-test/columnstore/bugfixes/mcol-4940.test | 14 ++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 mysql-test/columnstore/bugfixes/mcol-4940.result create mode 100644 mysql-test/columnstore/bugfixes/mcol-4940.test diff --git a/mysql-test/columnstore/bugfixes/mcol-4940.result b/mysql-test/columnstore/bugfixes/mcol-4940.result new file mode 100644 index 000000000..0ca364b2b --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-4940.result @@ -0,0 +1,11 @@ +DROP DATABASE IF EXISTS mcol_4940; +CREATE DATABASE mcol_4940; +USE mcol_4940; +create table rounding_table ( a int, b double, c double) engine=columnstore; +insert into rounding_table values (26805, 1252, -9647); +insert into rounding_table values (26806, 573, -2804.5); +SELECT CASE a WHEN 26805 THEN ROUND(c/b, 2) WHEN 26806 THEN b END MCOL4940 FROM ( SELECT a, SUM(b) b, SUM(c) c FROM rounding_table GROUP BY a ) abc ; +MCOL4940 +573 +-7.71 +DROP DATABASE mcol_4940; diff --git a/mysql-test/columnstore/bugfixes/mcol-4940.test b/mysql-test/columnstore/bugfixes/mcol-4940.test new file mode 100644 index 000000000..4d5ceb23d --- /dev/null +++ b/mysql-test/columnstore/bugfixes/mcol-4940.test @@ -0,0 +1,14 @@ +--source ../include/have_columnstore.inc +--disable_warnings +DROP DATABASE IF EXISTS mcol_4940; +--enable_warnings +CREATE DATABASE mcol_4940; +USE mcol_4940; + +create table rounding_table ( a int, b double, c double) engine=columnstore; +insert into rounding_table values (26805, 1252, -9647); +insert into rounding_table values (26806, 573, -2804.5); + +SELECT CASE a WHEN 26805 THEN ROUND(c/b, 2) WHEN 26806 THEN b END MCOL4940 FROM ( SELECT a, SUM(b) b, SUM(c) c FROM rounding_table GROUP BY a ) abc ; + +DROP DATABASE mcol_4940; From e35ed080986dc1865619fa6545b021caf273f6ca Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 22 Feb 2022 10:46:50 +0000 Subject: [PATCH 13/55] Another portion of vscode-specific dynamic files/paths to ignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index c35a65112..d1f85955f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,10 @@ *.so.* *.vpj .libs/ +.cache/ +compile_commands.json +*.code-workspace +settings.json config.log *~ *kdev* From 23c79968f647676ebf04e74a98a42309d50b2169 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Tue, 22 Feb 2022 16:45:26 +0300 Subject: [PATCH 14/55] Build Type was missed for build --- build/bootstrap_mcs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index a8fffec98..3cfea4319 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -155,7 +155,7 @@ build() message "building with flags $MDB_CMAKE_FLAGS" local CPUS=$(getconf _NPROCESSORS_ONLN) - cmake . -DCMAKE_BUILD_TYPE=$MDB_BUILD_TYPE $MDB_CMAKE_FLAGS && \ + cmake . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ make -j $CPUS install if [ $? -ne 0 ]; then From de6a51f7db849dab82b04590e77303cb42b357d1 Mon Sep 17 00:00:00 2001 From: mariadb-AlanMologorsky Date: Tue, 22 Feb 2022 07:15:39 +0300 Subject: [PATCH 15/55] Centos8 EOL fix [fix] .drone.jsonnet changing yum_vault_mirror, then centos:8 tests stop going to be flacky [fix] CI .drone.jsonnet duplicate usage of yum_vault_mirror --- .drone.jsonnet | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 87cd0e6a6..18c228432 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -39,7 +39,7 @@ local gcc_update_alternatives = 'update-alternatives --install /usr/bin/gcc gcc local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 '; -local yum_vault_mirror = "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-*; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-*; "; +local yum_vault_mirror = "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-*; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-*; dnf update -y; "; local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo '; local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; @@ -370,7 +370,6 @@ local Pipeline(branch, platform, event, arch='amd64') = { }, commands: [ 'cd /mdb/' + builddir, - if (platform == 'centos:8') then yum_vault_mirror else '', // Remove Debian build flags that could prevent ColumnStore from building "sed '/-DPLUGIN_COLUMNSTORE=NO/d' -i debian/rules", // Tweak debian packaging stuff From 86e495ae2ff19590dd1e0702d74dc3257965ca55 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 22 Feb 2022 16:50:12 +0000 Subject: [PATCH 16/55] MCOL-4809 This fixes Centos crash caused by combination of ::reserve() + vector index access --- primitives/linux-port/column.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 5ca4c3f6f..7e840373e 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1300,7 +1300,7 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* for (uint32_t j = 0; j < filterCount; ++j) { // Preload filter argument values only once. - filterArgsVectors[j] = simdProcessor.loadValue(*((FilterType*)&filterValues[j])); + filterArgsVectors.push_back(simdProcessor.loadValue(*((FilterType*)&filterValues[j]))); switch (filterCOPs[j]) { case (COMPARE_EQ): From dc4f6865d15a144d5b0bf218ea80f74b3491fd8c Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 23 Feb 2022 12:21:43 -0600 Subject: [PATCH 17/55] MCOL-3983: always set threadpool max threads in storagemanager (#2269) --- storage-manager/src/Downloader.cpp | 4 ++++ storage-manager/src/Synchronizer.cpp | 4 ++++ storage-manager/storagemanager.cnf.in | 6 ------ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/storage-manager/src/Downloader.cpp b/storage-manager/src/Downloader.cpp index 7d21b7b4f..1a0897f9f 100644 --- a/storage-manager/src/Downloader.cpp +++ b/storage-manager/src/Downloader.cpp @@ -214,7 +214,11 @@ void Downloader::configListener() // Downloader threads string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_downloads"); if (maxDownloads == 0) + { maxDownloads = 20; + workers.setMaxThreads(maxDownloads); + logger->log(LOG_INFO, "max_concurrent_downloads = %u",maxDownloads); + } if (stmp.empty()) { logger->log(LOG_CRIT, "max_concurrent_downloads is not set. Using current value = %u", maxDownloads); diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index ed02f5eb1..1c48f0887 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -884,7 +884,11 @@ void Synchronizer::configListener() // Uploader threads string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_uploads"); if (maxUploads == 0) + { maxUploads = 20; + threadPool->setMaxThreads(maxUploads); + logger->log(LOG_INFO, "max_concurrent_uploads = %u",maxUploads); + } if (stmp.empty()) { logger->log(LOG_CRIT, "max_concurrent_uploads is not set. Using current value = %u", maxUploads); diff --git a/storage-manager/storagemanager.cnf.in b/storage-manager/storagemanager.cnf.in index a1b086d4e..1b95faf5a 100644 --- a/storage-manager/storagemanager.cnf.in +++ b/storage-manager/storagemanager.cnf.in @@ -55,9 +55,6 @@ journal_path = @ENGINE_DATADIR@/storagemanager/journal # max_concurrent_downloads is what is sounds like, per node. # This is not a global setting. -# -# The default was changed from 20 to 21 as a temporary workaround -# for a bug. It can be anything but 20. max_concurrent_downloads = 21 # max_concurrent_uploads is what is sounds like, per node. @@ -66,9 +63,6 @@ max_concurrent_downloads = 21 # has low upstream bandwidth, consider lowering this value to the minimum # necessary to saturate your network. This will reduce the latency of certain # operations and improve your experience. -# -# The default was changed from 20 to 21 as a temporary workaround -# for a bug. It can be anything but 20. max_concurrent_uploads = 21 # common_prefix_depth is the depth of the common prefix that all files From 5a577553f794a0323d924b2ab10e300b4e5dcaee Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Thu, 24 Feb 2022 10:05:49 +0300 Subject: [PATCH 18/55] Hex double convertion --- utils/funcexp/func_hex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index 0e550d0e7..c1a111cb7 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -95,7 +95,7 @@ string Func_hex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, if ((val <= (double)numeric_limits::min()) || (val >= (double)numeric_limits::max())) dec = ~(int64_t)0; else - dec = (uint64_t)(val + (val > 0 ? 0.5 : -0.5)); + dec = static_cast(static_cast(val + (val > 0 ? 0.5 : -0.5))); retval = helpers::convNumToStr(dec, ans, 16); break; From b46f4b42b3ea1dd4ab2923bb73378b48e9fff065 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 22 Feb 2022 10:43:04 +0000 Subject: [PATCH 19/55] MCOL-4809 Vectorized comparison operations unit tests This commit replaces system googletest with 0.11.1 version compiled from sources to enable typed tests feature --- .drone.jsonnet | 26 ++++++++-------- CMakeLists.txt | 8 ----- cmake/FindGTest.cmake | 37 ---------------------- tests/CMakeLists.txt | 27 ++++++++++++++-- tests/simd_processors.cpp | 65 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 61 deletions(-) delete mode 100644 cmake/FindGTest.cmake create mode 100644 tests/simd_processors.cpp diff --git a/.drone.jsonnet b/.drone.jsonnet index 18c228432..b2d9d3f18 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -41,13 +41,13 @@ local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clan local yum_vault_mirror = "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-*; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-*; dnf update -y; "; local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo '; -local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; -local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; -local ubuntu18_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; -local debian10_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; -local opensuse_build_deps = 'zypper install -y liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + gcc_update_alternatives; -local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; -local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 && ' + gcc_update_alternatives; +local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y git pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; +local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake git lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; +local ubuntu18_04_deps = 'apt update && apt install -y gnupg git wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; +local debian10_deps = 'apt update && apt install -y gnupg git wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; +local opensuse_build_deps = 'zypper install -y git liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + gcc_update_alternatives; +local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; +local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 git && ' + gcc_update_alternatives; local platformMap(platform) = @@ -76,12 +76,12 @@ local testRun(platform) = local testPreparation(platform) = local platform_map = { - 'opensuse/leap:15': 'zypper install -y gtest boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake', - 'centos:7': 'yum -y install epel-release && yum install -y gtest-devel cppunit-devel cmake3 boost-devel snappy-devel', - 'centos:8': yum_vault_mirror + ' yum install -y dnf-plugins-core libarchive && yum config-manager --set-enabled powertools && yum install -y lz4 gtest-devel cppunit-devel cmake3 boost-devel snappy-devel', - 'debian:10': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev libsnappy-dev googletest cmake', - 'ubuntu:18.04': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev googletest libsnappy-dev cmake g++ && cd /usr/src/googletest; cmake . && cmake --build . --target install; cd -', - 'ubuntu:20.04': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev googletest libsnappy-dev cmake', + 'opensuse/leap:15': 'zypper install -y git boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake', + 'centos:7': 'yum -y install epel-release && yum install -y git cppunit-devel cmake3 boost-devel snappy-devel', + 'centos:8': yum_vault_mirror + ' yum install -y dnf-plugins-core libarchive && yum config-manager --set-enabled powertools && yum install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', + 'debian:10': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', + 'ubuntu:18.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake g++', + 'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', }; platform_map[platform]; diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b6fbf260..067f1c7e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,14 +149,6 @@ if (NOT CURL_FOUND) return() endif() -IF (WITH_UNITTESTS OR WITH_GTEST) - INCLUDE (FindGTest) - IF (NOT GTEST_FOUND) - MESSAGE(FATAL_ERROR "GSuite libs not found but are requested. Please install them or build.") - ENDIF() - SET (GTEST_LIBRARIES ${GTEST_LIBRARY} ${GTESTMAIN_LIBRARY} ${PTHREAD_LIBRARY}) -ENDIF() - FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) MESSAGE_ONCE(CS_NO_AWK "awk not found!") diff --git a/cmake/FindGTest.cmake b/cmake/FindGTest.cmake deleted file mode 100644 index 5860ca415..000000000 --- a/cmake/FindGTest.cmake +++ /dev/null @@ -1,37 +0,0 @@ -find_path(GTEST_ROOT_DIR - NAMES include/gtest/gtest.h -) - -find_library(GTEST_LIBRARY - NAMES gtest - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_library(GTESTMAIN_LIBRARY - NAMES gtest_main - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_library(PTHREAD_LIBRARY - NAMES pthread - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_path(GTEST_INCLUDE_DIR - NAMES gtest.h - HINTS ${GTEST_ROOT_DIR}/include/gtest -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GTest DEFAULT_MSG - GTEST_LIBRARY - GTESTMAIN_LIBRARY - PTHREAD_LIBRARY - GTEST_INCLUDE_DIR -) - -mark_as_advanced( - GTEST_ROOT_DIR - GTEST_LIBRARIES - GTEST_INCLUDE_DIR -) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index abbba30e4..b00c0482a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,35 +1,56 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}) +include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) if (WITH_UNITTESTS) - enable_testing() + set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external) + ExternalProject_Add(googletest + GIT_REPOSITORY https://github.com/google/googletest + GIT_TAG release-1.11.0 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DBUILD_SHARED_LIBS=ON + ) + + include_directories(${EXTERNAL_INSTALL_LOCATION}/include) + # lib64 for RPM-based distros + link_directories(${EXTERNAL_INSTALL_LOCATION}/lib ${EXTERNAL_INSTALL_LOCATION}/lib64) + set(GTEST_LIBRARIES gtest gtest_main pthread) include(GoogleTest) - find_package(GTest REQUIRED) #GoogleTest tests add_executable(rowgroup_tests rowgroup-tests.cpp) + add_dependencies(rowgroup_tests googletest) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(rowgroup_tests TEST_PREFIX columnstore:) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) + add_dependencies(mcs_decimal_tests googletest) target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(mcs_decimal_tests TEST_PREFIX columnstore:) add_executable(dataconvert_tests dataconvert-tests.cpp) + add_dependencies(dataconvert_tests googletest) target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(dataconvert_tests TEST_PREFIX columnstore:) add_executable(rebuild_em_tests rebuild-em-tests.cpp) + add_dependencies(rebuild_em_tests googletest) target_link_libraries(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) gtest_discover_tests(rebuild_em_tests TEST_PREFIX columnstore:) add_executable(compression_tests compression-tests.cpp) + add_dependencies(compression_tests googletest) target_link_libraries(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) gtest_discover_tests(compression_tests TEST_PREFIX columnstore:) add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp) + target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare) + add_dependencies(column_scan_filter_tests googletest) target_link_libraries(column_scan_filter_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) gtest_discover_tests(column_scan_filter_tests TEST_PREFIX columnstore:) + add_executable(simd_processors simd_processors.cpp) + add_dependencies(simd_processors googletest) + target_link_libraries(simd_processors ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) + gtest_discover_tests(simd_processors TEST_PREFIX columnstore:) + # CPPUNIT TESTS add_executable(we_shared_components_tests shared_components_tests.cpp) add_dependencies(we_shared_components_tests loggingcpp) diff --git a/tests/simd_processors.cpp b/tests/simd_processors.cpp new file mode 100644 index 000000000..519d02845 --- /dev/null +++ b/tests/simd_processors.cpp @@ -0,0 +1,65 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#if defined(__x86_64__) +#include +#include + +#include "simd_sse.h" +#include "datatypes/mcs_datatype.h" +#include "datatypes/mcs_int128.h" + +using namespace std; + +template +class SimdProcessorTypedTest : public testing::Test { + using IntegralType = T; + public: + + void SetUp() override + { + } +}; + +using SimdProcessor128TypedTestTypes = ::testing::Types; +TYPED_TEST_SUITE(SimdProcessorTypedTest, SimdProcessor128TypedTestTypes); + +TYPED_TEST(SimdProcessorTypedTest, SimdFilterProcessor_simd128) +{ + using Proc = typename simd::SimdFilterProcessor; + using SimdType = typename Proc::SimdType; + constexpr static simd::MT allTrue = 0xFFFF; + constexpr static simd::MT allFalse = 0x0; + Proc proc; + SimdType lhs = proc.loadValue((TypeParam)-2); + SimdType rhs = proc.loadValue((TypeParam)-3); + EXPECT_GT((uint64_t)-2LL, (uint64_t)-3LL); + EXPECT_EQ(proc.cmpGe(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpGt(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpGe(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpGt(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpLe(rhs, lhs), allTrue); + EXPECT_EQ(proc.cmpLt(rhs, lhs), allTrue); + EXPECT_EQ(proc.cmpLe(lhs, rhs), allFalse); + EXPECT_EQ(proc.cmpLt(lhs, rhs), allFalse); + EXPECT_EQ(proc.cmpEq(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpNe(rhs, lhs), allTrue); + lhs = proc.loadValue((TypeParam)-3); + EXPECT_EQ(proc.cmpEq(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpNe(rhs, lhs), allFalse); +} +#endif \ No newline at end of file From 806ff55e817334b094212801415e496b7436af94 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 28 Feb 2022 15:07:24 +0000 Subject: [PATCH 20/55] Centos 7 support and options for branches --- build/bootstrap_mcs.sh | 62 ++++++++++++++++++++++++++++++++++-------- build/utils.sh | 2 +- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 3cfea4319..e7f90f73a 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -17,16 +17,18 @@ if [ "$EUID" -ne 0 ] exit 1 fi -message "Building Mariadb Server from $MDB_SOURCE_PATH" +message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "openSUSE") +BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true +optparse.define short=b long=branch desc="Choouse git branch ('none' for menu)" variable=BRANCH source $( optparse.build ) @@ -35,12 +37,34 @@ if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then MCS_BUILD_TYPE=$selectedChoice fi -if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then +if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "Centos" ]]; then detect_distro fi INSTALL_PREFIX="/usr/" DATA_DIR="/var/lib/mysql/data" +CMAKE_BIN_NAME=cmake + +select_branch() +{ + if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then + if [[ $BRANCH = 'none' ]]; then + getChoice -q "Select your branch" -o BRANCHES + BRANCH=$selectedChoice + fi + cd $SCRIPT_LOCATION + message "Selecting $BRANCH branch for Columnstore" + git checkout $BRANCH + cd - + + message "Turning off Columnstore submodule auto update via gitconfig" + cd $MDB_SOURCE_PATH + git config submodule.storage/columnstore/columnstore.update none + cd - + fi + CURRENT_BRANCH=$(git branch --show-current) + message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch" +} install_deps() { @@ -55,8 +79,14 @@ install_deps() yum -y install epel-release \ && yum -y groupinstall "Development Tools" \ && yum config-manager --set-enabled powertools \ - && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ + && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel + if [[ "$OS_VERSION" == "7" ]]; then + yum -y install cmake3 + CMAKE_BIN_NAME=cmake3 + else + yum -y install cmake + fi elif [ $OS = 'openSUSE' ]; then zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel \ @@ -76,7 +106,7 @@ stop_service() check_service() { if systemctl is-active --quiet $1; then - message "$1 service started OK" + message "$1 service started$color_green OK $color_normal" else error "$1 service failed" service $1 status @@ -110,7 +140,8 @@ clean_old_installation() build() { - message "Building sources" + message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode" + local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO @@ -120,6 +151,7 @@ build() -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO + -DWITH_EMBEDDED_SERVER=OFF -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF -DWITH_SSL=system @@ -155,7 +187,7 @@ build() message "building with flags $MDB_CMAKE_FLAGS" local CPUS=$(getconf _NPROCESSORS_ONLN) - cmake . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ + ${CMAKE_BIN_NAME} . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ make -j $CPUS install if [ $? -ne 0 ]; then @@ -165,20 +197,26 @@ build() cd - } -install() + +check_user_and_group() { - message "Installing MariaDB" if [ -z "$(grep mysql /etc/passwd)" ]; then message "Adding user mysql into /etc/passwd" useradd -r -U mysql -d /var/lib/mysql fi if [ -z "$(grep mysql /etc/group)" ]; then - echo "You need to manually add mysql group into /etc/group, e.g. mysql:x:999" GroupID = `awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group` message "Adding group mysql with id $GroupID" groupadd -g GroupID mysql fi +} + +install() +{ + message "Installing MariaDB" + + check_user_and_group mkdir -p /etc/my.cnf.d @@ -189,7 +227,6 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' message "Running mysql_install_db" mysql_install_db --rpm --user=mysql mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so - chown mysql:mysql $INSTALL_PREFIX/lib/plugin/ha_columnstore.so mkdir -p /etc/columnstore @@ -244,6 +281,9 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' chmod 777 /var/log/mariadb/columnstore } + +select_branch + if [[ $SKIP_DEPS = false ]] ; then install_deps fi @@ -253,4 +293,4 @@ clean_old_installation build install start_service -message "FINISHED" +message "$color_green FINISHED $color_normal" diff --git a/build/utils.sh b/build/utils.sh index 100f968b6..133bfb036 100644 --- a/build/utils.sh +++ b/build/utils.sh @@ -54,7 +54,7 @@ detect_distro() OS_VERSION=$(uname -r) fi OS=$(echo $OS | cut -f 1 -d " ") - message "Detected $OS $OS_VERSION" + message "Detected $color_yellow$OS $OS_VERSION$color_normal" } menuStr="" From 22d0ba4c4abe67efeb3a5939191123ec36ef3bf2 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 24 Feb 2022 18:47:17 +0000 Subject: [PATCH 21/55] MCOL-5001 EM code preparation to merge EM and PP runtimes --- .drone.jsonnet | 11 +- exemgr/CMakeLists.txt | 2 +- exemgr/ExeMgr.rc | 102 -- exemgr/exemgr.cpp | 102 ++ exemgr/main.cpp | 1827 ------------------------------ exemgr/resource.h | 14 - exemgr/rssmonfcn.cpp | 69 ++ exemgr/rssmonfcn.h | 36 + exemgr/serviceexemgr.cpp | 475 ++++++++ exemgr/serviceexemgr.h | 342 ++++++ exemgr/sqlfrontsessionthread.cpp | 985 ++++++++++++++++ exemgr/sqlfrontsessionthread.h | 131 +++ 12 files changed, 2146 insertions(+), 1950 deletions(-) delete mode 100644 exemgr/ExeMgr.rc create mode 100644 exemgr/exemgr.cpp delete mode 100644 exemgr/main.cpp delete mode 100644 exemgr/resource.h create mode 100644 exemgr/rssmonfcn.cpp create mode 100644 exemgr/rssmonfcn.h create mode 100644 exemgr/serviceexemgr.cpp create mode 100644 exemgr/serviceexemgr.h create mode 100644 exemgr/sqlfrontsessionthread.cpp create mode 100644 exemgr/sqlfrontsessionthread.h diff --git a/.drone.jsonnet b/.drone.jsonnet index b2d9d3f18..69cea1e4e 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -41,15 +41,14 @@ local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clan local yum_vault_mirror = "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-*; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-*; dnf update -y; "; local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo '; -local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y git pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; -local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake git lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; -local ubuntu18_04_deps = 'apt update && apt install -y gnupg git wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; -local debian10_deps = 'apt update && apt install -y gnupg git wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; -local opensuse_build_deps = 'zypper install -y git liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + gcc_update_alternatives; +local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; +local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; +local ubuntu18_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; +local debian10_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; +local opensuse_build_deps = 'zypper install -y liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + gcc_update_alternatives; local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 git && ' + gcc_update_alternatives; - local platformMap(platform) = local platform_map = { 'opensuse/leap:15': opensuse_build_deps + ' && zypper ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=sles15 && make -j$(nproc) package', diff --git a/exemgr/CMakeLists.txt b/exemgr/CMakeLists.txt index e869e4371..729cc98cd 100644 --- a/exemgr/CMakeLists.txt +++ b/exemgr/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) +set(ExeMgr_SRCS serviceexemgr.cpp sqlfrontsessionthread.cpp rssmonfcn.cpp exemgr.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) add_executable(ExeMgr ${ExeMgr_SRCS}) diff --git a/exemgr/ExeMgr.rc b/exemgr/ExeMgr.rc deleted file mode 100644 index 02d4d6817..000000000 --- a/exemgr/ExeMgr.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB Execution Manager" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "ExeMgr" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "ExeMgr.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/exemgr/exemgr.cpp b/exemgr/exemgr.cpp new file mode 100644 index 000000000..fa1acec8d --- /dev/null +++ b/exemgr/exemgr.cpp @@ -0,0 +1,102 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/********************************************************************** + * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ + * + * + ***********************************************************************/ +/** + * @brief execution plan manager main program + * + * This is the ?main? program for dealing with execution plans and + * result sets. It sits in a loop waiting for CalpontExecutionPlan + * from the FEP. It then passes the CalpontExecutionPlan to the + * JobListFactory from which a JobList is obtained. This is passed + * to to the Query Manager running in the real-time portion of the + * EC. The ExecutionPlanManager waits until the Query Manager + * returns a result set for the job list. These results are passed + * into the CalpontResultFactory, which outputs a CalpontResultSet. + * The ExecutionPlanManager passes the CalpontResultSet into the + * VendorResultFactory which produces a result set tailored to the + * specific DBMS front end in use. The ExecutionPlanManager then + * sends the VendorResultSet back to the Calpont Database Connector + * on the Front-End Processor where it is returned to the DBMS + * front-end. + */ +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" +#include "serviceexemgr.h" + +int main(int argc, char* argv[]) +{ + opterr = 0; + exemgr::Opt opt(argc, argv); + + // Set locale language + setlocale(LC_ALL, ""); + setlocale(LC_NUMERIC, "C"); + + // This is unset due to the way we start it + // program_invocation_short_name = const_cast("ExeMgr"); + + // Initialize the charset library + MY_INIT(argv[0]); + // global ptr defined in serviceexemgr.cpp + exemgr::globServiceExeMgr = new exemgr::ServiceExeMgr(opt); + return exemgr::globServiceExeMgr->Run(); +} + diff --git a/exemgr/main.cpp b/exemgr/main.cpp deleted file mode 100644 index 8b84443f1..000000000 --- a/exemgr/main.cpp +++ /dev/null @@ -1,1827 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2019 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/********************************************************************** - * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ - * - * - ***********************************************************************/ -/** - * @brief execution plan manager main program - * - * This is the ?main? program for dealing with execution plans and - * result sets. It sits in a loop waiting for CalpontExecutionPlan - * from the FEP. It then passes the CalpontExecutionPlan to the - * JobListFactory from which a JobList is obtained. This is passed - * to to the Query Manager running in the real-time portion of the - * EC. The ExecutionPlanManager waits until the Query Manager - * returns a result set for the job list. These results are passed - * into the CalpontResultFactory, which outputs a CalpontResultSet. - * The ExecutionPlanManager passes the CalpontResultSet into the - * VendorResultFactory which produces a result set tailored to the - * specific DBMS front end in use. The ExecutionPlanManager then - * sends the VendorResultSet back to the Calpont Database Connector - * on the Front-End Processor where it is returned to the DBMS - * front-end. - */ -#include -#include -#include -#include - -#undef root_name -#include - -#include "calpontselectexecutionplan.h" -#include "mcsanalyzetableexecutionplan.h" -#include "activestatementcounter.h" -#include "distributedenginecomm.h" -#include "resourcemanager.h" -#include "configcpp.h" -#include "queryteleserverparms.h" -#include "iosocket.h" -#include "joblist.h" -#include "joblistfactory.h" -#include "oamcache.h" -#include "simplecolumn.h" -#include "bytestream.h" -#include "telestats.h" -#include "messageobj.h" -#include "messagelog.h" -#include "sqllogger.h" -#include "femsghandler.h" -#include "idberrorinfo.h" -#include "MonitorProcMem.h" -#include "liboamcpp.h" -#include "crashtrace.h" -#include "service.h" - -#include -#include -#include - -#include "dbrm.h" - -#include "mariadb_my_sys.h" -#include "statistics.h" - -class Opt -{ - public: - int m_debug; - bool m_e; - bool m_fg; - Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) - { - int c; - while ((c = getopt(argc, argv, "edf")) != EOF) - { - switch (c) - { - case 'd': m_debug++; break; - - case 'e': m_e = true; break; - - case 'f': m_fg = true; break; - - case '?': - default: break; - } - } - } -}; - -class ServiceExeMgr : public Service, public Opt -{ - protected: - void log(logging::LOG_TYPE type, const std::string& str) - { - logging::LoggingID logid(16); - logging::Message::Args args; - logging::Message message(8); - args.add(strerror(errno)); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(type, message, logid); - } - - public: - ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt) - { - } - void LogErrno() override - { - log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); - } - void ParentLogChildMessage(const std::string& str) override - { - log(logging::LOG_TYPE_INFO, str); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } -}; - -namespace -{ -// If any flags other than the table mode flags are set, produce output to screeen -const uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & - ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); - -int gDebug; - -const unsigned logDefaultMsg = logging::M0000; -const unsigned logDbProfStartStatement = logging::M0028; -const unsigned logDbProfEndStatement = logging::M0029; -const unsigned logStartSql = logging::M0041; -const unsigned logEndSql = logging::M0042; -const unsigned logRssTooBig = logging::M0044; -const unsigned logDbProfQueryStats = logging::M0047; -const unsigned logExeMgrExcpt = logging::M0055; - -logging::Logger msgLog(16); - -typedef std::map SessionMemMap_t; -SessionMemMap_t sessionMemMap; // track memory% usage during a query -std::mutex sessionMemMapMutex; - -//...The FrontEnd may establish more than 1 connection (which results in -// more than 1 ExeMgr thread) per session. These threads will share -// the same CalpontSystemCatalog object for that session. Here, we -// define a std::map to track how many threads are sharing each session, so -// that we know when we can safely delete a CalpontSystemCatalog object -// shared by multiple threads per session. -typedef std::map ThreadCntPerSessionMap_t; -ThreadCntPerSessionMap_t threadCntPerSessionMap; -std::mutex threadCntPerSessionMapMutex; - -// This var is only accessed using thread-safe inc/dec calls -ActiveStatementCounter* statementsRunningCount; - -joblist::DistributedEngineComm* ec; - -auto rm = joblist::ResourceManager::instance(true); - -int toInt(const std::string& val) -{ - if (val.length() == 0) - return -1; - - return static_cast(config::Config::fromText(val)); -} - -const std::string ExeMgr("ExeMgr1"); - -const std::string prettyPrintMiniInfo(const std::string& in) -{ - // 1. take the std::string and tok it by '\n' - // 2. for each part in each line calc the longest part - // 3. padding to each longest value, output a header and the lines - typedef boost::tokenizer > my_tokenizer; - boost::char_separator sep1("\n"); - my_tokenizer tok1(in, sep1); - std::vector lines; - std::string header = "Desc Mode Table TableOID ReferencedColumns PIO LIO PBE Elapsed Rows"; - const int header_parts = 10; - lines.push_back(header); - - for (my_tokenizer::const_iterator iter1 = tok1.begin(); iter1 != tok1.end(); ++iter1) - { - if (!iter1->empty()) - lines.push_back(*iter1); - } - - std::vector lens; - - for (int i = 0; i < header_parts; i++) - lens.push_back(0); - - std::vector > lineparts; - std::vector::iterator iter2; - int j; - - for (iter2 = lines.begin(), j = 0; iter2 != lines.end(); ++iter2, j++) - { - boost::char_separator sep2(" "); - my_tokenizer tok2(*iter2, sep2); - int i; - std::vector parts; - my_tokenizer::iterator iter3; - - for (iter3 = tok2.begin(), i = 0; iter3 != tok2.end(); ++iter3, i++) - { - if (i >= header_parts) - break; - - std::string part(*iter3); - - if (j != 0 && i == 8) - part.resize(part.size() - 3); - - assert(i < header_parts); - - if (part.size() > lens[i]) - lens[i] = part.size(); - - parts.push_back(part); - } - - assert(i == header_parts); - lineparts.push_back(parts); - } - - std::ostringstream oss; - - std::vector >::iterator iter1 = lineparts.begin(); - std::vector >::iterator end1 = lineparts.end(); - - oss << "\n"; - - while (iter1 != end1) - { - std::vector::iterator iter2 = iter1->begin(); - std::vector::iterator end2 = iter1->end(); - assert(distance(iter2, end2) == header_parts); - int i = 0; - - while (iter2 != end2) - { - assert(i < header_parts); - oss << std::setw(lens[i]) << std::left << *iter2 << " "; - ++iter2; - i++; - } - - oss << "\n"; - ++iter1; - } - - return oss.str(); -} - -const std::string timeNow() -{ - time_t outputTime = time(0); - struct tm ltm; - char buf[32]; // ctime(3) says at least 26 - size_t len = 0; -#ifdef _MSC_VER - asctime_s(buf, 32, localtime_r(&outputTime, <m)); -#else - asctime_r(localtime_r(&outputTime, <m), buf); -#endif - len = strlen(buf); - - if (len > 0) - --len; - - if (buf[len] == '\n') - buf[len] = 0; - - return buf; -} - -querytele::QueryTeleServerParms gTeleServerParms; - -class SessionThread -{ - public: - SessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, - joblist::ResourceManager* rm) - : fIos(ios) - , fEc(ec) - , fRm(rm) - , fStatsRetrieved(false) - , fTeleClient(gTeleServerParms) - , fOamCachePtr(oam::OamCache::makeOamCache()) - { - } - - private: - messageqcpp::IOSocket fIos; - joblist::DistributedEngineComm* fEc; - joblist::ResourceManager* fRm; - querystats::QueryStats fStats; - - // Variables used to store return stats - bool fStatsRetrieved; - - querytele::QueryTeleClient fTeleClient; - - oam::OamCache* fOamCachePtr; // this ptr is copyable... - - //...Reinitialize stats for start of a new query - void initStats(uint32_t sessionId, std::string& sqlText) - { - initMaxMemPct(sessionId); - - fStats.reset(); - fStats.setStartTime(); - fStats.fSessionID = sessionId; - fStats.fQuery = sqlText; - fStatsRetrieved = false; - } - - //...Get % memory usage during latest query for sesssionId. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static uint64_t getMaxMemPct(uint32_t sessionId) - { - uint64_t maxMemoryPct = 0; - - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); - - if (mapIter != sessionMemMap.end()) - { - maxMemoryPct = (uint64_t)mapIter->second; - } - } - - return maxMemoryPct; - } - - //...Delete sessionMemMap entry for the specified session's memory % use. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void deleteMaxMemPct(uint32_t sessionId) - { - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); - - if (mapIter != sessionMemMap.end()) - { - sessionMemMap.erase(sessionId); - } - } - } - - //...Get and log query stats to specified output stream - const std::string formatQueryStats( - joblist::SJLP& jl, // joblist associated with query - const std::string& label, // header label to print in front of log output - bool includeNewLine, // include line breaks in query stats std::string - bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned) - { - std::ostringstream os; - - // Get stats if not already acquired for current query - if (!fStatsRetrieved) - { - if (wantExtendedStats) - { - // wait for the ei data to be written by another thread (brain-dead) - struct timespec req = {0, 250000}; // 250 usec -#ifdef _MSC_VER - Sleep(20); // 20ms on Windows -#else - nanosleep(&req, 0); -#endif - } - - // Get % memory usage during current query for sessionId - jl->querySummary(wantExtendedStats); - fStats = jl->queryStats(); - fStats.fMaxMemPct = getMaxMemPct(fStats.fSessionID); - fStats.fRows = rowsReturned; - fStatsRetrieved = true; - } - - std::string queryMode; - queryMode = (vtableModeOn ? "Distributed" : "Standard"); - - // Log stats to specified output stream - os << label << ": MaxMemPct-" << fStats.fMaxMemPct << "; NumTempFiles-" << fStats.fNumFiles - << "; TempFileSpace-" << roundBytes(fStats.fFileBytes) << "; ApproxPhyI/O-" << fStats.fPhyIO - << "; CacheI/O-" << fStats.fCacheIO << "; BlocksTouched-" << fStats.fMsgRcvCnt; - - if (includeNewLine) - os << std::endl << " "; // insert line break - else - os << "; "; // continue without line break - - os << "PartitionBlocksEliminated-" << fStats.fCPBlocksSkipped << "; MsgBytesIn-" - << roundBytes(fStats.fMsgBytesIn) << "; MsgBytesOut-" << roundBytes(fStats.fMsgBytesOut) << "; Mode-" - << queryMode; - - return os.str(); - } - - //...Increment the number of threads using the specified sessionId - static void incThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex); - ThreadCntPerSessionMap_t::iterator mapIter = threadCntPerSessionMap.find(sessionId); - - if (mapIter == threadCntPerSessionMap.end()) - threadCntPerSessionMap.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); - else - mapIter->second++; - } - - //...Decrement the number of threads using the specified sessionId. - //...When the thread count for a sessionId reaches 0, the corresponding - //...CalpontSystemCatalog objects are deleted. - //...The user query and its associated catalog query have a different - //...session Id where the highest bit is flipped. - //...The object with id(sessionId | 0x80000000) cannot be removed before - //...user query session completes because the sysdata may be used for - //...debugging/stats purpose, such as result graph, etc. - static void decThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex); - ThreadCntPerSessionMap_t::iterator mapIter = threadCntPerSessionMap.find(sessionId); - - if (mapIter != threadCntPerSessionMap.end()) - { - if (--mapIter->second == 0) - { - threadCntPerSessionMap.erase(mapIter); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); - } - } - } - - //...Init sessionMemMap entry for specified session to 0 memory %. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void initMaxMemPct(uint32_t sessionId) - { - if (sessionId < 0x80000000) - { - // std::cout << "Setting pct to 0 for session " << sessionId << std::endl; - std::lock_guard lk(sessionMemMapMutex); - SessionMemMap_t::iterator mapIter = sessionMemMap.find(sessionId); - - if (mapIter == sessionMemMap.end()) - { - sessionMemMap[sessionId] = 0; - } - else - { - mapIter->second = 0; - } - } - } - - //... Round off to human readable format (KB, MB, or GB). - const std::string roundBytes(uint64_t value) const - { - const char* units[] = {"B", "KB", "MB", "GB", "TB"}; - uint64_t i = 0, up = 0; - uint64_t roundedValue = value; - - while (roundedValue > 1024 && i < 4) - { - up = (roundedValue & 512); - roundedValue /= 1024; - i++; - } - - if (up) - roundedValue++; - - std::ostringstream oss; - oss << roundedValue << units[i]; - return oss.str(); - } - - //...Round off to nearest (1024*1024) MB - uint64_t roundMB(uint64_t value) const - { - uint64_t roundedValue = value >> 20; - - if (value & 524288) - roundedValue++; - - return roundedValue; - } - - void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms) - { - for (execplan::CalpontSelectExecutionPlan::RMParmVec::const_iterator it = parms.begin(); - it != parms.end(); ++it) - { - switch (it->id) - { - case execplan::PMSMALLSIDEMEMORY: - { - fRm->addHJPmMaxSmallSideMap(it->sessionId, it->value); - break; - } - - case execplan::UMSMALLSIDEMEMORY: - { - fRm->addHJUmMaxSmallSideMap(it->sessionId, it->value); - break; - } - - default:; - } - } - } - - void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, - boost::shared_ptr csc) - { - const execplan::CalpontSelectExecutionPlan::ColumnMap& colMap = csep.columnMap(); - execplan::CalpontSelectExecutionPlan::ColumnMap::const_iterator it; - std::string schemaName; - - for (it = colMap.begin(); it != colMap.end(); ++it) - { - const auto sc = dynamic_cast((it->second).get()); - - if (sc) - { - schemaName = sc->schemaName(); - - // only the first time a schema is got will actually query - // system catalog. System catalog keeps a schema name std::map. - // if a schema exists, the call getSchemaInfo returns without - // doing anything. - if (!schemaName.empty()) - csc->getSchemaInfo(schemaName); - } - } - - execplan::CalpontSelectExecutionPlan::SelectList::const_iterator subIt; - - for (subIt = csep.derivedTableList().begin(); subIt != csep.derivedTableList().end(); ++subIt) - { - buildSysCache(*(dynamic_cast(subIt->get())), csc); - } - } - - void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg) - { - messageqcpp::ByteStream emsgBs; - messageqcpp::ByteStream tbs; - tbs << code; - fIos.write(tbs); - emsgBs << emsg; - fIos.write(emsgBs); - } - - void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted) - { - messageqcpp::ByteStream::quadbyte qb; - execplan::MCSAnalyzeTableExecutionPlan caep; - - bs = fIos.read(); - caep.unserialize(bs); - - statementsRunningCount->incr(stmtCounted); - jl = joblist::JobListFactory::makeJobList(&caep, fRm, false, true); - - // Joblist is empty. - if (jl->status() == logging::statisticsJobListEmpty) - { - if (caep.traceOn()) - std::cout << "JobList is empty " << std::endl; - - jl.reset(); - bs.restart(); - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - statementsRunningCount->decr(stmtCounted); - return; - } - - if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) - { - std::cout << "fEc setup " << std::endl; - fEc->Setup(); - } - - if (jl->status() == 0) - { - std::string emsg; - - if (jl->putEngineComm(fEc) != 0) - throw std::runtime_error(jl->errMsg()); - } - else - { - throw std::runtime_error("ExeMgr: could not build a JobList!"); - } - - // Execute a joblist. - jl->doQuery(); - - FEMsgHandler msgHandler(jl, &fIos); - - msgHandler.start(); - auto rowCount = jl->projectTable(100, bs); - msgHandler.stop(); - - auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); - - if (caep.traceOn()) - std::cout << "Row count " << rowCount << std::endl; - - // Process `RowGroup`, increase an epoch and save statistics to the file. - auto* statisticsManager = statistics::StatisticsManager::instance(); - statisticsManager->analyzeColumnKeyTypes(outRG, caep.traceOn()); - statisticsManager->incEpoch(); - statisticsManager->saveToFile(); - - // Distribute statistics across all ExeMgr clients if possible. - statistics::StatisticsDistributor::instance()->distributeStatistics(); - - // Send the signal back to front-end. - bs.restart(); - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - statementsRunningCount->decr(stmtCounted); - } - - void analyzeTableHandleStats(messageqcpp::ByteStream& bs) - { - messageqcpp::ByteStream::quadbyte qb; -#ifdef DEBUG_STATISTICS - std::cout << "Get distributed statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - bs = fIos.read(); -#ifdef DEBUG_STATISTICS - std::cout << "Read the hash from statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - uint64_t dataHashRec; - bs >> dataHashRec; - - uint64_t dataHash = statistics::StatisticsManager::instance()->computeHashFromStats(); - // The stats are the same. - if (dataHash == dataHashRec) - { -#ifdef DEBUG_STATISTICS - std::cout << "The hash is the same as rec hash on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - return; - } - - bs.restart(); - qb = ANALYZE_TABLE_NEED_STATS; - bs << qb; - fIos.write(bs); - - bs.restart(); - bs = fIos.read(); -#ifdef DEBUG_STATISTICS - std::cout << "Read statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - statistics::StatisticsManager::instance()->unserialize(bs); - statistics::StatisticsManager::instance()->saveToFile(); - -#ifdef DEBUG_STATISTICS - std::cout << "Write flag on ExeMgr(Client) to ExeMgr(Server)" << std::endl; -#endif - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - } - - public: - void operator()() - { - messageqcpp::ByteStream bs, inbs; - execplan::CalpontSelectExecutionPlan csep; - csep.sessionID(0); - joblist::SJLP jl; - bool incSessionThreadCnt = true; - std::mutex jlMutex; - std::condition_variable jlCleanupDone; - int destructing = 0; - - bool selfJoin = false; - bool tryTuples = false; - bool usingTuples = false; - bool stmtCounted = false; - - try - { - for (;;) - { - selfJoin = false; - tryTuples = false; - usingTuples = false; - - if (jl) - { - // puts the real destruction in another thread to avoid - // making the whole session wait. It can take several seconds. - std::unique_lock scoped(jlMutex); - destructing++; - std::thread bgdtor( - [jl, &jlMutex, &jlCleanupDone, &destructing] - { - std::unique_lock scoped(jlMutex); - const_cast(jl).reset(); // this happens second; does real destruction - if (--destructing == 0) - jlCleanupDone.notify_one(); - }); - jl.reset(); // this runs first - bgdtor.detach(); - } - - bs = fIos.read(); - - if (bs.length() == 0) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### Got a close(1) for session id " << csep.sessionID() << std::endl; - - // connection closed by client - fIos.close(); - break; - } - else if (bs.length() < 4) // Not a CalpontSelectExecutionPlan - { - if (gDebug) - std::cout << "### Got a not-a-plan for session id " << csep.sessionID() << " with length " - << bs.length() << std::endl; - - fIos.close(); - break; - } - else if (bs.length() == 4) // possible tuple flag - { - messageqcpp::ByteStream::quadbyte qb; - bs >> qb; - - if (qb == 4) // UM wants new tuple i/f - { - if (gDebug) - std::cout << "### UM wants tuples" << std::endl; - - tryTuples = true; - // now wait for the CSEP... - bs = fIos.read(); - } - else if (qb == 5) // somebody wants stats - { - bs.restart(); - qb = statementsRunningCount->cur(); - bs << qb; - qb = statementsRunningCount->waiting(); - bs << qb; - fIos.write(bs); - fIos.close(); - break; - } - else if (qb == ANALYZE_TABLE_EXECUTE) - { - analyzeTableExecute(bs, jl, stmtCounted); - continue; - } - else if (qb == ANALYZE_TABLE_REC_STATS) - { - analyzeTableHandleStats(bs); - continue; - } - else - { - if (gDebug) - std::cout << "### Got a not-a-plan value " << qb << std::endl; - - fIos.close(); - break; - } - } - - new_plan: - try - { - csep.unserialize(bs); - } - catch (logging::IDBExcept& ex) - { - // We can get here on illegal function parameter data type, e.g. - // SELECT blob_column|1 FROM t1; - statementsRunningCount->decr(stmtCounted); - writeCodeAndError(ex.errorCode(), std::string(ex.what())); - continue; - } - - querytele::QueryTeleStats qts; - - if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) - { - qts.query_uuid = csep.uuid(); - qts.msg_type = querytele::QueryTeleStats::QT_START; - qts.start_time = querytele::QueryTeleClient::timeNowms(); - qts.query = csep.data(); - qts.session_id = csep.sessionID(); - qts.query_type = csep.queryType(); - qts.system_name = fOamCachePtr->getSystemName(); - qts.module_name = fOamCachePtr->getModuleName(); - qts.local_query = csep.localQuery(); - qts.schema_name = csep.schemaName(); - fTeleClient.postQueryTele(qts); - } - - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", got a CSEP" << std::endl; - - setRMParms(csep.rmParms()); - // Re-establish lost PP connections. - if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) - { - fEc->Setup(); - } - // @bug 1021. try to get schema cache for a come in query. - // skip system catalog queries. - if (!csep.isInternal()) - { - boost::shared_ptr csc = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(csep.sessionID()); - buildSysCache(csep, csc); - } - - // As soon as we have a session id for this thread, update the - // thread count per session; only do this once per thread. - // Mask 0x80000000 is for associate user query and csc query - if (incSessionThreadCnt) - { - incThreadCntPerSession(csep.sessionID() | 0x80000000); - incSessionThreadCnt = false; - } - - bool needDbProfEndStatementMsg = false; - logging::Message::Args args; - std::string sqlText = csep.data(); - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - - // Initialize stats for this query, including - // init sessionMemMap entry for this session to 0 memory %. - // We will need this later for traceOn() or if we receive a - // table request with qb=3 (see below). This is also recorded - // as query start time. - initStats(csep.sessionID(), sqlText); - fStats.fQueryType = csep.queryType(); - - // Log start and end statement if tracing is enabled. Keep in - // mind the trace flag won't be set for system catalog queries. - if (csep.traceOn()) - { - args.reset(); - args.add((int)csep.statementID()); - args.add((int)csep.verID().currentScn); - args.add(sqlText); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfStartStatement, args, li); - needDbProfEndStatementMsg = true; - } - - // Don't log subsequent self joins after first. - if (selfJoin) - sqlText = ""; - - std::ostringstream oss; - oss << sqlText << "; |" << csep.schemaName() << "|"; - logging::SQLLogger sqlLog(oss.str(), li); - - statementsRunningCount->incr(stmtCounted); - - if (tryTuples) - { - try // @bug2244: try/catch around fIos.write() calls responding to makeTupleList - { - jl = joblist::JobListFactory::makeJobList(&csep, fRm, true, true); - // assign query stats - jl->queryStats(fStats); - - if ((jl->status()) == 0 && (jl->putEngineComm(fEc) == 0)) - { - usingTuples = true; - - // Tell the FE that we're sending tuples back, not TableBands - writeCodeAndError(0, "NOERROR"); - auto tjlp = dynamic_cast(jl.get()); - assert(tjlp); - messageqcpp::ByteStream tbs; - tbs << tjlp->getOutputRowGroup(); - fIos.write(tbs); - } - else - { - const std::string emsg = jl->errMsg(); - statementsRunningCount->decr(stmtCounted); - writeCodeAndError(jl->status(), emsg); - std::cerr << "ExeMgr: could not build a tuple joblist: " << emsg << std::endl; - continue; - } - } - catch (std::exception& ex) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: error writing makeJoblist " - "response; " - << ex.what(); - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: unknown error writing makeJoblist " - "response; "; - throw std::runtime_error(errMsg.str()); - } - - if (!usingTuples) - { - if (gDebug) - std::cout << "### UM wanted tuples but it didn't work out :-(" << std::endl; - } - else - { - if (gDebug) - std::cout << "### UM wanted tuples and we'll do our best;-)" << std::endl; - } - } - else - { - usingTuples = false; - jl = joblist::JobListFactory::makeJobList(&csep, fRm, false, true); - - if (jl->status() == 0) - { - std::string emsg; - - if (jl->putEngineComm(fEc) != 0) - throw std::runtime_error(jl->errMsg()); - } - else - { - throw std::runtime_error("ExeMgr: could not build a JobList!"); - } - } - - jl->doQuery(); - - execplan::CalpontSystemCatalog::OID tableOID; - bool swallowRows = false; - joblist::DeliveredTableMap tm; - uint64_t totalBytesSent = 0; - uint64_t totalRowCount = 0; - - // Project each table as the FE asks for it - for (;;) - { - bs = fIos.read(); - - if (bs.length() == 0) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### Got a close(2) for session id " << csep.sessionID() << std::endl; - - break; - } - - if (gDebug && bs.length() > 4) - std::cout << "### For session id " << csep.sessionID() << ", got too many bytes = " << bs.length() - << std::endl; - - // TODO: Holy crud! Can this be right? - //@bug 1444 Yes, if there is a self-join - if (bs.length() > 4) - { - selfJoin = true; - statementsRunningCount->decr(stmtCounted); - goto new_plan; - } - - assert(bs.length() == 4); - - messageqcpp::ByteStream::quadbyte qb; - - try // @bug2244: try/catch around fIos.write() calls responding to qb command - { - bs >> qb; - - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", got a command = " << qb - << std::endl; - - if (qb == 0) - { - // No more tables, query is done - break; - } - else if (qb == 1) - { - // super-secret flag indicating that the UM is going to scarf down all the rows in the - // query. - swallowRows = true; - tm = jl->deliveredTables(); - continue; - } - else if (qb == 2) - { - // UM just wants any table - assert(swallowRows); - auto iter = tm.begin(); - - if (iter == tm.end()) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", returning end flag" - << std::endl; - - bs.restart(); - bs << (messageqcpp::ByteStream::byte)1; - fIos.write(bs); - continue; - } - - tableOID = iter->first; - } - else if (qb == 3) // special option-UM wants job stats std::string - { - std::string statsString; - - // Log stats std::string to be sent back to front end - statsString = formatQueryStats( - jl, "Query Stats", false, - !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), - (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), totalRowCount); - - bs.restart(); - bs << statsString; - - if ((csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG) != 0) - { - bs << jl->extendedInfo(); - bs << prettyPrintMiniInfo(jl->miniInfo()); - } - else - { - std::string empty; - bs << empty; - bs << empty; - } - - // send stats to connector for inserting to the querystats table - fStats.serialize(bs); - fIos.write(bs); - continue; - } - // for table mode handling - else if (qb == 4) - { - statementsRunningCount->decr(stmtCounted); - bs = fIos.read(); - goto new_plan; - } - else // (qb > 3) - { - // Return table bands for the requested tableOID - tableOID = static_cast(qb); - } - } - catch (std::exception& ex) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: error writing qb response " - "for qb cmd " - << qb << "; " << ex.what(); - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: unknown error writing qb response " - "for qb cmd " - << qb; - throw std::runtime_error(errMsg.str()); - } - - if (swallowRows) - tm.erase(tableOID); - - FEMsgHandler msgHandler(jl, &fIos); - - if (tableOID == 100) - msgHandler.start(); - - //...Loop serializing table bands projected for the tableOID - for (;;) - { - uint32_t rowCount; - - rowCount = jl->projectTable(tableOID, bs); - - msgHandler.stop(); - - if (jl->status()) - { - const auto errInfo = logging::IDBErrorInfo::instance(); - - if (jl->errMsg().length() != 0) - bs << jl->errMsg(); - else - bs << errInfo->errorMsg(jl->status()); - } - - try // @bug2244: try/catch around fIos.write() calls projecting rows - { - if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) - { - // Skip the write to the front end until the last empty band. Used to time queries - // through without any front end waiting. - if (tableOID < 3000 || rowCount == 0) - fIos.write(bs); - } - else - { - fIos.write(bs); - } - } - catch (std::exception& ex) - { - msgHandler.stop(); - std::ostringstream errMsg; - errMsg << "ExeMgr: error projecting rows " - "for tableOID: " - << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount << "; " - << ex.what(); - jl->abort(); - - while (rowCount) - rowCount = jl->projectTable(tableOID, bs); - - if (tableOID == 100 && msgHandler.aborted()) - { - /* TODO: modularize the cleanup code, as well as - * the rest of this fcn */ - - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - fIos.close(); - return; - } - - // std::cout << "connection drop\n"; - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - msgHandler.stop(); - errMsg << "ExeMgr: unknown error projecting rows " - "for tableOID: " - << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount; - jl->abort(); - - while (rowCount) - rowCount = jl->projectTable(tableOID, bs); - - throw std::runtime_error(errMsg.str()); - } - - totalRowCount += rowCount; - totalBytesSent += bs.length(); - - if (rowCount == 0) - { - msgHandler.stop(); - // No more bands, table is done - bs.reset(); - - // @bug 2083 decr active statement count here for table mode. - if (!usingTuples) - statementsRunningCount->decr(stmtCounted); - - break; - } - else - { - bs.restart(); - } - } // End of loop to project and serialize table bands for a table - } // End of loop to process tables - - // @bug 828 - if (csep.traceOn()) - jl->graph(csep.sessionID()); - - if (needDbProfEndStatementMsg) - { - std::string ss; - std::ostringstream prefix; - prefix << "ses:" << csep.sessionID() << " Query Totals"; - - // Log stats std::string to standard out - ss = formatQueryStats(jl, prefix.str(), true, - !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), - (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), - totalRowCount); - //@Bug 1306. Added timing info for real time tracking. - std::cout << ss << " at " << timeNow() << std::endl; - - // log query stats to debug log file - args.reset(); - args.add((int)csep.statementID()); - args.add(fStats.fMaxMemPct); - args.add(fStats.fNumFiles); - args.add(fStats.fFileBytes); // log raw byte count instead of MB - args.add(fStats.fPhyIO); - args.add(fStats.fCacheIO); - args.add(fStats.fMsgRcvCnt); - args.add(fStats.fMsgBytesIn); - args.add(fStats.fMsgBytesOut); - args.add(fStats.fCPBlocksSkipped); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfQueryStats, args, li); - //@bug 1327 - deleteMaxMemPct(csep.sessionID()); - // Calling reset here, will cause joblist destructor to be - // called, which "joins" the threads. We need to do that - // here to make sure all syslogging from all the threads - // are complete; and that our logDbProfEndStatement will - // appear "last" in the syslog for this SQL statement. - // puts the real destruction in another thread to avoid - // making the whole session wait. It can take several seconds. - int stmtID = csep.statementID(); - std::unique_lock scoped(jlMutex); - // C7's compiler complains about the msgLog capture here - // msgLog is global scope, and passed by copy, so, unclear - // what the warning is about. - destructing++; - std::thread bgdtor( - [jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing] - { - std::unique_lock scoped(jlMutex); - const_cast(jl).reset(); // this happens second; does real destruction - logging::Message::Args args; - args.add(stmtID); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, logDbProfEndStatement, args, li); - if (--destructing == 0) - jlCleanupDone.notify_one(); - }); - jl.reset(); // this happens first - bgdtor.detach(); - } - else - // delete sessionMemMap entry for this session's memory % use - deleteMaxMemPct(csep.sessionID()); - - std::string endtime(timeNow()); - - if ((csep.traceFlags() & flagsWantOutput) && (csep.sessionID() < 0x80000000)) - { - std::cout << "For session " << csep.sessionID() << ": " << totalBytesSent << " bytes sent back at " - << endtime << std::endl; - - // @bug 663 - Implemented caltraceon(16) to replace the - // $FIFO_SINK compiler definition in pColStep. - // This option consumes rows in the project steps. - if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS4) - { - std::cout << std::endl; - std::cout << "**** No data returned to DM. Rows consumed " - "in ProjectSteps - caltrace(16) is on (FIFO_SINK)." - " ****" - << std::endl; - std::cout << std::endl; - } - else if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) - { - std::cout << std::endl; - std::cout << "**** No data returned to DM - caltrace(8) is " - "on (SWALLOW_ROWS_EXEMGR). ****" - << std::endl; - std::cout << std::endl; - } - } - - statementsRunningCount->decr(stmtCounted); - - if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) - { - qts.msg_type = querytele::QueryTeleStats::QT_SUMMARY; - qts.max_mem_pct = fStats.fMaxMemPct; - qts.num_files = fStats.fNumFiles; - qts.phy_io = fStats.fPhyIO; - qts.cache_io = fStats.fCacheIO; - qts.msg_rcv_cnt = fStats.fMsgRcvCnt; - qts.cp_blocks_skipped = fStats.fCPBlocksSkipped; - qts.msg_bytes_in = fStats.fMsgBytesIn; - qts.msg_bytes_out = fStats.fMsgBytesOut; - qts.rows = totalRowCount; - qts.end_time = querytele::QueryTeleClient::timeNowms(); - qts.session_id = csep.sessionID(); - qts.query_type = csep.queryType(); - qts.query = csep.data(); - qts.system_name = fOamCachePtr->getSystemName(); - qts.module_name = fOamCachePtr->getModuleName(); - qts.local_query = csep.localQuery(); - fTeleClient.postQueryTele(qts); - } - } - - // Release CSC object (for sessionID) that was added by makeJobList() - // Mask 0x80000000 is for associate user query and csc query. - // (actual joblist destruction happens at the top of this loop) - decThreadCntPerSession(csep.sessionID() | 0x80000000); - } - catch (std::exception& ex) - { - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - std::cerr << "### ExeMgr ses:" << csep.sessionID() << " caught: " << ex.what() << std::endl; - logging::Message::Args args; - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - args.add(ex.what()); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); - fIos.close(); - } - catch (...) - { - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - std::cerr << "### Exception caught!" << std::endl; - logging::Message::Args args; - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - args.add("ExeMgr caught unknown exception"); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); - fIos.close(); - } - - // make sure we don't leave scope while joblists are being destroyed - std::unique_lock scoped(jlMutex); - while (destructing > 0) - jlCleanupDone.wait(scoped); - } -}; - -class RssMonFcn : public utils::MonitorProcMem -{ - public: - RssMonFcn(size_t maxPct, int pauseSeconds) : MonitorProcMem(maxPct, 0, 21, pauseSeconds) - { - } - - /*virtual*/ - void operator()() const - { - for (;;) - { - size_t rssMb = rss(); - size_t pct = rssMb * 100 / fMemTotal; - - if (pct > fMaxPct) - { - if (fMaxPct >= 95) - { - std::cerr << "Too much memory allocated!" << std::endl; - logging::Message::Args args; - args.add((int)pct); - args.add((int)fMaxPct); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logRssTooBig, args, logging::LoggingID(16)); - exit(1); - } - - if (statementsRunningCount->cur() == 0) - { - std::cerr << "Too much memory allocated!" << std::endl; - logging::Message::Args args; - args.add((int)pct); - args.add((int)fMaxPct); - msgLog.logMessage(logging::LOG_TYPE_WARNING, logRssTooBig, args, logging::LoggingID(16)); - exit(1); - } - - std::cerr << "Too much memory allocated, but stmts running" << std::endl; - } - - // Update sessionMemMap entries lower than current mem % use - { - std::lock_guard lk(sessionMemMapMutex); - - for (SessionMemMap_t::iterator mapIter = sessionMemMap.begin(); mapIter != sessionMemMap.end(); - ++mapIter) - { - if (pct > mapIter->second) - { - mapIter->second = pct; - } - } - } - - pause_(); - } - } -}; - -#ifdef _MSC_VER -void exit_(int) -{ - exit(0); -} -#endif - -void added_a_pm(int) -{ - logging::LoggingID logid(21, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("exeMgr caught SIGHUP. Resetting connections"); - msg.format(args1); - std::cout << msg.msg().c_str() << std::endl; - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); - - if (ec) - { - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - oamCache->forceReload(); - ec->Setup(); - } -} - -void printTotalUmMemory(int sig) -{ - int64_t num = rm->availableMemory(); - std::cout << "Total UM memory available: " << num << std::endl; -} - -void setupSignalHandlers() -{ -#ifdef _MSC_VER - signal(SIGINT, exit_); - signal(SIGTERM, exit_); -#else - struct sigaction ign; - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - - sigaction(SIGPIPE, &ign, 0); - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = added_a_pm; - sigaction(SIGHUP, &ign, 0); - ign.sa_handler = printTotalUmMemory; - sigaction(SIGUSR1, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); -#endif -} - -int8_t setupCwd(joblist::ResourceManager* rm) -{ - std::string workdir = rm->getScWorkingDir(); - int8_t rc = chdir(workdir.c_str()); - - if (rc < 0 || access(".", W_OK) != 0) - rc = chdir("/tmp"); - - return (rc < 0) ? -5 : rc; -} - -void startRssMon(size_t maxPct, int pauseSeconds) -{ - new boost::thread(RssMonFcn(maxPct, pauseSeconds)); -} - -int setupResources() -{ -#ifdef _MSC_VER - // FIXME: -#else - struct rlimit rlim; - - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -1; - } - - rlim.rlim_cur = rlim.rlim_max = 65536; - - if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -2; - } - - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -3; - } - - if (rlim.rlim_cur != 65536) - { - return -4; - } - -#endif - return 0; -} - -} // namespace - -void cleanTempDir() -{ - using TempDirPurpose = config::Config::TempDirPurpose; - struct Dirs - { - std::string section; - std::string allowed; - TempDirPurpose purpose; - }; - std::vector dirs{{"HashJoin", "AllowDiskBasedJoin", TempDirPurpose::Joins}, - {"RowAggregation", "AllowDiskBasedAggregation", TempDirPurpose::Aggregates}}; - const auto config = config::Config::makeConfig(); - - for (const auto& dir : dirs) - { - std::string allowStr = config->getConfig(dir.section, dir.allowed); - bool allow = (allowStr == "Y" || allowStr == "y"); - - std::string tmpPrefix = config->getTempFileDir(dir.purpose); - - if (allow && tmpPrefix.empty()) - { - std::cerr << "Empty tmp directory name for " << dir.section << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Empty tmp directory name for:"); - args.add(dir.section); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_CRITICAL, message, logid); - } - - tmpPrefix += "/"; - - idbassert(tmpPrefix != "/"); - - /* This is quite scary as ExeMgr usually runs as root */ - try - { - if (allow) - { - boost::filesystem::remove_all(tmpPrefix); - } - boost::filesystem::create_directories(tmpPrefix); - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Exception whilst cleaning tmpdir: "); - args.add(ex.what()); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); - } - catch (...) - { - std::cerr << "Caught unknown exception during tmpdir cleanup" << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Unknown exception whilst cleaning tmpdir"); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); - } - } -} - -int ServiceExeMgr::Child() -{ - gDebug = m_debug; - -#ifdef _MSC_VER - // FIXME: -#else - - // Make sure CSC thinks it's on a UM or else bucket reuse stuff below will stall - if (!m_e) - setenv("CALPONT_CSC_IDENT", "um", 1); - -#endif - setupSignalHandlers(); - int err = 0; - if (!m_debug) - err = setupResources(); - std::string errMsg; - - switch (err) - { - case -1: - case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; - - case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; - - case -4: - errMsg = "Could not install file limits to required value, please see non-root install documentation"; - break; - - default: errMsg = "Couldn't change working directory or unknown error"; break; - } - - err = setupCwd(rm); - - if (err < 0) - { - oam::Oam oam; - logging::Message::Args args; - logging::Message message; - args.add(errMsg); - message.format(args); - logging::LoggingID lid(16); - logging::MessageLog ml(lid); - ml.logCriticalMessage(message); - std::cerr << errMsg << std::endl; - - NotifyServiceInitializationFailed(); - return 2; - } - - cleanTempDir(); - - logging::MsgMap msgMap; - msgMap[logDefaultMsg] = logging::Message(logDefaultMsg); - msgMap[logDbProfStartStatement] = logging::Message(logDbProfStartStatement); - msgMap[logDbProfEndStatement] = logging::Message(logDbProfEndStatement); - msgMap[logStartSql] = logging::Message(logStartSql); - msgMap[logEndSql] = logging::Message(logEndSql); - msgMap[logRssTooBig] = logging::Message(logRssTooBig); - msgMap[logDbProfQueryStats] = logging::Message(logDbProfQueryStats); - msgMap[logExeMgrExcpt] = logging::Message(logExeMgrExcpt); - msgLog.msgMap(msgMap); - - ec = joblist::DistributedEngineComm::instance(rm, true); - ec->Open(); - - bool tellUser = true; - - messageqcpp::MessageQueueServer* mqs; - - statementsRunningCount = new ActiveStatementCounter(rm->getEmExecQueueSize()); - - for (;;) - { - try - { - mqs = new messageqcpp::MessageQueueServer(ExeMgr, rm->getConfig(), messageqcpp::ByteStream::BlockSize, - 64); - break; - } - catch (std::runtime_error& re) - { - std::string what = re.what(); - - if (what.find("Address already in use") != std::string::npos) - { - if (tellUser) - { - std::cerr << "Address already in use, retrying..." << std::endl; - tellUser = false; - } - - sleep(5); - } - else - { - throw; - } - } - } - - // class jobstepThreadPool is used by other processes. We can't call - // resourcemanaager (rm) functions during the static creation of threadpool - // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). - // From the pools perspective, it has no idea if it is ExeMgr doing the - // creation, so it has no idea which way to set the flag. So we set the max here. - joblist::JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); - joblist::JobStep::jobstepThreadPool.setName("ExeMgrJobList"); - - if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") - { - joblist::JobStep::jobstepThreadPool.setDebug(true); - joblist::JobStep::jobstepThreadPool.invoke( - threadpool::ThreadPoolMonitor(&joblist::JobStep::jobstepThreadPool)); - } - - int serverThreads = rm->getEmServerThreads(); - int maxPct = rm->getEmMaxPct(); - int pauseSeconds = rm->getEmSecondsBetweenMemChecks(); - int priority = rm->getEmPriority(); - - FEMsgHandler::threadPool.setMaxThreads(serverThreads); - FEMsgHandler::threadPool.setName("FEMsgHandler"); - - if (maxPct > 0) - startRssMon(maxPct, pauseSeconds); - -#ifndef _MSC_VER - setpriority(PRIO_PROCESS, 0, priority); -#endif - - std::string teleServerHost(rm->getConfig()->getConfig("QueryTele", "Host")); - - if (!teleServerHost.empty()) - { - int teleServerPort = toInt(rm->getConfig()->getConfig("QueryTele", "Port")); - - if (teleServerPort > 0) - { - gTeleServerParms.host = teleServerHost; - gTeleServerParms.port = teleServerPort; - } - } - - NotifyServiceStarted(); - - std::cout << "Starting ExeMgr: st = " << serverThreads << ", qs = " << rm->getEmExecQueueSize() - << ", mx = " << maxPct << ", cf = " << rm->getConfig()->configFile() << std::endl; - - { - BRM::DBRM* dbrm = new BRM::DBRM(); - dbrm->setSystemQueryReady(true); - delete dbrm; - } - - threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); - exeMgrThreadPool.setName("ExeMgrServer"); - - if (rm->getExeMgrThreadPoolDebug() == "Y" || rm->getExeMgrThreadPoolDebug() == "y") - { - exeMgrThreadPool.setDebug(true); - exeMgrThreadPool.invoke(threadpool::ThreadPoolMonitor(&exeMgrThreadPool)); - } - - // Load statistics. - try - { - statistics::StatisticsManager::instance()->loadFromFile(); - } - catch (...) - { - std::cerr << "Cannot load statistics from file " << std::endl; - } - - for (;;) - { - messageqcpp::IOSocket ios; - ios = mqs->accept(); - exeMgrThreadPool.invoke(SessionThread(ios, ec, rm)); - } - - exeMgrThreadPool.wait(); - - return 0; -} - -int main(int argc, char* argv[]) -{ - opterr = 0; - Opt opt(argc, argv); - - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); - - // This is unset due to the way we start it - program_invocation_short_name = const_cast("ExeMgr"); - - // Initialize the charset library - MY_INIT(argv[0]); - - return ServiceExeMgr(opt).Run(); -} - diff --git a/exemgr/resource.h b/exemgr/resource.h deleted file mode 100644 index 3e2717d59..000000000 --- a/exemgr/resource.h +++ /dev/null @@ -1,14 +0,0 @@ -//{{NO_DEPENDENCIES}} -// Microsoft Visual C++ generated include file. -// Used by ExeMgr.rc - -// Next default values for new objects -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif diff --git a/exemgr/rssmonfcn.cpp b/exemgr/rssmonfcn.cpp new file mode 100644 index 000000000..8a9b62d83 --- /dev/null +++ b/exemgr/rssmonfcn.cpp @@ -0,0 +1,69 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include + +#include "rssmonfcn.h" +#include "serviceexemgr.h" + +namespace exemgr +{ + void RssMonFcn::operator()() const + { + logging::Logger& msgLog = globServiceExeMgr->getLogger(); + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + for (;;) + { + size_t rssMb = rss(); + size_t pct = rssMb * 100 / fMemTotal; + + if (pct > fMaxPct) + { + if (fMaxPct >= 95) + { + std::cerr << "Too much memory allocated!" << std::endl; + logging::Message::Args args; + args.add((int)pct); + args.add((int)fMaxPct); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logRssTooBig, args, logging::LoggingID(16)); + exit(1); + } + + if (statementsRunningCount->cur() == 0) + { + std::cerr << "Too much memory allocated!" << std::endl; + logging::Message::Args args; + args.add((int)pct); + args.add((int)fMaxPct); + msgLog.logMessage(logging::LOG_TYPE_WARNING, ServiceExeMgr::logRssTooBig, args, logging::LoggingID(16)); + exit(1); + } + + std::cerr << "Too much memory allocated, but stmts running" << std::endl; + } + + // Update sessionMemMap entries lower than current mem % use + globServiceExeMgr->updateSessionMap(pct); + + pause_(); + } + } + void startRssMon(size_t maxPct, int pauseSeconds) + { + new std::thread(RssMonFcn(maxPct, pauseSeconds)); + } +} // namespace \ No newline at end of file diff --git a/exemgr/rssmonfcn.h b/exemgr/rssmonfcn.h new file mode 100644 index 000000000..341038e91 --- /dev/null +++ b/exemgr/rssmonfcn.h @@ -0,0 +1,36 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include + +#include "MonitorProcMem.h" + +namespace exemgr +{ +class RssMonFcn : public utils::MonitorProcMem +{ + public: + RssMonFcn(size_t maxPct, int pauseSeconds) : MonitorProcMem(maxPct, 0, 21, pauseSeconds) + { + } + + void operator()() const; +}; + +} // namespace \ No newline at end of file diff --git a/exemgr/serviceexemgr.cpp b/exemgr/serviceexemgr.cpp new file mode 100644 index 000000000..33ef3d8d2 --- /dev/null +++ b/exemgr/serviceexemgr.cpp @@ -0,0 +1,475 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2019-22 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/********************************************************************** + * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ + * + * + ***********************************************************************/ +/** + * @brief execution plan manager main program + * + * This is the ?main? program for dealing with execution plans and + * result sets. It sits in a loop waiting for CalpontExecutionPlan + * from the FEP. It then passes the CalpontExecutionPlan to the + * JobListFactory from which a JobList is obtained. This is passed + * to to the Query Manager running in the real-time portion of the + * EC. The ExecutionPlanManager waits until the Query Manager + * returns a result set for the job list. These results are passed + * into the CalpontResultFactory, which outputs a CalpontResultSet. + * The ExecutionPlanManager passes the CalpontResultSet into the + * VendorResultFactory which produces a result set tailored to the + * specific DBMS front end in use. The ExecutionPlanManager then + * sends the VendorResultSet back to the Calpont Database Connector + * on the Front-End Processor where it is returned to the DBMS + * front-end. + */ +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" +#include "serviceexemgr.h" +#include "sqlfrontsessionthread.h" + + +namespace exemgr +{ + ServiceExeMgr* globServiceExeMgr = nullptr; + void startRssMon(size_t maxPct, int pauseSeconds); + + void added_a_pm(int) + { + logging::LoggingID logid(21, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("exeMgr caught SIGHUP. Resetting connections"); + msg.format(args1); + std::cout << msg.msg().c_str() << std::endl; + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); + + auto* dec = exemgr::globServiceExeMgr->getDec(); + if (dec) + { + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + oamCache->forceReload(); + dec->Setup(); + } + } + + void printTotalUmMemory(int sig) + { + int64_t num = globServiceExeMgr->getRm().availableMemory(); + std::cout << "Total UM memory available: " << num << std::endl; + } + + void ServiceExeMgr::setupSignalHandlers() + { + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + + sigaction(SIGPIPE, &ign, 0); + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = exemgr::added_a_pm; + sigaction(SIGHUP, &ign, 0); + ign.sa_handler = exemgr::printTotalUmMemory; + sigaction(SIGUSR1, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + } + + void cleanTempDir() + { + using TempDirPurpose = config::Config::TempDirPurpose; + struct Dirs + { + std::string section; + std::string allowed; + TempDirPurpose purpose; + }; + std::vector dirs{{"HashJoin", "AllowDiskBasedJoin", TempDirPurpose::Joins}, + {"RowAggregation", "AllowDiskBasedAggregation", TempDirPurpose::Aggregates}}; + const auto config = config::Config::makeConfig(); + + for (const auto& dir : dirs) + { + std::string allowStr = config->getConfig(dir.section, dir.allowed); + bool allow = (allowStr == "Y" || allowStr == "y"); + + std::string tmpPrefix = config->getTempFileDir(dir.purpose); + + if (allow && tmpPrefix.empty()) + { + std::cerr << "Empty tmp directory name for " << dir.section << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Empty tmp directory name for:"); + args.add(dir.section); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_CRITICAL, message, logid); + } + + tmpPrefix += "/"; + + idbassert(tmpPrefix != "/"); + + /* This is quite scary as ExeMgr usually runs as root */ + try + { + if (allow) + { + boost::filesystem::remove_all(tmpPrefix); + } + boost::filesystem::create_directories(tmpPrefix); + } + catch (const std::exception& ex) + { + std::cerr << ex.what() << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Exception whilst cleaning tmpdir: "); + args.add(ex.what()); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); + } + catch (...) + { + std::cerr << "Caught unknown exception during tmpdir cleanup" << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Unknown exception whilst cleaning tmpdir"); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); + } + } + } + + const std::string ServiceExeMgr::prettyPrintMiniInfo(const std::string& in) + { + // 1. take the std::string and tok it by '\n' + // 2. for each part in each line calc the longest part + // 3. padding to each longest value, output a header and the lines + typedef boost::tokenizer > my_tokenizer; + boost::char_separator sep1("\n"); + my_tokenizer tok1(in, sep1); + std::vector lines; + std::string header = "Desc Mode Table TableOID ReferencedColumns PIO LIO PBE Elapsed Rows"; + const int header_parts = 10; + lines.push_back(header); + + for (my_tokenizer::const_iterator iter1 = tok1.begin(); iter1 != tok1.end(); ++iter1) + { + if (!iter1->empty()) + lines.push_back(*iter1); + } + + std::vector lens; + + for (int i = 0; i < header_parts; i++) + lens.push_back(0); + + std::vector > lineparts; + std::vector::iterator iter2; + int j; + + for (iter2 = lines.begin(), j = 0; iter2 != lines.end(); ++iter2, j++) + { + boost::char_separator sep2(" "); + my_tokenizer tok2(*iter2, sep2); + int i; + std::vector parts; + my_tokenizer::iterator iter3; + + for (iter3 = tok2.begin(), i = 0; iter3 != tok2.end(); ++iter3, i++) + { + if (i >= header_parts) + break; + + std::string part(*iter3); + + if (j != 0 && i == 8) + part.resize(part.size() - 3); + + assert(i < header_parts); + + if (part.size() > lens[i]) + lens[i] = part.size(); + + parts.push_back(part); + } + + assert(i == header_parts); + lineparts.push_back(parts); + } + + std::ostringstream oss; + + std::vector >::iterator iter1 = lineparts.begin(); + std::vector >::iterator end1 = lineparts.end(); + + oss << "\n"; + + while (iter1 != end1) + { + std::vector::iterator iter2 = iter1->begin(); + std::vector::iterator end2 = iter1->end(); + assert(distance(iter2, end2) == header_parts); + int i = 0; + + while (iter2 != end2) + { + assert(i < header_parts); + oss << std::setw(lens[i]) << std::left << *iter2 << " "; + ++iter2; + i++; + } + + oss << "\n"; + ++iter1; + } + + return oss.str(); + } + + int ServiceExeMgr::Child() + { + // Make sure CSC thinks it's on a UM or else bucket reuse stuff below will stall + if (!m_e) + setenv("CALPONT_CSC_IDENT", "um", 1); + + setupSignalHandlers(); + int err = 0; + if (!m_debug) + err = setupResources(); + std::string errMsg; + + switch (err) + { + case -1: + case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; + + case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; + + case -4: + errMsg = "Could not install file limits to required value, please see non-root install documentation"; + break; + + default: errMsg = "Couldn't change working directory or unknown error"; break; + } + + err = setupCwd(); + + if (err < 0) + { + oam::Oam oam; + logging::Message::Args args; + logging::Message message; + args.add(errMsg); + message.format(args); + logging::LoggingID lid(16); + logging::MessageLog ml(lid); + ml.logCriticalMessage(message); + std::cerr << errMsg << std::endl; + + NotifyServiceInitializationFailed(); + return 2; + } + + cleanTempDir(); + + logging::MsgMap msgMap; + msgMap[logDefaultMsg] = logging::Message(logDefaultMsg); + msgMap[logDbProfStartStatement] = logging::Message(logDbProfStartStatement); + msgMap[logDbProfEndStatement] = logging::Message(logDbProfEndStatement); + msgMap[logStartSql] = logging::Message(logStartSql); + msgMap[logEndSql] = logging::Message(logEndSql); + msgMap[logRssTooBig] = logging::Message(logRssTooBig); + msgMap[logDbProfQueryStats] = logging::Message(logDbProfQueryStats); + msgMap[logExeMgrExcpt] = logging::Message(logExeMgrExcpt); + msgLog_.msgMap(msgMap); + + dec_ = joblist::DistributedEngineComm::instance(rm_, true); + dec_->Open(); + + bool tellUser = true; + + messageqcpp::MessageQueueServer* mqs; + + statementsRunningCount_ = new ActiveStatementCounter(rm_->getEmExecQueueSize()); + const std::string ExeMgr = "ExeMgr1"; + for (;;) + { + try + { + mqs = new messageqcpp::MessageQueueServer(ExeMgr, rm_->getConfig(), messageqcpp::ByteStream::BlockSize, 64); + break; + } + catch (std::runtime_error& re) + { + std::string what = re.what(); + + if (what.find("Address already in use") != std::string::npos) + { + if (tellUser) + { + std::cerr << "Address already in use, retrying..." << std::endl; + tellUser = false; + } + + sleep(5); + } + else + { + throw; + } + } + } + + // class jobstepThreadPool is used by other processes. We can't call + // resourcemanaager (rm) functions during the static creation of threadpool + // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). + // From the pools perspective, it has no idea if it is ExeMgr doing the + // creation, so it has no idea which way to set the flag. So we set the max here. + joblist::JobStep::jobstepThreadPool.setMaxThreads(rm_->getJLThreadPoolSize()); + joblist::JobStep::jobstepThreadPool.setName("ExeMgrJobList"); + + if (rm_->getJlThreadPoolDebug() == "Y" || rm_->getJlThreadPoolDebug() == "y") + { + joblist::JobStep::jobstepThreadPool.setDebug(true); + joblist::JobStep::jobstepThreadPool.invoke( + threadpool::ThreadPoolMonitor(&joblist::JobStep::jobstepThreadPool)); + } + + int serverThreads = rm_->getEmServerThreads(); + int maxPct = rm_->getEmMaxPct(); + int pauseSeconds = rm_->getEmSecondsBetweenMemChecks(); + int priority = rm_->getEmPriority(); + + FEMsgHandler::threadPool.setMaxThreads(serverThreads); + FEMsgHandler::threadPool.setName("FEMsgHandler"); + + if (maxPct > 0) + { + // Defined in rssmonfcn.cpp + exemgr::startRssMon(maxPct, pauseSeconds); + } + + setpriority(PRIO_PROCESS, 0, priority); + + std::string teleServerHost(rm_->getConfig()->getConfig("QueryTele", "Host")); + + if (!teleServerHost.empty()) + { + int teleServerPort = toInt(rm_->getConfig()->getConfig("QueryTele", "Port")); + + if (teleServerPort > 0) + { + teleServerParms_ = querytele::QueryTeleServerParms(teleServerHost, teleServerPort); + } + } + + NotifyServiceStarted(); + + std::cout << "Starting ExeMgr: st = " << serverThreads << ", qs = " << rm_->getEmExecQueueSize() + << ", mx = " << maxPct << ", cf = " << rm_->getConfig()->configFile() << std::endl; + + { + BRM::DBRM* dbrm = new BRM::DBRM(); + dbrm->setSystemQueryReady(true); + delete dbrm; + } + + threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); + exeMgrThreadPool.setName("ExeMgrServer"); + + if (rm_->getExeMgrThreadPoolDebug() == "Y" || rm_->getExeMgrThreadPoolDebug() == "y") + { + exeMgrThreadPool.setDebug(true); + exeMgrThreadPool.invoke(threadpool::ThreadPoolMonitor(&exeMgrThreadPool)); + } + + // Load statistics. + try + { + statistics::StatisticsManager::instance()->loadFromFile(); + } + catch (...) + { + std::cerr << "Cannot load statistics from file " << std::endl; + } + + for (;;) + { + messageqcpp::IOSocket ios; + ios = mqs->accept(); + exeMgrThreadPool.invoke(exemgr::SQLFrontSessionThread(ios, dec_, rm_)); + } + + exeMgrThreadPool.wait(); + + return 0; + } +} // namespace diff --git a/exemgr/serviceexemgr.h b/exemgr/serviceexemgr.h new file mode 100644 index 000000000..544978ee7 --- /dev/null +++ b/exemgr/serviceexemgr.h @@ -0,0 +1,342 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" + +namespace exemgr +{ + class Opt + { + public: + int m_debug; + bool m_e; + bool m_fg; + Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) + { + int c; + while ((c = getopt(argc, argv, "edf")) != EOF) + { + switch (c) + { + case 'd': m_debug++; break; + + case 'e': m_e = true; break; + + case 'f': m_fg = true; break; + + case '?': + default: break; + } + } + } + int getDebugLevel() const + { + return m_debug; + } + }; + + class ServiceExeMgr : public Service, public Opt + { + using SessionMemMap_t = std::map; + using ThreadCntPerSessionMap_t = std::map; + protected: + void log(logging::LOG_TYPE type, const std::string& str) + { + logging::LoggingID logid(16); + logging::Message::Args args; + logging::Message message(8); + args.add(strerror(errno)); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(type, message, logid); + } + + public: + ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) + { + bool runningWithExeMgr = true; + rm_ = joblist::ResourceManager::instance(runningWithExeMgr); + } + void LogErrno() override + { + log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); + } + void ParentLogChildMessage(const std::string& str) override + { + log(logging::LOG_TYPE_INFO, str); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } + static const constexpr unsigned logDefaultMsg = logging::M0000; + static const constexpr unsigned logDbProfStartStatement = logging::M0028; + static const constexpr unsigned logDbProfEndStatement = logging::M0029; + static const constexpr unsigned logStartSql = logging::M0041; + static const constexpr unsigned logEndSql = logging::M0042; + static const constexpr unsigned logRssTooBig = logging::M0044; + static const constexpr unsigned logDbProfQueryStats = logging::M0047; + static const constexpr unsigned logExeMgrExcpt = logging::M0055; + // If any flags other than the table mode flags are set, produce output to screeen + static const constexpr uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & + ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); + logging::Logger& getLogger() + { + return msgLog_; + } + void updateSessionMap(const size_t pct) + { + std::lock_guard lk(sessionMemMapMutex_); + + for (auto mapIter = sessionMemMap_.begin(); mapIter != sessionMemMap_.end(); ++mapIter) + { + if (pct > mapIter->second) + { + mapIter->second = pct; + } + } + } + ThreadCntPerSessionMap_t& getThreadCntPerSessionMap() + { + return threadCntPerSessionMap_; + } + std::mutex& getThreadCntPerSessionMapMutex() + { + return threadCntPerSessionMapMutex_; + } + void initMaxMemPct(uint32_t sessionId) + { + // WIP + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter == sessionMemMap_.end()) + { + sessionMemMap_[sessionId] = 0; + } + else + { + mapIter->second = 0; + } + } + } + uint64_t getMaxMemPct(const uint32_t sessionId) + { + uint64_t maxMemoryPct = 0; + // WIP + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter != sessionMemMap_.end()) + { + maxMemoryPct = (uint64_t)mapIter->second; + } + } + + return maxMemoryPct; + } + void deleteMaxMemPct(uint32_t sessionId) + { + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter != sessionMemMap_.end()) + { + sessionMemMap_.erase(sessionId); + } + } + } + //...Increment the number of threads using the specified sessionId + void incThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex_); + auto mapIter = threadCntPerSessionMap_.find(sessionId); + + if (mapIter == threadCntPerSessionMap_.end()) + threadCntPerSessionMap_.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); + else + mapIter->second++; + } + //...Decrement the number of threads using the specified sessionId. + //...When the thread count for a sessionId reaches 0, the corresponding + //...CalpontSystemCatalog objects are deleted. + //...The user query and its associated catalog query have a different + //...session Id where the highest bit is flipped. + //...The object with id(sessionId | 0x80000000) cannot be removed before + //...user query session completes because the sysdata may be used for + //...debugging/stats purpose, such as result graph, etc. + void decThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex_); + auto mapIter = threadCntPerSessionMap_.find(sessionId); + + if (mapIter != threadCntPerSessionMap_.end()) + { + if (--mapIter->second == 0) + { + threadCntPerSessionMap_.erase(mapIter); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); + } + } + } + ActiveStatementCounter* getStatementsRunningCount() + { + return statementsRunningCount_; + } + joblist::DistributedEngineComm* getDec() + { + return dec_; + } + int toInt(const std::string& val) + { + if (val.length() == 0) + return -1; + + return static_cast(config::Config::fromText(val)); + } + const std::string prettyPrintMiniInfo(const std::string& in); + + const std::string timeNow() + { + time_t outputTime = time(0); + struct tm ltm; + char buf[32]; // ctime(3) says at least 26 + size_t len = 0; + asctime_r(localtime_r(&outputTime, <m), buf); + len = strlen(buf); + + if (len > 0) + --len; + + if (buf[len] == '\n') + buf[len] = 0; + + return buf; + } + querytele::QueryTeleServerParms& getTeleServerParms() + { + return teleServerParms_; + } + joblist::ResourceManager& getRm() + { + return *rm_; + } + private: + void setupSignalHandlers(); + int8_t setupCwd() + { + std::string workdir = rm_->getScWorkingDir(); + int8_t rc = chdir(workdir.c_str()); + + if (rc < 0 || access(".", W_OK) != 0) + rc = chdir("/tmp"); + + return (rc < 0) ? -5 : rc; + } + int setupResources() + { + struct rlimit rlim; + + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -1; + } + rlim.rlim_cur = rlim.rlim_max = 65536; + + if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -2; + } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -3; + } + if (rlim.rlim_cur != 65536) + { + return -4; + } + return 0; + } + + logging::Logger msgLog_; + SessionMemMap_t sessionMemMap_; // track memory% usage during a query + std::mutex sessionMemMapMutex_; + //...The FrontEnd may establish more than 1 connection (which results in + // more than 1 ExeMgr thread) per session. These threads will share + // the same CalpontSystemCatalog object for that session. Here, we + // define a std::map to track how many threads are sharing each session, so + // that we know when we can safely delete a CalpontSystemCatalog object + // shared by multiple threads per session. + ThreadCntPerSessionMap_t threadCntPerSessionMap_; + std::mutex threadCntPerSessionMapMutex_; + ActiveStatementCounter* statementsRunningCount_; + joblist::DistributedEngineComm* dec_; + joblist::ResourceManager* rm_; + // Its attributes are set in Child() + querytele::QueryTeleServerParms teleServerParms_; + }; + extern ServiceExeMgr* globServiceExeMgr; +} \ No newline at end of file diff --git a/exemgr/sqlfrontsessionthread.cpp b/exemgr/sqlfrontsessionthread.cpp new file mode 100644 index 000000000..3b5094e7a --- /dev/null +++ b/exemgr/sqlfrontsessionthread.cpp @@ -0,0 +1,985 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "sqlfrontsessionthread.h" + +namespace exemgr +{ + uint64_t SQLFrontSessionThread::getMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->getMaxMemPct(sessionId); + } + void SQLFrontSessionThread::deleteMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->deleteMaxMemPct(sessionId); + } + void SQLFrontSessionThread::incThreadCntPerSession(uint32_t sessionId) + { + return globServiceExeMgr->incThreadCntPerSession(sessionId); + } + void SQLFrontSessionThread::decThreadCntPerSession(uint32_t sessionId) + { + return globServiceExeMgr->decThreadCntPerSession(sessionId); + } + + void SQLFrontSessionThread::initMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->initMaxMemPct(sessionId); + } + //...Get and log query stats to specified output stream + const std::string SQLFrontSessionThread::formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned) + { + std::ostringstream os; + + // Get stats if not already acquired for current query + if (!fStatsRetrieved) + { + if (wantExtendedStats) + { + // wait for the ei data to be written by another thread (brain-dead) + struct timespec req = {0, 250000}; // 250 usec + nanosleep(&req, 0); + } + + // Get % memory usage during current query for sessionId + jl->querySummary(wantExtendedStats); + fStats = jl->queryStats(); + fStats.fMaxMemPct = getMaxMemPct(fStats.fSessionID); + fStats.fRows = rowsReturned; + fStatsRetrieved = true; + } + + std::string queryMode; + queryMode = (vtableModeOn ? "Distributed" : "Standard"); + + // Log stats to specified output stream + os << label << ": MaxMemPct-" << fStats.fMaxMemPct << "; NumTempFiles-" << fStats.fNumFiles + << "; TempFileSpace-" << roundBytes(fStats.fFileBytes) << "; ApproxPhyI/O-" << fStats.fPhyIO + << "; CacheI/O-" << fStats.fCacheIO << "; BlocksTouched-" << fStats.fMsgRcvCnt; + + if (includeNewLine) + os << std::endl << " "; // insert line break + else + os << "; "; // continue without line break + + os << "PartitionBlocksEliminated-" << fStats.fCPBlocksSkipped << "; MsgBytesIn-" + << roundBytes(fStats.fMsgBytesIn) << "; MsgBytesOut-" << roundBytes(fStats.fMsgBytesOut) << "; Mode-" + << queryMode; + + return os.str(); + } + + //... Round off to human readable format (KB, MB, or GB). + const std::string SQLFrontSessionThread::roundBytes(uint64_t value) const + { + const char* units[] = {"B", "KB", "MB", "GB", "TB"}; + uint64_t i = 0, up = 0; + uint64_t roundedValue = value; + + while (roundedValue > 1024 && i < 4) + { + up = (roundedValue & 512); + roundedValue /= 1024; + i++; + } + + if (up) + roundedValue++; + + std::ostringstream oss; + oss << roundedValue << units[i]; + return oss.str(); + } + + //...Round off to nearest (1024*1024) MB + uint64_t SQLFrontSessionThread::roundMB(uint64_t value) const + { + uint64_t roundedValue = value >> 20; + + if (value & 524288) + roundedValue++; + + return roundedValue; + } + + void SQLFrontSessionThread::setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms) + { + for (execplan::CalpontSelectExecutionPlan::RMParmVec::const_iterator it = parms.begin(); + it != parms.end(); ++it) + { + switch (it->id) + { + case execplan::PMSMALLSIDEMEMORY: + { + globServiceExeMgr->getRm().addHJPmMaxSmallSideMap(it->sessionId, it->value); + break; + } + + case execplan::UMSMALLSIDEMEMORY: + { + globServiceExeMgr->getRm().addHJUmMaxSmallSideMap(it->sessionId, it->value); + break; + } + + default:; + } + } + } + + void SQLFrontSessionThread::buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc) + { + const execplan::CalpontSelectExecutionPlan::ColumnMap& colMap = csep.columnMap(); + std::string schemaName; + + for (auto it = colMap.begin(); it != colMap.end(); ++it) + { + const auto sc = dynamic_cast((it->second).get()); + + if (sc) + { + schemaName = sc->schemaName(); + + // only the first time a schema is got will actually query + // system catalog. System catalog keeps a schema name std::map. + // if a schema exists, the call getSchemaInfo returns without + // doing anything. + if (!schemaName.empty()) + csc->getSchemaInfo(schemaName); + } + } + + execplan::CalpontSelectExecutionPlan::SelectList::const_iterator subIt; + + for (subIt = csep.derivedTableList().begin(); subIt != csep.derivedTableList().end(); ++subIt) + { + buildSysCache(*(dynamic_cast(subIt->get())), csc); + } + } + + void SQLFrontSessionThread::writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg) + { + messageqcpp::ByteStream emsgBs; + messageqcpp::ByteStream tbs; + tbs << code; + fIos.write(tbs); + emsgBs << emsg; + fIos.write(emsgBs); + } + + void SQLFrontSessionThread::analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted) + { + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + messageqcpp::ByteStream::quadbyte qb; + execplan::MCSAnalyzeTableExecutionPlan caep; + + bs = fIos.read(); + caep.unserialize(bs); + + statementsRunningCount->incr(stmtCounted); + jl = joblist::JobListFactory::makeJobList(&caep, fRm, false, true); + + // Joblist is empty. + if (jl->status() == logging::statisticsJobListEmpty) + { + if (caep.traceOn()) + std::cout << "JobList is empty " << std::endl; + + jl.reset(); + bs.restart(); + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + statementsRunningCount->decr(stmtCounted); + return; + } + + if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) + { + std::cout << "fEc setup " << std::endl; + fEc->Setup(); + } + + if (jl->status() == 0) + { + std::string emsg; + + if (jl->putEngineComm(fEc) != 0) + throw std::runtime_error(jl->errMsg()); + } + else + { + throw std::runtime_error("ExeMgr: could not build a JobList!"); + } + + // Execute a joblist. + jl->doQuery(); + + FEMsgHandler msgHandler(jl, &fIos); + + msgHandler.start(); + auto rowCount = jl->projectTable(100, bs); + msgHandler.stop(); + + auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); + + if (caep.traceOn()) + std::cout << "Row count " << rowCount << std::endl; + + // Process `RowGroup`, increase an epoch and save statistics to the file. + auto* statisticsManager = statistics::StatisticsManager::instance(); + statisticsManager->analyzeColumnKeyTypes(outRG, caep.traceOn()); + statisticsManager->incEpoch(); + statisticsManager->saveToFile(); + + // Distribute statistics across all ExeMgr clients if possible. + statistics::StatisticsDistributor::instance()->distributeStatistics(); + + // Send the signal back to front-end. + bs.restart(); + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + statementsRunningCount->decr(stmtCounted); + } + + void SQLFrontSessionThread::analyzeTableHandleStats(messageqcpp::ByteStream& bs) + { + messageqcpp::ByteStream::quadbyte qb; +#ifdef DEBUG_STATISTICS + std::cout << "Get distributed statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + bs = fIos.read(); +#ifdef DEBUG_STATISTICS + std::cout << "Read the hash from statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + uint64_t dataHashRec; + bs >> dataHashRec; + + uint64_t dataHash = statistics::StatisticsManager::instance()->computeHashFromStats(); + // The stats are the same. + if (dataHash == dataHashRec) + { +#ifdef DEBUG_STATISTICS + std::cout << "The hash is the same as rec hash on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + return; + } + + bs.restart(); + qb = ANALYZE_TABLE_NEED_STATS; + bs << qb; + fIos.write(bs); + + bs.restart(); + bs = fIos.read(); +#ifdef DEBUG_STATISTICS + std::cout << "Read statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + statistics::StatisticsManager::instance()->unserialize(bs); + statistics::StatisticsManager::instance()->saveToFile(); + +#ifdef DEBUG_STATISTICS + std::cout << "Write flag on ExeMgr(Client) to ExeMgr(Server)" << std::endl; +#endif + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + } + + void SQLFrontSessionThread::operator()() + { + messageqcpp::ByteStream bs, inbs; + execplan::CalpontSelectExecutionPlan csep; + csep.sessionID(0); + joblist::SJLP jl; + bool incSQLFrontSessionThreadCnt = true; + std::mutex jlMutex; + std::condition_variable jlCleanupDone; + int destructing = 0; + int gDebug = globServiceExeMgr->getDebugLevel(); + logging::Logger& msgLog = globServiceExeMgr->getLogger(); + + bool selfJoin = false; + bool tryTuples = false; + bool usingTuples = false; + bool stmtCounted = false; + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + + try + { + for (;;) + { + selfJoin = false; + tryTuples = false; + usingTuples = false; + + if (jl) + { + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + std::unique_lock scoped(jlMutex); + destructing++; + std::thread bgdtor( + [jl, &jlMutex, &jlCleanupDone, &destructing] + { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this runs first + bgdtor.detach(); + } + + bs = fIos.read(); + + if (bs.length() == 0) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### Got a close(1) for session id " << csep.sessionID() << std::endl; + + // connection closed by client + fIos.close(); + break; + } + else if (bs.length() < 4) // Not a CalpontSelectExecutionPlan + { + if (gDebug) + std::cout << "### Got a not-a-plan for session id " << csep.sessionID() << " with length " + << bs.length() << std::endl; + + fIos.close(); + break; + } + else if (bs.length() == 4) // possible tuple flag + { + messageqcpp::ByteStream::quadbyte qb; + bs >> qb; + + if (qb == 4) // UM wants new tuple i/f + { + if (gDebug) + std::cout << "### UM wants tuples" << std::endl; + + tryTuples = true; + // now wait for the CSEP... + bs = fIos.read(); + } + else if (qb == 5) // somebody wants stats + { + bs.restart(); + qb = statementsRunningCount->cur(); + bs << qb; + qb = statementsRunningCount->waiting(); + bs << qb; + fIos.write(bs); + fIos.close(); + break; + } + else if (qb == ANALYZE_TABLE_EXECUTE) + { + analyzeTableExecute(bs, jl, stmtCounted); + continue; + } + else if (qb == ANALYZE_TABLE_REC_STATS) + { + analyzeTableHandleStats(bs); + continue; + } + else + { + if (gDebug) + std::cout << "### Got a not-a-plan value " << qb << std::endl; + + fIos.close(); + break; + } + } + + new_plan: + try + { + csep.unserialize(bs); + } + catch (logging::IDBExcept& ex) + { + // We can get here on illegal function parameter data type, e.g. + // SELECT blob_column|1 FROM t1; + statementsRunningCount->decr(stmtCounted); + writeCodeAndError(ex.errorCode(), std::string(ex.what())); + continue; + } + + querytele::QueryTeleStats qts; + + if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) + { + qts.query_uuid = csep.uuid(); + qts.msg_type = querytele::QueryTeleStats::QT_START; + qts.start_time = querytele::QueryTeleClient::timeNowms(); + qts.query = csep.data(); + qts.session_id = csep.sessionID(); + qts.query_type = csep.queryType(); + qts.system_name = fOamCachePtr->getSystemName(); + qts.module_name = fOamCachePtr->getModuleName(); + qts.local_query = csep.localQuery(); + qts.schema_name = csep.schemaName(); + fTeleClient.postQueryTele(qts); + } + + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", got a CSEP" << std::endl; + + setRMParms(csep.rmParms()); + // Re-establish lost PP connections. + if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) + { + fEc->Setup(); + } + // @bug 1021. try to get schema cache for a come in query. + // skip system catalog queries. + if (!csep.isInternal()) + { + boost::shared_ptr csc = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(csep.sessionID()); + buildSysCache(csep, csc); + } + + // As soon as we have a session id for this thread, update the + // thread count per session; only do this once per thread. + // Mask 0x80000000 is for associate user query and csc query + if (incSQLFrontSessionThreadCnt) + { + // WIP + incThreadCntPerSession(csep.sessionID() | 0x80000000); + incSQLFrontSessionThreadCnt = false; + } + + bool needDbProfEndStatementMsg = false; + logging::Message::Args args; + std::string sqlText = csep.data(); + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + + // Initialize stats for this query, including + // init sessionMemMap entry for this session to 0 memory %. + // We will need this later for traceOn() or if we receive a + // table request with qb=3 (see below). This is also recorded + // as query start time. + initStats(csep.sessionID(), sqlText); + fStats.fQueryType = csep.queryType(); + + // Log start and end statement if tracing is enabled. Keep in + // mind the trace flag won't be set for system catalog queries. + if (csep.traceOn()) + { + args.reset(); + args.add((int)csep.statementID()); + args.add((int)csep.verID().currentScn); + args.add(sqlText); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfStartStatement, args, li); + needDbProfEndStatementMsg = true; + } + + // Don't log subsequent self joins after first. + if (selfJoin) + sqlText = ""; + + std::ostringstream oss; + oss << sqlText << "; |" << csep.schemaName() << "|"; + logging::SQLLogger sqlLog(oss.str(), li); + + statementsRunningCount->incr(stmtCounted); + + if (tryTuples) + { + try // @bug2244: try/catch around fIos.write() calls responding to makeTupleList + { + jl = joblist::JobListFactory::makeJobList(&csep, fRm, true, true); + // assign query stats + jl->queryStats(fStats); + + if ((jl->status()) == 0 && (jl->putEngineComm(fEc) == 0)) + { + usingTuples = true; + + // Tell the FE that we're sending tuples back, not TableBands + writeCodeAndError(0, "NOERROR"); + auto tjlp = dynamic_cast(jl.get()); + assert(tjlp); + messageqcpp::ByteStream tbs; + tbs << tjlp->getOutputRowGroup(); + fIos.write(tbs); + } + else + { + const std::string emsg = jl->errMsg(); + statementsRunningCount->decr(stmtCounted); + writeCodeAndError(jl->status(), emsg); + std::cerr << "ExeMgr: could not build a tuple joblist: " << emsg << std::endl; + continue; + } + } + catch (std::exception& ex) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: error writing makeJoblist " + "response; " + << ex.what(); + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: unknown error writing makeJoblist " + "response; "; + throw std::runtime_error(errMsg.str()); + } + + if (!usingTuples) + { + if (gDebug) + std::cout << "### UM wanted tuples but it didn't work out :-(" << std::endl; + } + else + { + if (gDebug) + std::cout << "### UM wanted tuples and we'll do our best;-)" << std::endl; + } + } + else + { + usingTuples = false; + jl = joblist::JobListFactory::makeJobList(&csep, fRm, false, true); + + if (jl->status() == 0) + { + std::string emsg; + + if (jl->putEngineComm(fEc) != 0) + throw std::runtime_error(jl->errMsg()); + } + else + { + throw std::runtime_error("ExeMgr: could not build a JobList!"); + } + } + + jl->doQuery(); + + execplan::CalpontSystemCatalog::OID tableOID; + bool swallowRows = false; + joblist::DeliveredTableMap tm; + uint64_t totalBytesSent = 0; + uint64_t totalRowCount = 0; + + // Project each table as the FE asks for it + for (;;) + { + bs = fIos.read(); + + if (bs.length() == 0) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### Got a close(2) for session id " << csep.sessionID() << std::endl; + + break; + } + + if (gDebug && bs.length() > 4) + std::cout << "### For session id " << csep.sessionID() << ", got too many bytes = " << bs.length() + << std::endl; + + // TODO: Holy crud! Can this be right? + //@bug 1444 Yes, if there is a self-join + if (bs.length() > 4) + { + selfJoin = true; + statementsRunningCount->decr(stmtCounted); + goto new_plan; + } + + assert(bs.length() == 4); + + messageqcpp::ByteStream::quadbyte qb; + + try // @bug2244: try/catch around fIos.write() calls responding to qb command + { + bs >> qb; + + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", got a command = " << qb + << std::endl; + + if (qb == 0) + { + // No more tables, query is done + break; + } + else if (qb == 1) + { + // super-secret flag indicating that the UM is going to scarf down all the rows in the + // query. + swallowRows = true; + tm = jl->deliveredTables(); + continue; + } + else if (qb == 2) + { + // UM just wants any table + assert(swallowRows); + auto iter = tm.begin(); + + if (iter == tm.end()) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", returning end flag" + << std::endl; + + bs.restart(); + bs << (messageqcpp::ByteStream::byte)1; + fIos.write(bs); + continue; + } + + tableOID = iter->first; + } + else if (qb == 3) // special option-UM wants job stats std::string + { + std::string statsString; + + // Log stats std::string to be sent back to front end + statsString = formatQueryStats( + jl, "Query Stats", false, + !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), + (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), totalRowCount); + + bs.restart(); + bs << statsString; + + if ((csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG) != 0) + { + bs << jl->extendedInfo(); + bs << globServiceExeMgr->prettyPrintMiniInfo(jl->miniInfo()); + } + else + { + std::string empty; + bs << empty; + bs << empty; + } + + // send stats to connector for inserting to the querystats table + fStats.serialize(bs); + fIos.write(bs); + continue; + } + // for table mode handling + else if (qb == 4) + { + statementsRunningCount->decr(stmtCounted); + bs = fIos.read(); + goto new_plan; + } + else // (qb > 3) + { + // Return table bands for the requested tableOID + tableOID = static_cast(qb); + } + } + catch (std::exception& ex) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: error writing qb response " + "for qb cmd " + << qb << "; " << ex.what(); + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: unknown error writing qb response " + "for qb cmd " + << qb; + throw std::runtime_error(errMsg.str()); + } + + if (swallowRows) + tm.erase(tableOID); + + FEMsgHandler msgHandler(jl, &fIos); + + if (tableOID == 100) + msgHandler.start(); + + //...Loop serializing table bands projected for the tableOID + for (;;) + { + uint32_t rowCount; + + rowCount = jl->projectTable(tableOID, bs); + + msgHandler.stop(); + + if (jl->status()) + { + const auto errInfo = logging::IDBErrorInfo::instance(); + + if (jl->errMsg().length() != 0) + bs << jl->errMsg(); + else + bs << errInfo->errorMsg(jl->status()); + } + + try // @bug2244: try/catch around fIos.write() calls projecting rows + { + if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) + { + // Skip the write to the front end until the last empty band. Used to time queries + // through without any front end waiting. + if (tableOID < 3000 || rowCount == 0) + fIos.write(bs); + } + else + { + fIos.write(bs); + } + } + catch (std::exception& ex) + { + msgHandler.stop(); + std::ostringstream errMsg; + errMsg << "ExeMgr: error projecting rows " + "for tableOID: " + << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount << "; " + << ex.what(); + jl->abort(); + + while (rowCount) + rowCount = jl->projectTable(tableOID, bs); + + if (tableOID == 100 && msgHandler.aborted()) + { + /* TODO: modularize the cleanup code, as well as + * the rest of this fcn */ + + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + fIos.close(); + return; + } + + // std::cout << "connection drop\n"; + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + msgHandler.stop(); + errMsg << "ExeMgr: unknown error projecting rows " + "for tableOID: " + << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount; + jl->abort(); + + while (rowCount) + rowCount = jl->projectTable(tableOID, bs); + + throw std::runtime_error(errMsg.str()); + } + + totalRowCount += rowCount; + totalBytesSent += bs.length(); + + if (rowCount == 0) + { + msgHandler.stop(); + // No more bands, table is done + bs.reset(); + + // @bug 2083 decr active statement count here for table mode. + if (!usingTuples) + statementsRunningCount->decr(stmtCounted); + + break; + } + else + { + bs.restart(); + } + } // End of loop to project and serialize table bands for a table + } // End of loop to process tables + + // @bug 828 + if (csep.traceOn()) + jl->graph(csep.sessionID()); + + if (needDbProfEndStatementMsg) + { + std::string ss; + std::ostringstream prefix; + prefix << "ses:" << csep.sessionID() << " Query Totals"; + + // Log stats std::string to standard out + ss = formatQueryStats(jl, prefix.str(), true, + !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), + (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), + totalRowCount); + //@Bug 1306. Added timing info for real time tracking. + std::cout << ss << " at " << globServiceExeMgr->timeNow() << std::endl; + + // log query stats to debug log file + args.reset(); + args.add((int)csep.statementID()); + args.add(fStats.fMaxMemPct); + args.add(fStats.fNumFiles); + args.add(fStats.fFileBytes); // log raw byte count instead of MB + args.add(fStats.fPhyIO); + args.add(fStats.fCacheIO); + args.add(fStats.fMsgRcvCnt); + args.add(fStats.fMsgBytesIn); + args.add(fStats.fMsgBytesOut); + args.add(fStats.fCPBlocksSkipped); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfQueryStats, args, li); + //@bug 1327 + deleteMaxMemPct(csep.sessionID()); + // Calling reset here, will cause joblist destructor to be + // called, which "joins" the threads. We need to do that + // here to make sure all syslogging from all the threads + // are complete; and that our logDbProfEndStatement will + // appear "last" in the syslog for this SQL statement. + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + int stmtID = csep.statementID(); + std::unique_lock scoped(jlMutex); + destructing++; + std::thread bgdtor( + [jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing, &msgLog] + { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + logging::Message::Args args; + args.add(stmtID); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfEndStatement, args, li); + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this happens first + bgdtor.detach(); + } + else + // delete sessionMemMap entry for this session's memory % use + deleteMaxMemPct(csep.sessionID()); + + std::string endtime(globServiceExeMgr->timeNow()); + + if ((csep.traceFlags() & globServiceExeMgr->flagsWantOutput) && (csep.sessionID() < 0x80000000)) + { + std::cout << "For session " << csep.sessionID() << ": " << totalBytesSent << " bytes sent back at " + << endtime << std::endl; + + // @bug 663 - Implemented caltraceon(16) to replace the + // $FIFO_SINK compiler definition in pColStep. + // This option consumes rows in the project steps. + if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS4) + { + std::cout << std::endl; + std::cout << "**** No data returned to DM. Rows consumed " + "in ProjectSteps - caltrace(16) is on (FIFO_SINK)." + " ****" + << std::endl; + std::cout << std::endl; + } + else if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) + { + std::cout << std::endl; + std::cout << "**** No data returned to DM - caltrace(8) is " + "on (SWALLOW_ROWS_EXEMGR). ****" + << std::endl; + std::cout << std::endl; + } + } + + statementsRunningCount->decr(stmtCounted); + + if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) + { + qts.msg_type = querytele::QueryTeleStats::QT_SUMMARY; + qts.max_mem_pct = fStats.fMaxMemPct; + qts.num_files = fStats.fNumFiles; + qts.phy_io = fStats.fPhyIO; + qts.cache_io = fStats.fCacheIO; + qts.msg_rcv_cnt = fStats.fMsgRcvCnt; + qts.cp_blocks_skipped = fStats.fCPBlocksSkipped; + qts.msg_bytes_in = fStats.fMsgBytesIn; + qts.msg_bytes_out = fStats.fMsgBytesOut; + qts.rows = totalRowCount; + qts.end_time = querytele::QueryTeleClient::timeNowms(); + qts.session_id = csep.sessionID(); + qts.query_type = csep.queryType(); + qts.query = csep.data(); + qts.system_name = fOamCachePtr->getSystemName(); + qts.module_name = fOamCachePtr->getModuleName(); + qts.local_query = csep.localQuery(); + fTeleClient.postQueryTele(qts); + } + } + + // Release CSC object (for sessionID) that was added by makeJobList() + // Mask 0x80000000 is for associate user query and csc query. + // (actual joblist destruction happens at the top of this loop) + decThreadCntPerSession(csep.sessionID() | 0x80000000); + } + catch (std::exception& ex) + { + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + std::cerr << "### ExeMgr ses:" << csep.sessionID() << " caught: " << ex.what() << std::endl; + logging::Message::Args args; + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + args.add(ex.what()); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logExeMgrExcpt, args, li); + fIos.close(); + } + catch (...) + { + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + std::cerr << "### Exception caught!" << std::endl; + logging::Message::Args args; + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + args.add("ExeMgr caught unknown exception"); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logExeMgrExcpt, args, li); + fIos.close(); + } + + // make sure we don't leave scope while joblists are being destroyed + std::unique_lock scoped(jlMutex); + while (destructing > 0) + jlCleanupDone.wait(scoped); +} +}; // namespace exemgr \ No newline at end of file diff --git a/exemgr/sqlfrontsessionthread.h b/exemgr/sqlfrontsessionthread.h new file mode 100644 index 000000000..ca605ea79 --- /dev/null +++ b/exemgr/sqlfrontsessionthread.h @@ -0,0 +1,131 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" +#include "serviceexemgr.h" + +namespace exemgr +{ + class SQLFrontSessionThread + { + public: + SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, + joblist::ResourceManager* rm) + : fIos(ios) + , fEc(ec) + , fRm(rm) + , fStatsRetrieved(false) + , fTeleClient(globServiceExeMgr->getTeleServerParms()) + , fOamCachePtr(oam::OamCache::makeOamCache()) + { + } + + private: + messageqcpp::IOSocket fIos; + joblist::DistributedEngineComm* fEc; + joblist::ResourceManager* fRm; + querystats::QueryStats fStats; + + // Variables used to store return stats + bool fStatsRetrieved; + + querytele::QueryTeleClient fTeleClient; + + oam::OamCache* fOamCachePtr; // this ptr is copyable... + + //...Reinitialize stats for start of a new query + void initStats(uint32_t sessionId, std::string& sqlText) + { + initMaxMemPct(sessionId); + + fStats.reset(); + fStats.setStartTime(); + fStats.fSessionID = sessionId; + fStats.fQuery = sqlText; + fStatsRetrieved = false; + } + //...Get % memory usage during latest query for sesssionId. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static uint64_t getMaxMemPct(uint32_t sessionId); + //...Delete sessionMemMap entry for the specified session's memory % use. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void deleteMaxMemPct(uint32_t sessionId); + //...Get and log query stats to specified output stream + const std::string formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); + static void incThreadCntPerSession(uint32_t sessionId); + static void decThreadCntPerSession(uint32_t sessionId); + //...Init sessionMemMap entry for specified session to 0 memory %. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void initMaxMemPct(uint32_t sessionId); + //... Round off to human readable format (KB, MB, or GB). + const std::string roundBytes(uint64_t value) const; + void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); + void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc); + void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); + void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); + void analyzeTableHandleStats(messageqcpp::ByteStream& bs); + uint64_t roundMB(uint64_t value) const; + public: + void operator()(); + }; +} \ No newline at end of file From c6c36eb622aabc9c1f05b0bfcac12a4f6a269345 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 1 Mar 2022 08:44:39 -0600 Subject: [PATCH 22/55] MCOL-5002 dev use largeRG when indexing by largeKeyColumns[] --- utils/joiner/tuplejoiner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index fc5a5052f..ac925e6ea 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -603,7 +603,7 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin if (UNLIKELY(inUM() && (joinType & MATCHNULLS) && !isNull && !typelessJoin)) { - if (smallRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) + if (largeRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) { uint bucket = bucketPicker((char*)&(joblist::LONGDOUBLENULL), sizeof(joblist::LONGDOUBLENULL), bpSeed) & bucketMask; @@ -612,7 +612,7 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin for (; range.first != range.second; ++range.first) matches->push_back(range.first->second); } - else if (!smallRG.usesStringTable()) + else if (!largeRG.usesStringTable()) { auto nullVal = getJoinNullValue(); uint bucket = bucketPicker((char*)&nullVal, sizeof(nullVal), bpSeed) & bucketMask; From 53b9a2a0f9e8b8eb30321c36f2265ecdd7b767e4 Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Fri, 4 Feb 2022 11:55:09 +0000 Subject: [PATCH 23/55] MCOL-4580 extent elimination for dictionary-based text/varchar types The idea is relatively simple - encode prefixes of collated strings as integers and use them to compute extents' ranges. Then we can eliminate extents with strings. The actual patch does have all the code there but miss one important step: we do not keep collation index, we keep charset index. Because of this, some of the tests in the bugfix suite fail and thus main functionality is turned off. The reason of this patch to be put into PR at all is that it contains changes that made CHAR/VARCHAR columns unsigned. This change is needed in vectorization work. --- datatypes/mcs_datatype.h | 7 +- datatypes/numericliteral.h | 1 + dbcon/execplan/aggregatecolumn.cpp | 2 + dbcon/execplan/calpontsystemcatalog.h | 30 +----- dbcon/execplan/constantcolumn.cpp | 14 +-- dbcon/execplan/predicateoperator.cpp | 32 ++++--- dbcon/execplan/simplecolumn.cpp | 10 ++ dbcon/execplan/treenode.h | 39 +++++--- dbcon/joblist/batchprimitiveprocessor-jl.cpp | 32 ++++++- dbcon/joblist/batchprimitiveprocessor-jl.h | 7 +- dbcon/joblist/columncommand-jl.cpp | 79 ++++++++++++++- dbcon/joblist/columncommand-jl.h | 5 +- dbcon/joblist/command-jl.h | 5 + dbcon/joblist/dictstep-jl.cpp | 61 +++++++++++- dbcon/joblist/dictstep-jl.h | 15 +++ dbcon/joblist/groupconcat.cpp | 10 +- dbcon/joblist/jlf_tuplejoblist.cpp | 5 +- dbcon/joblist/lbidlist.cpp | 37 +++++--- dbcon/joblist/lbidlist.h | 7 +- dbcon/joblist/primitivestep.h | 7 +- dbcon/joblist/tuple-bps.cpp | 45 +++++---- .../basic/r/ctype_extent_koi8u.result | 16 ++-- .../basic/r/ctype_extent_latin1.result | 24 ++--- .../columnstore/bugfixes/mcol-4940.result | 2 +- .../columnstore/bugfixes/mcol-4940.test | 3 +- primitives/linux-port/column.cpp | 6 +- primitives/linux-port/dictionary.cpp | 14 +++ primitives/linux-port/primitiveprocessor.h | 11 ++- .../primproc/batchprimitiveprocessor.cpp | 10 +- primitives/primproc/batchprimitiveprocessor.h | 2 + primitives/primproc/columncommand.cpp | 8 +- primitives/primproc/dictstep.cpp | 17 +++- primitives/primproc/dictstep.h | 1 + utils/common/CMakeLists.txt | 3 +- utils/common/collation.h | 6 ++ utils/common/nullvaluemanip.cpp | 1 + utils/common/string_prefixes.cpp | 51 ++++++++++ utils/common/string_prefixes.h | 32 +++++++ utils/funcexp/func_monthname.cpp | 16 +++- utils/funcexp/func_round.cpp | 34 ++++--- utils/funcexp/functor_str.h | 2 + utils/loggingcpp/exceptclasses.h | 1 - versioning/BRM/extentmap.cpp | 54 +++++++---- writeengine/bulk/we_bulkload.cpp | 2 +- writeengine/bulk/we_bulkloadbuffer.cpp | 2 +- writeengine/bulk/we_bulkloadbuffer.h | 2 +- writeengine/bulk/we_colextinf.cpp | 2 +- writeengine/server/we_ddlcommandproc.cpp | 12 ++- writeengine/server/we_dmlcommandproc.cpp | 11 ++- writeengine/server/we_server.cpp | 3 +- writeengine/shared/we_brm.h | 22 ++++- writeengine/shared/we_type.h | 2 + writeengine/splitter/we_brmupdater.cpp | 8 ++ writeengine/wrapper/writeengine.cpp | 95 ++++++++++++++----- 54 files changed, 698 insertions(+), 227 deletions(-) create mode 100644 utils/common/string_prefixes.cpp create mode 100644 utils/common/string_prefixes.h diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 9c58214bb..8e2c551cf 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -482,7 +482,11 @@ inline bool isUnsigned(const datatypes::SystemCatalog::ColDataType type) case datatypes::SystemCatalog::USMALLINT: case datatypes::SystemCatalog::UMEDINT: case datatypes::SystemCatalog::UINT: - case datatypes::SystemCatalog::UBIGINT: return true; + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: + case datatypes::SystemCatalog::TEXT: + case datatypes::SystemCatalog::VARBINARY: return true; default: return false; } @@ -2520,4 +2524,3 @@ class TypeHandlerTimestamp : public TypeHandlerTemporal }; } // end of namespace datatypes - diff --git a/datatypes/numericliteral.h b/datatypes/numericliteral.h index 0640f2a91..cdddad95d 100644 --- a/datatypes/numericliteral.h +++ b/datatypes/numericliteral.h @@ -336,6 +336,7 @@ class SignedInteger : public Parser::DD2OM { public: using DD2OM::DD2OM; + bool isNull() const { return UnsignedInteger::isNull(); diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index e94588955..49b01735e 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -424,6 +424,8 @@ void AggregateColumn::evaluate(Row& row, bool& isNull) else fResult.intVal = atoll((char*)&fResult.origIntVal); + fResult.uintVal = fResult.intVal; + break; case CalpontSystemCatalog::BIGINT: diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 3492b8d16..055d4846c 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -945,17 +945,18 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c break; } + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: { int colWidth = ct.colWidth; if (colWidth <= 8) { - if ((colWidth == 1) && ((int8_t)joblist::CHAR1NULL == val)) + if ((colWidth == 1) && ((uint8_t)joblist::CHAR1NULL == (uint8_t)val)) ret = true; - else if ((colWidth == 2) && ((int16_t)joblist::CHAR2NULL == val)) + else if ((colWidth == 2) && ((uint16_t)joblist::CHAR2NULL == (uint16_t)val)) ret = true; - else if ((colWidth < 5) && ((int32_t)joblist::CHAR4NULL == val)) + else if ((colWidth < 5) && ((uint32_t)joblist::CHAR4NULL == (uint32_t)val)) ret = true; else if ((int64_t)joblist::CHAR8NULL == val) ret = true; @@ -964,7 +965,6 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c { throw std::logic_error("Not a int column."); } - break; } @@ -1075,27 +1075,6 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c break; } - case execplan::CalpontSystemCatalog::VARCHAR: - { - int colWidth = ct.colWidth; - - if (colWidth <= 8) - { - if ((colWidth < 3) && ((int16_t)joblist::CHAR2NULL == val)) - ret = true; - else if ((colWidth < 5) && ((int32_t)joblist::CHAR4NULL == val)) - ret = true; - else if ((int64_t)joblist::CHAR8NULL == val) - ret = true; - } - else - { - throw std::logic_error("Not a int column."); - } - - break; - } - case execplan::CalpontSystemCatalog::UTINYINT: { if (joblist::UTINYINTNULL == (uint8_t)val) @@ -1278,4 +1257,3 @@ const std::string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt); bool ctListSort(const CalpontSystemCatalog::ColType& a, const CalpontSystemCatalog::ColType& b); } // namespace execplan - diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index bcdc67676..acfb883d9 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -48,18 +48,8 @@ ConstantColumn::ConstantColumn(const string& sql, TYPE type) { fResult.strVal = sql; - if (type == LITERAL && sql.length() < 9) - { - memcpy(tmp, sql.c_str(), sql.length()); - memset(tmp + sql.length(), 0, 8); - fResult.uintVal = uint64ToStr(*((uint64_t*)tmp)); - fResult.intVal = (int64_t)fResult.uintVal; - } - else - { - fResult.intVal = atoll(sql.c_str()); - fResult.uintVal = strtoull(sql.c_str(), NULL, 0); - } + fResult.intVal = atoll(sql.c_str()); + fResult.uintVal = strtoull(sql.c_str(), NULL, 0); fResult.floatVal = atof(sql.c_str()); fResult.doubleVal = atof(sql.c_str()); diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index 465f32af2..6ff4d311b 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -264,20 +264,6 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; } } - // If both sides are unsigned, use UBIGINT as result type, otherwise - // "promote" to BIGINT. - else if (isUnsigned(l.colDataType) && isUnsigned(r.colDataType)) - { - fOperationType.colDataType = execplan::CalpontSystemCatalog::UBIGINT; - fOperationType.colWidth = 8; - } - else if ((isSignedInteger(l.colDataType) && isUnsigned(r.colDataType)) || - (isUnsigned(l.colDataType) && isSignedInteger(r.colDataType)) || - (isSignedInteger(l.colDataType) && isSignedInteger(r.colDataType))) - { - fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; - fOperationType.colWidth = 8; - } else if ((l.colDataType == execplan::CalpontSystemCatalog::CHAR || l.colDataType == execplan::CalpontSystemCatalog::VARCHAR || l.colDataType == execplan::CalpontSystemCatalog::TEXT) && @@ -321,6 +307,20 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 255; } } + // If both sides are unsigned, use UBIGINT as result type, otherwise + // "promote" to BIGINT. + else if (isUnsigned(l.colDataType) && isInteger(l.colDataType) && isUnsigned(r.colDataType) && isInteger(r.colDataType)) + { + fOperationType.colDataType = execplan::CalpontSystemCatalog::UBIGINT; + fOperationType.colWidth = 8; + } + else if ((isSignedInteger(l.colDataType) && isUnsigned(r.colDataType) && isInteger(r.colDataType)) || + (isUnsigned(l.colDataType) && isInteger(l.colDataType) && isSignedInteger(r.colDataType)) || + (isSignedInteger(l.colDataType) && isSignedInteger(r.colDataType))) + { + fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; + fOperationType.colWidth = 8; + } else if (l.colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE || r.colDataType == execplan::CalpontSystemCatalog::LONGDOUBLE) { @@ -410,7 +410,9 @@ bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedCol if (isNull) return false; - return numericCompare(val1, rop->getIntVal(row, isNull)) && !isNull; + int64_t val2 = rop->getIntVal(row, isNull); + + return numericCompare(val1, val2) && !isNull; } case execplan::CalpontSystemCatalog::UBIGINT: diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 6ecb8b1cb..2e3226f5f 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -565,6 +565,16 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) else fResult.intVal = atoll((char*)&fResult.origIntVal); + // MCOL-4580 - related, probably can be marked with XXX. + // This does not fail in any tests, but it is considered wrong. + // The reasonin behind that is that we changed signedness if characters to unsigned + // and it might be a case with short strings that they were copied as is using + // uint64ToStr encoding into int64_t values. So, potentially, unsuspecting code + // may use getUintVal instead of getIntVal to process short char column, getting + // unitialized value and give floating behavior. + // None of our tests failed, though. + fResult.uintVal = fResult.intVal; + break; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 105765aed..afc0625c9 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -36,6 +36,7 @@ #include "columnwidth.h" #include "mcs_decimal.h" #include "mcs_int64.h" +#include "numericliteral.h" namespace messageqcpp { @@ -664,25 +665,19 @@ inline int64_t TreeNode::getIntVal() switch (fResultType.colDataType) { case CalpontSystemCatalog::CHAR: - if (fResultType.colWidth <= 8) - return fResult.intVal; - - return atoll(fResult.strVal.c_str()); - case CalpontSystemCatalog::VARCHAR: - if (fResultType.colWidth <= 7) - return fResult.intVal; - - return atoll(fResult.strVal.c_str()); - - // FIXME: ??? case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: - if (fResultType.colWidth <= 7) - return fResult.intVal; - - return atoll(fResult.strVal.c_str()); + { + datatypes::DataCondition cnverr; + literal::Converter cnv(fResult.strVal, cnverr); + if (datatypes::DataCondition::Code(cnverr) != 0) + { + cerr << "error in int conversion from '" << fResult.strVal << "'"; + } + return cnv.toSInt(cnverr); + } case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::TINYINT: @@ -721,6 +716,20 @@ inline uint64_t TreeNode::getUintVal() { switch (fResultType.colDataType) { + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + datatypes::DataCondition cnverr; + literal::Converter cnv(fResult.strVal, cnverr); + if (datatypes::DataCondition::Code(cnverr) != 0) + { + cerr << "error in unsigned int conversion from '" << fResult.strVal << "'"; + } + return cnv.toXIntPositive(cnverr); + } case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 702c4c091..4699042bc 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -50,6 +50,8 @@ using namespace messageqcpp; using namespace rowgroup; using namespace joiner; +//#define XXX_BATCHPRIMPROC_TOKENS_RANGES_XXX + namespace joblist { BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm) @@ -152,6 +154,21 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pDictionaryStep& step) cc->setBatchPrimitiveProcessor(this); cc->setQueryUuid(step.queryUuid()); cc->setStepUuid(uuid); + +#if defined(XXX_BATCHPRIMPROC_TOKENS_RANGES_XXX) + if (filterSteps.size() > 0) + { + size_t stepsIndex = filterSteps.size() - 1; + SCommand prevCC = filterSteps[stepsIndex]; + ColumnCommandJL* pcc = dynamic_cast(prevCC.get()); + DictStepJL* ccc = dynamic_cast(cc.get()); + if (pcc && ccc) + { + filterSteps[stepsIndex].reset( + new ColumnCommandJL(*pcc, *ccc)); // column command will use same filters. + } + } +#endif filterSteps.push_back(cc); filterCount++; needStrValues = true; @@ -443,6 +460,7 @@ void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, vector> *lbid; + in >> tmp64; *min = (int64_t)tmp64; in >> tmp64; @@ -712,8 +730,9 @@ bool BatchPrimitiveProcessorJL::countThisMsg(messageqcpp::ByteStream& in) const } if (data[offset] != 0) - offset += (data[offset + CP_FLAG_AND_LBID] * 2) + CP_FLAG_AND_LBID + - 1; // skip the CP data with wide min/max values (16/32 bytes each) + offset += (data[offset + CP_FLAG_AND_LBID + 1] * 2) + CP_FLAG_AND_LBID + 1 + + 1; // skip the CP data with wide min/max values (16/32 bytes each). we also skip + // cpFromDictScan flag. else offset += CP_FLAG_AND_LBID; // skip only the "valid CP data" & LBID bytes } @@ -750,9 +769,10 @@ void BatchPrimitiveProcessorJL::deserializeAggregateResult(ByteStream* in, vecto } void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* out, bool* validCPData, - uint64_t* lbid, int128_t* min, int128_t* max, - uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, - bool* countThis, uint32_t threadID, bool* hasWideColumn, + uint64_t* lbid, bool* fromDictScan, int128_t* min, + int128_t* max, uint32_t* cachedIO, uint32_t* physIO, + uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, + bool* hasWideColumn, const execplan::CalpontSystemCatalog::ColType& colType) const { uint64_t tmp64; @@ -789,6 +809,8 @@ void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* { in >> *lbid; in >> tmp8; + *fromDictScan = tmp8 != 0; + in >> tmp8; *hasWideColumn = (tmp8 > utils::MAXLEGACYWIDTH); if (UNLIKELY(*hasWideColumn)) { diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index 9cddd8945..a249b3102 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -167,9 +167,9 @@ class BatchPrimitiveProcessorJL uint32_t* touchedBlocks) const; void deserializeAggregateResults(messageqcpp::ByteStream* in, std::vector* out) const; void getRowGroupData(messageqcpp::ByteStream& in, std::vector* out, bool* validCPData, - uint64_t* lbid, int128_t* min, int128_t* max, uint32_t* cachedIO, uint32_t* physIO, - uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, bool* hasBinaryColumn, - const execplan::CalpontSystemCatalog::ColType& colType) const; + uint64_t* lbid, bool* fromDictScan, int128_t* min, int128_t* max, uint32_t* cachedIO, + uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, + bool* hasBinaryColumn, const execplan::CalpontSystemCatalog::ColType& colType) const; void deserializeAggregateResult(messageqcpp::ByteStream* in, std::vector* out) const; bool countThisMsg(messageqcpp::ByteStream& in) const; @@ -365,4 +365,3 @@ class BatchPrimitiveProcessorJL }; } // namespace joblist - diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 8e42a73e6..bc441168b 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -130,6 +130,59 @@ ColumnCommandJL::ColumnCommandJL(const pColStep& step) fFilesPerColumnPartition = cf->uFromText(fpc); } +ColumnCommandJL::ColumnCommandJL(const ColumnCommandJL& prevCmd, const DictStepJL& dictWithFilters) +{ + BRM::DBRM dbrm; + + /* grab necessary vars from scan */ + traceFlags = prevCmd.traceFlags; + // we should call this constructor only when paired with dictionary + // and in that case previous command should not have any filters and + // should be "dict" (tokens) column command. + idbassert(dictWithFilters.getFilterCount() == 0 || prevCmd.filterCount == 0); + idbassert(prevCmd.fIsDict); + + // need to reencode filters. + filterString = dictWithFilters.reencodedFilterString(); + // we have a limitation here. + // consider this: textcol IS NULL AND textcol IN ('a', 'b') + // XXX: should check. + if (filterString.length() > 0 && (BOP = dictWithFilters.getBop() || prevCmd.filterString.length() < 1)) + { + filterCount = dictWithFilters.getFilterCount(); + BOP = dictWithFilters.getBop(); + fContainsRanges = true; + } + else + { + filterCount = prevCmd.filterCount; + filterString = prevCmd.filterString; + BOP = prevCmd.BOP; + } + isScan = prevCmd.isScan; + colType = prevCmd.colType; + extents = prevCmd.extents; + OID = prevCmd.OID; + colName = prevCmd.colName; + rpbShift = prevCmd.rpbShift; + fIsDict = prevCmd.fIsDict; + fLastLbid = prevCmd.fLastLbid; + lbid = prevCmd.lbid; + traceFlags = prevCmd.traceFlags; + dbroot = prevCmd.dbroot; + numDBRoots = prevCmd.numDBRoots; + + /* I think modmask isn't necessary for scans */ + divShift = prevCmd.divShift; + modMask = (1 << divShift) - 1; + + // @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use + // in RID calculation. + fFilesPerColumnPartition = prevCmd.fFilesPerColumnPartition; + // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + fExtentsPerSegmentFile = prevCmd.fExtentsPerSegmentFile; +} + ColumnCommandJL::~ColumnCommandJL() { } @@ -141,9 +194,22 @@ void ColumnCommandJL::createCommand(ByteStream& bs) const colType.serialize(bs); bs << (uint8_t)isScan; bs << traceFlags; - bs << filterString; - bs << BOP; - bs << filterCount; + if (isDict() && fContainsRanges) + { + // XXX: we should discern here between IS (NOT) NULL and other filters. + ByteStream empty; + auto zeroFC = filterCount; + bs << empty; + bs << BOP; + zeroFC = 0; + bs << zeroFC; + } + else + { + bs << filterString; + bs << BOP; + bs << filterCount; + } serializeInlineVector(bs, fLastLbid); CommandJL::createCommand(bs); @@ -250,7 +316,7 @@ string ColumnCommandJL::toString() { ostringstream ret; - ret << "ColumnCommandJL: " << filterCount << " filters colwidth=" << colType.colWidth << " oid=" << OID + ret << "ColumnCommandJL: " << filterCount << " filters, BOP=" << ((int)BOP) << ", colwidth=" << colType.colWidth << " oid=" << OID << " name=" << colName; if (isScan) @@ -286,4 +352,9 @@ void ColumnCommandJL::reloadExtents() sort(extents.begin(), extents.end(), BRM::ExtentSorter()); } +bool ColumnCommandJL::getIsDict() +{ + return fIsDict; +} + }; // namespace joblist diff --git a/dbcon/joblist/columncommand-jl.h b/dbcon/joblist/columncommand-jl.h index ad36f72aa..4a864a253 100644 --- a/dbcon/joblist/columncommand-jl.h +++ b/dbcon/joblist/columncommand-jl.h @@ -33,6 +33,7 @@ #include "primitivestep.h" #include "command-jl.h" +#include "dictstep-jl.h" namespace joblist { @@ -41,6 +42,7 @@ class ColumnCommandJL : public CommandJL public: ColumnCommandJL(const pColScanStep&, std::vector lastLBID); ColumnCommandJL(const pColStep&); + ColumnCommandJL(const ColumnCommandJL&, const DictStepJL&); virtual ~ColumnCommandJL(); virtual void createCommand(messageqcpp::ByteStream& bs) const; @@ -111,6 +113,7 @@ class ColumnCommandJL : public CommandJL std::vector fLastLbid; bool fIsDict; + bool fContainsRanges = false; // @Bug 2889. Added two members below for drop partition enhancement. // RJD: make sure that we keep enough significant digits around for partition math @@ -125,7 +128,7 @@ class ColumnCommandJL : public CommandJL public: // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile) static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; + bool getIsDict() override; }; } // namespace joblist - diff --git a/dbcon/joblist/command-jl.h b/dbcon/joblist/command-jl.h index a45fc5a67..3921b419b 100644 --- a/dbcon/joblist/command-jl.h +++ b/dbcon/joblist/command-jl.h @@ -96,6 +96,11 @@ class CommandJL virtual CommandType getCommandType() = 0; + virtual bool getIsDict() + { + return false; + } + protected: BatchPrimitiveProcessorJL* bpp; uint32_t OID; diff --git a/dbcon/joblist/dictstep-jl.cpp b/dbcon/joblist/dictstep-jl.cpp index 2c3c51737..1e39d2407 100644 --- a/dbcon/joblist/dictstep-jl.cpp +++ b/dbcon/joblist/dictstep-jl.cpp @@ -29,6 +29,7 @@ // #include "bpp-jl.h" +#include "string_prefixes.h" using namespace std; using namespace messageqcpp; @@ -50,7 +51,6 @@ DictStepJL::DictStepJL(const pDictionaryStep& dict) if (hasEqFilter) { - // cout << "saw eqfilter\n"; eqOp = dict.tmpCOP; eqFilter = dict.eqFilter; } @@ -120,4 +120,63 @@ void DictStepJL::setWidth(uint16_t w) colWidth = w; } +messageqcpp::ByteStream DictStepJL::reencodedFilterString() const +{ + messageqcpp::ByteStream bs; + + if (hasEqFilter) + { + idbassert(filterCount == eqFilter.size()); + + for (uint32_t i = 0; i < filterCount; i++) + { + uint8_t roundFlag = 0; + int64_t encodedPrefix = encodeStringPrefix((unsigned char*)eqFilter[i].c_str(), eqFilter[i].size(), charsetNumber); + bs << eqOp; + bs << roundFlag; + bs << encodedPrefix; + } + } + else + { + messageqcpp::ByteStream filterStringCopy( + filterString); // XXX I am not sure about real semantics of messagecpp::ByteStream. So - copy. + // please erfer to pdictionary.cpp in this dicrectory, addFilter function for a proper encoding of string + // filters. + for (uint32_t i = 0; i < filterCount; i++) + { + uint8_t cop, roundFlag = 0; + uint16_t size; + const uint8_t* ptr; + int64_t encodedPrefix; + filterStringCopy >> cop; + // as we are dealing with prefixes, we have to use "... or equal" conditions instead of + // strict ones. + // Consider this: ... WHERE col > 'customer#001' AND col < 'customer#100'. + // "Working with prefixes of 8 bytes" means these conditions reduce to ... WHERE col > 'customer' AND + // col < 'customer' and their AND relation is impossible to satisfy. We do not pass this string to + // primproc and that means we can reencode operation codes here. + switch (cop) + { + case COMPARE_LT: + case COMPARE_NGE: cop = COMPARE_LE; break; + + case COMPARE_GT: + case COMPARE_NLE: cop = COMPARE_GE; break; + + default: break; + } + + bs << cop; + bs << roundFlag; + filterStringCopy >> size; + ptr = filterStringCopy.buf(); + encodedPrefix = encodeStringPrefix(ptr, size, charsetNumber); + bs << encodedPrefix; + filterStringCopy.advance(size); + } + } + return bs; +} + }; // namespace joblist diff --git a/dbcon/joblist/dictstep-jl.h b/dbcon/joblist/dictstep-jl.h index 7bf38fa10..5fe0c618e 100644 --- a/dbcon/joblist/dictstep-jl.h +++ b/dbcon/joblist/dictstep-jl.h @@ -61,6 +61,21 @@ class DictStepJL : public CommandJL void createCommand(messageqcpp::ByteStream&) const; void runCommand(messageqcpp::ByteStream&) const; + messageqcpp::ByteStream getFilterString() const + { + return filterString; + } + uint32_t getFilterCount() const + { + return filterCount; + } + messageqcpp::ByteStream reencodedFilterString() const; + + uint8_t getBop() const + { + return BOP; + } + private: DictStepJL(const DictStepJL&); diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 26e8dd023..f935f3db1 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -373,7 +373,15 @@ void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, cons } else { - fRow.setIntField(row.getIntField(mapping[i]), i); + if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::CHAR || + fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::VARCHAR) + { + fRow.setIntField(row.getUintField(mapping[i]), i); + } + else + { + fRow.setIntField(row.getIntField(mapping[i]), i); + } } } } diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index d388ce9f0..33a1f0104 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -1140,9 +1140,10 @@ bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) for (unsigned i = 0; i < numOfStepsAddToBps; i++) { - bps->setBPP((it + i)->get()); + auto pp = (it + i)->get(); + bps->setBPP(pp); bps->setStepCount(); - bps->setLastTupleId((it + i)->get()->tupleId()); + bps->setLastTupleId(pp->tupleId()); } it += itInc; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index e784eb26f..34a5f23d5 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -338,8 +338,8 @@ int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, int64_t lbid, } template -void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, const CalpontSystemCatalog::ColType& type, - bool validData) +void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, bool dictScan, + const CalpontSystemCatalog::ColType& type, bool validData) { MinMaxPartition* mmp = NULL; #ifdef DEBUG @@ -372,18 +372,20 @@ void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, const CalpontSystemCatal if (mmp->isValid == BRM::CP_INVALID) { - if (datatypes::isCharType(type.colDataType)) + if (!dictScan && datatypes::isCharType(type.colDataType)) { datatypes::Charset cs(const_cast(type).getCharset()); if (datatypes::TCharShort::strnncollsp(cs, min, mmp->min, type.colWidth) < 0 || - mmp->min == numeric_limits::max()) + // WIP + static_cast(mmp->min) == numeric_limits::max()) mmp->min = min; if (datatypes::TCharShort::strnncollsp(cs, max, mmp->max, type.colWidth) > 0 || - mmp->max == numeric_limits::min()) + // WIP + static_cast(mmp->max) == numeric_limits::min()) mmp->max = max; } - else if (datatypes::isUnsigned(type.colDataType)) + else if (dictScan || datatypes::isUnsigned(type.colDataType)) { if (static_cast(min) < static_cast(mmp->min)) mmp->min = min; @@ -526,7 +528,7 @@ bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType t case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: return size < 8; + case CalpontSystemCatalog::TEXT: return size <= 8; case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: @@ -695,15 +697,19 @@ bool LBIDList::checkRangeOverlap(T min, T max, T tmin, T tmax, bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const messageqcpp::ByteStream* bs, const uint16_t NOPS, - const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP) + const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP, + bool isDict) { int length = bs->length(), pos = 0; const char* MsgDataPtr = (const char*)bs->buf(); bool scan = true; int64_t value = 0; int128_t bigValue = 0; - bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); - bool bIsChar = datatypes::isCharType(ct.colDataType); + // MCOL-4580 - related. + // We definitely can compute isDict flag themselves here, as we have column type and width. + // But, we may also use already computed isDict flags in the steps, available with getIsDict() method.. + bool bIsUnsigned = isDict || datatypes::isUnsigned(ct.colDataType); + bool bIsChar = !isDict && datatypes::isCharType(ct.colDataType); for (int i = 0; i < NOPS; i++) { @@ -800,9 +806,12 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, { continue; } - else if (execplan::isNull(value, ct)) // This will work even if the data column is unsigned. + else { - continue; + if (execplan::isNull(value, ct)) // This will work even if the data column is unsigned. + { + continue; + } } if (bIsChar) @@ -898,11 +907,11 @@ template bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* const tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType colDataType); -template void LBIDList::UpdateMinMax(int128_t min, int128_t max, int64_t lbid, +template void LBIDList::UpdateMinMax(int128_t min, int128_t max, int64_t lbid, bool dictScan, const execplan::CalpontSystemCatalog::ColType& type, bool validData = true); -template void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, +template void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, bool dictScan, const execplan::CalpontSystemCatalog::ColType& type, bool validData = true); diff --git a/dbcon/joblist/lbidlist.h b/dbcon/joblist/lbidlist.h index 1e49db07f..bf7c6310b 100644 --- a/dbcon/joblist/lbidlist.h +++ b/dbcon/joblist/lbidlist.h @@ -98,8 +98,8 @@ class LBIDList execplan::CalpontSystemCatalog::ColDataType type); template - void UpdateMinMax(T min, T max, int64_t lbid, const execplan::CalpontSystemCatalog::ColType& type, - bool validData = true); + void UpdateMinMax(T min, T max, int64_t lbid, bool dictScan, + const execplan::CalpontSystemCatalog::ColType& type, bool validData = true); void UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType); @@ -107,7 +107,8 @@ class LBIDList bool CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const messageqcpp::ByteStream* MsgDataPtr, const uint16_t NOPS, - const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP); + const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP, + bool isDict); template bool checkSingleValue(T min, T max, T value, const execplan::CalpontSystemCatalog::ColType& type); diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index 3bcbd8582..d08580b4f 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -1054,9 +1054,10 @@ class BatchPrimitive : public JobStep, public PrimitiveMsg, public DECEventListe struct _CPInfo { - _CPInfo(int64_t MIN, int64_t MAX, uint64_t l, bool val) : min(MIN), max(MAX), LBID(l), valid(val){}; + _CPInfo(int64_t MIN, int64_t MAX, uint64_t l, bool dictScan, bool val) + : min(MIN), max(MAX), LBID(l), valid(val), dictScan(dictScan) {}; _CPInfo(int128_t BIGMIN, int128_t BIGMAX, uint64_t l, bool val) - : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val){}; + : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val), dictScan(false) {}; union { int128_t bigMin; @@ -1069,6 +1070,7 @@ struct _CPInfo }; uint64_t LBID; bool valid; + bool dictScan; }; /** @brief class TupleBPS @@ -1834,4 +1836,3 @@ class PseudoColStep : public pColStep }; } // namespace joblist - diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index 532ce54ba..df0daa3f5 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -75,6 +75,7 @@ using namespace rowgroup; #include "querytele.h" using namespace querytele; +#include "columnwidth.h" #include "pseudocolumn.h" //#define DEBUG 1 @@ -865,6 +866,7 @@ void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) vector cpColVec; vector lbidListVec; ColumnCommandJL* colCmd = 0; + bool defaultScanFlag = true; // @bug 2123. We call this earlier in the process for the hash join estimation process now. Return if // we've already done the work. @@ -876,7 +878,9 @@ void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) fCPEvaluated = true; if (colCmdVec.size() == 0) - return; + { + defaultScanFlag = false; // no reason to scan if there are no commands. + } for (uint32_t i = 0; i < colCmdVec.size(); i++) { @@ -902,30 +906,28 @@ void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) } if (cpColVec.size() == 0) - return; + { + defaultScanFlag = true; // no reason to scan if there are no predicates to evaluate. + } const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); for (uint32_t idx = 0; idx < numExtents; idx++) { - scanFlags[idx] = true; + scanFlags[idx] = defaultScanFlag; - for (uint32_t i = 0; i < cpColVec.size(); i++) + for (uint32_t i = 0; scanFlags[idx] && i < cpColVec.size(); i++) { colCmd = cpColVec[i]; const EMEntry& extent = colCmd->getExtents()[idx]; /* If any column filter eliminates an extent, it doesn't get scanned */ - scanFlags[idx] = - scanFlags[idx] && (ignoreCP || extent.partition.cprange.isValid != BRM::CP_VALID || - lbidListVec[i]->CasualPartitionPredicate( - extent.partition.cprange, &(colCmd->getFilterString()), - colCmd->getFilterCount(), colCmd->getColType(), colCmd->getBOP())); - - if (!scanFlags[idx]) - { - break; - } + scanFlags[idx] = scanFlags[idx] && (extent.colWid <= utils::MAXCOLUMNWIDTH) && // XXX: change to named constant. + (ignoreCP || extent.partition.cprange.isValid != BRM::CP_VALID || + colCmd->getColType().colWidth != extent.colWid || + lbidListVec[i]->CasualPartitionPredicate( + extent.partition.cprange, &(colCmd->getFilterString()), colCmd->getFilterCount(), + colCmd->getColType(), colCmd->getBOP(), colCmd->getIsDict())); } } @@ -2008,9 +2010,10 @@ void TupleBPS::processByteStreamVector(vectorgetRowGroupData(*bs, &fromPrimProc, &validCPData, &lbid, &min, &max, &cachedIO, &physIO, - &touchedBlocks, &unused, threadID, &hasBinaryColumn, fColType); + fBPP->getRowGroupData(*bs, &fromPrimProc, &validCPData, &lbid, &fromDictScan, &min, &max, &cachedIO, + &physIO, &touchedBlocks, &unused, threadID, &hasBinaryColumn, fColType); // Another layer of messiness. Need to refactor this fcn. while (!fromPrimProc.empty() && !cancelled()) @@ -2180,7 +2183,7 @@ void TupleBPS::processByteStreamVector(vectorread_some(uniqueID, fNumThreads, bsv, &flowControlOn); @@ -2361,11 +2366,13 @@ void TupleBPS::receiveMultiPrimitiveMessages() { if (fColType.colWidth > 8) { - lbidList->UpdateMinMax(cpv[i].bigMin, cpv[i].bigMax, cpv[i].LBID, fColType, cpv[i].valid); + lbidList->UpdateMinMax(cpv[i].bigMin, cpv[i].bigMax, cpv[i].LBID, cpv[i].dictScan, fColType, + cpv[i].valid); } else { - lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, fColType, cpv[i].valid); + lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, cpv[i].dictScan, fColType, + cpv[i].valid); } } } diff --git a/mysql-test/columnstore/basic/r/ctype_extent_koi8u.result b/mysql-test/columnstore/basic/r/ctype_extent_koi8u.result index 231c29855..eeb715c77 100644 --- a/mysql-test/columnstore/basic/r/ctype_extent_koi8u.result +++ b/mysql-test/columnstore/basic/r/ctype_extent_koi8u.result @@ -87,7 +87,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET koi8u COLLATE koi8u_general_ci) c1 HEX(c1) Ъ─ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_koi8u t1 c1 80FF 80FF c1 Ъ─ Level Code Message @@ -100,7 +100,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET koi8u COLLATE koi8u_general_ci) c1 HEX(c1) ЪЪЪ─ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_koi8u t1 c1 80FFFFFF 80FFFFFF c1 ЪЪЪ─ Level Code Message @@ -127,7 +127,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET koi8u COLLATE koi8u_general_nopad_ci) c1 HEX(c1) Ъ─ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_koi8u t1 c1 80FF 80FF c1 Ъ─ Level Code Message @@ -140,7 +140,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET koi8u COLLATE koi8u_general_nopad_ci) c1 HEX(c1) ЪЪЪ─ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_koi8u t1 c1 80FFFFFF 80FFFFFF c1 ЪЪЪ─ Level Code Message @@ -167,7 +167,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET koi8u COLLATE koi8u_bin) c1 HEX(c1) Ъ─ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_koi8u t1 c1 80FF 80FF c1 Ъ─ Level Code Message @@ -180,7 +180,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET koi8u COLLATE koi8u_bin) c1 HEX(c1) ЪЪЪ─ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_koi8u t1 c1 80FFFFFF 80FFFFFF c1 ЪЪЪ─ Level Code Message @@ -207,7 +207,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET koi8u COLLATE koi8u_nopad_bin) c1 HEX(c1) Ъ─ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_koi8u t1 c1 80FF 80FF c1 Ъ─ Level Code Message @@ -220,7 +220,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET koi8u COLLATE koi8u_nopad_bin) c1 HEX(c1) ЪЪЪ─ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_koi8u t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_koi8u t1 c1 80FFFFFF 80FFFFFF c1 ЪЪЪ─ Level Code Message diff --git a/mysql-test/columnstore/basic/r/ctype_extent_latin1.result b/mysql-test/columnstore/basic/r/ctype_extent_latin1.result index fb338da26..70cbb89da 100644 --- a/mysql-test/columnstore/basic/r/ctype_extent_latin1.result +++ b/mysql-test/columnstore/basic/r/ctype_extent_latin1.result @@ -99,7 +99,7 @@ CREATE TABLE t1 (c1 CHAR(1) CHARACTER SET latin1 COLLATE latin1_swedish_ci) c1 HEX(c1) é E9 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFFFFE9 FFFFFFFFFFFFFFE9 +mcs_ctype_extent_latin1 t1 c1 E9 E9 c1 é Level Code Message @@ -112,7 +112,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET latin1 COLLATE latin1_swedish_ci) c1 HEX(c1) ÿ€ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_latin1 t1 c1 80FF 80FF c1 ÿ€ Level Code Message @@ -125,7 +125,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET latin1 COLLATE latin1_swedish_ci) c1 HEX(c1) ÿÿÿ€ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_latin1 t1 c1 80FFFFFF 80FFFFFF c1 ÿÿÿ€ Level Code Message @@ -152,7 +152,7 @@ CREATE TABLE t1 (c1 CHAR(1) CHARACTER SET latin1 COLLATE latin1_swedish_nopad_ci c1 HEX(c1) é E9 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFFFFE9 FFFFFFFFFFFFFFE9 +mcs_ctype_extent_latin1 t1 c1 E9 E9 c1 é Level Code Message @@ -165,7 +165,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET latin1 COLLATE latin1_swedish_nopad_ci c1 HEX(c1) ÿ€ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_latin1 t1 c1 80FF 80FF c1 ÿ€ Level Code Message @@ -178,7 +178,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET latin1 COLLATE latin1_swedish_nopad_ci c1 HEX(c1) ÿÿÿ€ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_latin1 t1 c1 80FFFFFF 80FFFFFF c1 ÿÿÿ€ Level Code Message @@ -205,7 +205,7 @@ CREATE TABLE t1 (c1 CHAR(1) CHARACTER SET latin1 COLLATE latin1_bin) c1 HEX(c1) é E9 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFFFFE9 FFFFFFFFFFFFFFE9 +mcs_ctype_extent_latin1 t1 c1 E9 E9 c1 é Level Code Message @@ -218,7 +218,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET latin1 COLLATE latin1_bin) c1 HEX(c1) ÿ€ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_latin1 t1 c1 80FF 80FF c1 ÿ€ Level Code Message @@ -231,7 +231,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET latin1 COLLATE latin1_bin) c1 HEX(c1) ÿÿÿ€ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_latin1 t1 c1 80FFFFFF 80FFFFFF c1 ÿÿÿ€ Level Code Message @@ -258,7 +258,7 @@ CREATE TABLE t1 (c1 CHAR(1) CHARACTER SET latin1 COLLATE latin1_nopad_bin) c1 HEX(c1) é E9 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFFFFE9 FFFFFFFFFFFFFFE9 +mcs_ctype_extent_latin1 t1 c1 E9 E9 c1 é Level Code Message @@ -271,7 +271,7 @@ CREATE TABLE t1 (c1 CHAR(2) CHARACTER SET latin1 COLLATE latin1_nopad_bin) c1 HEX(c1) ÿ€ FF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFFFFFF80FF FFFFFFFFFFFF80FF +mcs_ctype_extent_latin1 t1 c1 80FF 80FF c1 ÿ€ Level Code Message @@ -284,7 +284,7 @@ CREATE TABLE t1 (c1 CHAR(4) CHARACTER SET latin1 COLLATE latin1_nopad_bin) c1 HEX(c1) ÿÿÿ€ FFFFFF80 table_schema table_name column_name hex(CAST(e.min_value AS SIGNED)) hex(CAST(e.max_value AS SIGNED)) -mcs_ctype_extent_latin1 t1 c1 FFFFFFFF80FFFFFF FFFFFFFF80FFFFFF +mcs_ctype_extent_latin1 t1 c1 80FFFFFF 80FFFFFF c1 ÿÿÿ€ Level Code Message diff --git a/mysql-test/columnstore/bugfixes/mcol-4940.result b/mysql-test/columnstore/bugfixes/mcol-4940.result index 0ca364b2b..abb04e159 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4940.result +++ b/mysql-test/columnstore/bugfixes/mcol-4940.result @@ -6,6 +6,6 @@ insert into rounding_table values (26805, 1252, -9647); insert into rounding_table values (26806, 573, -2804.5); SELECT CASE a WHEN 26805 THEN ROUND(c/b, 2) WHEN 26806 THEN b END MCOL4940 FROM ( SELECT a, SUM(b) b, SUM(c) c FROM rounding_table GROUP BY a ) abc ; MCOL4940 -573 -7.71 +573 DROP DATABASE mcol_4940; diff --git a/mysql-test/columnstore/bugfixes/mcol-4940.test b/mysql-test/columnstore/bugfixes/mcol-4940.test index 4d5ceb23d..82c417811 100644 --- a/mysql-test/columnstore/bugfixes/mcol-4940.test +++ b/mysql-test/columnstore/bugfixes/mcol-4940.test @@ -8,7 +8,8 @@ USE mcol_4940; create table rounding_table ( a int, b double, c double) engine=columnstore; insert into rounding_table values (26805, 1252, -9647); insert into rounding_table values (26806, 573, -2804.5); - + +--sorted_result SELECT CASE a WHEN 26805 THEN ROUND(c/b, 2) WHEN 26806 THEN b END MCOL4940 FROM ( SELECT a, SUM(b) b, SUM(c) c FROM rounding_table GROUP BY a ) abc ; DROP DATABASE mcol_4940; diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 7e840373e..ff74406a9 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -42,6 +42,8 @@ using namespace boost; #include "simd_sse.h" #include "utils/common/columnwidth.h" +#include "exceptclasses.h" + using namespace logging; using namespace dbbc; using namespace primitives; @@ -1690,7 +1692,9 @@ void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, C dataType == execplan::CalpontSystemCatalog::TEXT) && !isDictTokenScan(in)) { - filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); + using UT = typename std::conditional::value, T, + typename datatypes::make_unsigned::type>::type; + filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); return; } diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index 360d8b607..40cac5014 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -31,6 +31,7 @@ using namespace std; #include "messageobj.h" #include "exceptclasses.h" #include "dataconvert.h" +#include "string_prefixes.h" #include using namespace logging; @@ -391,8 +392,13 @@ void PrimitiveProcessor::nextSig(int NVALS, const PrimToken* tokens, p_DataValue } void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector* out, bool skipNulls, +#if defined(XXX_PRIMITIVES_TOKEN_RANGES_XXX) + uint32_t charsetNumber, boost::shared_ptr eqFilter, + uint8_t eqOp, uint64_t minMax[2]) +#else uint32_t charsetNumber, boost::shared_ptr eqFilter, uint8_t eqOp) +#endif { PrimToken* outToken; const DictFilterElement* filter = 0; @@ -437,6 +443,14 @@ void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector* out, sigptr.len != -1; nextSig(in->NVALS, in->tokens, &sigptr, in->OutputType, (in->InputFlags ? true : false), skipNulls)) { +#if defined(XXX_PRIMITIVES_TOKEN_RANGES_XXX) + if (minMax) + { + uint64_t v = encodeStringPrefix_check_null(sigptr.data, sigptr.len, charsetNumber); + minMax[1] = minMax[1] < v ? v : minMax[1]; + minMax[0] = minMax[0] > v ? v : minMax[0]; + } +#endif // do aggregate processing if (in->OutputType & OT_AGGREGATE) { diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index 73e6e90fb..f6c50b49d 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -54,6 +54,9 @@ class PrimTest; +// XXX: turn off dictionary range setting during scan. +#define XXX_PRIMITIVES_TOKEN_RANGES_XXX + namespace primitives { enum ColumnFilterMode @@ -423,7 +426,13 @@ class PrimitiveProcessor // void p_ColAggregate(const NewColAggRequestHeader *in, NewColAggResultHeader *out); void p_Dictionary(const DictInput* in, std::vector* out, bool skipNulls, uint32_t charsetNumber, - boost::shared_ptr eqFilter, uint8_t eqOp); +#if !defined(XXX_PRIMITIVES_TOKEN_RANGES_XXX) + boost::shared_ptr eqFilter, uint8_t eqOp +#else + boost::shared_ptr eqFilter, uint8_t eqOp, + uint64_t minMax[2] // as name suggests, [0] is min, [1] is max. +#endif + ); inline void setLogicalBlockMode(bool b) { diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index dab8c6693..50306b335 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -117,6 +117,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() , validCPData(false) , minVal(MAX64) , maxVal(MIN64) +, cpDataFromDictScan(false) , lbidForCP(0) , hasWideColumnOut(false) , busyLoaderCount(0) @@ -138,6 +139,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() , processorThreads(0) , ptMask(0) , firstInstance(false) + , valuesLBID(0) { pp.setLogicalBlockMode(true); pp.setBlockPtr((int*)blockData); @@ -167,6 +169,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, , validCPData(false) , minVal(MAX64) , maxVal(MIN64) + , cpDataFromDictScan(false) , lbidForCP(0) , hasWideColumnOut(false) , busyLoaderCount(0) @@ -186,10 +189,10 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, , sockIndex(0) , endOfJoinerRan(false) , processorThreads(_processorThreads) - , // processorThreads(32), // ptMask(processorThreads - 1), - firstInstance(true) + , firstInstance(true) + , valuesLBID(0) { // promote processorThreads to next power of 2. also need to change the name to bucketCount or similar processorThreads = nextPowOf2(processorThreads); @@ -2010,6 +2013,7 @@ void BatchPrimitiveProcessor::writeProjectionPreamble() { *serialized << (uint8_t)1; *serialized << lbidForCP; + *serialized << ((uint8_t)cpDataFromDictScan); if (UNLIKELY(hasWideColumnOut)) { // PSA width @@ -2108,6 +2112,7 @@ void BatchPrimitiveProcessor::makeResponse() { *serialized << (uint8_t)1; *serialized << lbidForCP; + *serialized << ((uint8_t)cpDataFromDictScan); if (UNLIKELY(hasWideColumnOut)) { @@ -2208,6 +2213,7 @@ int BatchPrimitiveProcessor::operator()() } validCPData = false; + cpDataFromDictScan = false; #ifdef PRIMPROC_STOPWATCH stopwatch->start("BPP() execute"); execute(stopwatch); diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index 42dbe6e46..d930099a7 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -256,6 +256,7 @@ class BatchPrimitiveProcessor int128_t max128Val; int64_t maxVal; }; + bool cpDataFromDictScan; uint64_t lbidForCP; bool hasWideColumnOut; @@ -431,6 +432,7 @@ class BatchPrimitiveProcessor uint processorThreads; uint ptMask; bool firstInstance; + uint64_t valuesLBID; friend class Command; friend class ColumnCommand; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index a7069359b..315d9293e 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -106,8 +106,8 @@ void ColumnCommand::execute() { values = bpp->values; wide128Values = bpp->wide128Values; + bpp->valuesLBID = lbid; } - _execute(); } @@ -225,9 +225,13 @@ void ColumnCommand::issuePrimitive() loadData(); if (!suppressFilter) + { bpp->getPrimitiveProcessor().setParsedColumnFilter(parsedColumnFilter); + } else + { bpp->getPrimitiveProcessor().setParsedColumnFilter(emptyFilter); + } switch (colType.colWidth) { @@ -282,6 +286,7 @@ void ColumnCommand::updateCPDataNarrow() if (_isScan) { bpp->validCPData = (outMsg->ValidMinMax && !wasVersioned); + bpp->cpDataFromDictScan = false; bpp->lbidForCP = lbid; bpp->maxVal = static_cast(outMsg->Max); bpp->minVal = static_cast(outMsg->Min); @@ -295,6 +300,7 @@ void ColumnCommand::updateCPDataWide() if (_isScan) { bpp->validCPData = (outMsg->ValidMinMax && !wasVersioned); + bpp->cpDataFromDictScan = false; bpp->lbidForCP = lbid; if (colType.isWideDecimalType()) { diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index 60adfa573..7688a9f9f 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -46,6 +46,8 @@ extern uint32_t dictBufferSize; DictStep::DictStep() : Command(DICT_STEP), strValues(NULL), filterCount(0), bufferSize(0) { + fMinMax[0] = MAX_UBIGINT; + fMinMax[1] = MIN_UBIGINT; } DictStep::~DictStep() @@ -65,6 +67,8 @@ DictStep& DictStep::operator=(const DictStep& d) eqOp = d.eqOp; filterCount = d.filterCount; charsetNumber = d.charsetNumber; + fMinMax[0] = d.fMinMax[0]; + fMinMax[1] = d.fMinMax[1]; return *this; } @@ -147,8 +151,11 @@ void DictStep::issuePrimitive(bool isFilter) bpp->physIO += blocksRead; bpp->touchedBlocks++; } - +#if !defined(XXX_PRIMITIVES_TOKEN_RANGES_XXX) bpp->pp.p_Dictionary(primMsg, &result, isFilter, charsetNumber, eqFilter, eqOp); +#else + bpp->pp.p_Dictionary(primMsg, &result, isFilter, charsetNumber, eqFilter, eqOp, fMinMax); +#endif } void DictStep::copyResultToTmpSpace(OrderedToken* ot) @@ -390,6 +397,14 @@ void DictStep::_execute() copyResultToFinalPosition(newRidList.get()); copyRidsForFilterCmd(); } + if (fMinMax[0] <= fMinMax[1] && bpp->valuesLBID != 0) + { + bpp->validCPData = true; + bpp->cpDataFromDictScan = true; + bpp->lbidForCP = bpp->valuesLBID; + bpp->maxVal = fMinMax[1]; + bpp->minVal = fMinMax[0]; + } // cout << "DS: /_execute()\n"; } diff --git a/primitives/primproc/dictstep.h b/primitives/primproc/dictstep.h index 10af6c009..577abe6f3 100644 --- a/primitives/primproc/dictstep.h +++ b/primitives/primproc/dictstep.h @@ -158,6 +158,7 @@ class DictStep : public Command bool hasEqFilter; boost::shared_ptr eqFilter; uint8_t eqOp; // COMPARE_EQ or COMPARE_NE + uint64_t fMinMax[2]; friend class RTSCommand; }; diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index 137ddcb06..17d78d476 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -11,7 +11,8 @@ set(common_LIB_SRCS nullvaluemanip.cpp threadnaming.cpp utils_utf8.cpp - statistics.cpp) + statistics.cpp + string_prefixes.cpp) add_library(common SHARED ${common_LIB_SRCS}) diff --git a/utils/common/collation.h b/utils/common/collation.h index f80fa1004..951a2e03c 100644 --- a/utils/common/collation.h +++ b/utils/common/collation.h @@ -177,6 +177,12 @@ class Charset bool res = !mCharset->wildcmp(subject.str(), subject.end(), pattern.str(), pattern.end(), '\\', '_', '%'); return neg ? !res : res; } + size_t strnxfrm(uchar* dst, size_t dstlen, uint nweights, const uchar* src, size_t srclen, uint flags) + { + idbassert(mCharset->coll); + + return mCharset->coll->strnxfrm(mCharset, dst, dstlen, nweights, src, srclen, flags); + } }; class CollationAwareHasher : public Charset diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index da2fd76e6..76e373511 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -198,6 +198,7 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt os << "getSignedNullValue(): got bad column type (" << t << "). Width=" << colWidth << endl; throw logic_error(os.str()); } + } } // namespace utils diff --git a/utils/common/string_prefixes.cpp b/utils/common/string_prefixes.cpp new file mode 100644 index 000000000..9f9c5c2e7 --- /dev/null +++ b/utils/common/string_prefixes.cpp @@ -0,0 +1,51 @@ +/* + Copyright (C) 2021, 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/* handling of the conversion of string prefixes to int64_t for quick range checking */ + +#include "collation.h" +#include "joblisttypes.h" + +#include "string_prefixes.h" + +// XXX: string (or, actually, a BLOB) with all NUL chars will be encoded into zero. Which corresponds to +// encoding of empty string, or NULL. +int64_t encodeStringPrefix(const uint8_t* str, size_t len, int charsetNumber) +{ + datatypes::Charset cset(charsetNumber); + uint8_t fixedLenPrefix[8]; + memset(fixedLenPrefix, 0, sizeof(fixedLenPrefix)); + cset.strnxfrm(fixedLenPrefix, sizeof(fixedLenPrefix), 8, str, len, 0); + int64_t acc = 0; + size_t i; + for (i = 0; i < 8; i++) + { + uint8_t byte = fixedLenPrefix[i]; + acc = (acc << 8) + byte; + } + return acc; +} + +int64_t encodeStringPrefix_check_null(const uint8_t* str, size_t len, int charsetNumber) +{ + if (len < 1) + { + return joblist::UBIGINTNULL; + } + return encodeStringPrefix(str, len, charsetNumber); +} diff --git a/utils/common/string_prefixes.h b/utils/common/string_prefixes.h new file mode 100644 index 000000000..750c552b5 --- /dev/null +++ b/utils/common/string_prefixes.h @@ -0,0 +1,32 @@ +/* + Copyright (C) 2021, 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/* handling of the conversion of string prefixes to int64_t for quick range checking */ + +#pragma once + +#include +#include + +// Encode string prefix into an int64_t, packing as many chars from string as possible +// into the result and respecting the collation provided by charsetNumber. +// +// For one example, for CI Czech collation, encodeStringPrefix("cz") < encodeStringPrefix("CH"). +int64_t encodeStringPrefix(const uint8_t* str, size_t len, int charsetNumber); + +int64_t encodeStringPrefix_check_null(const uint8_t* str, size_t len, int charsetNumber); diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index c674f454e..217a25a67 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -45,7 +45,7 @@ CalpontSystemCatalog::ColType Func_monthname::operationType(FunctionParm& fp, string Func_monthname::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int32_t month = getIntVal(row, parm, isNull, op_ct); + int32_t month = getIntValInternal(row, parm, isNull, op_ct); if (month == -1) return ""; @@ -74,8 +74,8 @@ int64_t Func_monthname::getTimestampIntVal(rowgroup::Row& row, FunctionParm& par return val; } -int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType& op_ct) +int64_t Func_monthname::getIntValInternal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + CalpontSystemCatalog::ColType& op_ct) { int64_t val = 0; dataconvert::DateTime aDateTime; @@ -165,12 +165,20 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& break; - default: isNull = true; return -1; + default: + isNull = true; + return -1; } return -1; } +int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + return getIntValInternal(row, parm, isNull, op_ct); +} + double Func_monthname::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index c6e673361..d5a973792 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -41,6 +41,8 @@ using namespace logging; #include "funchelpers.h" +#include "exceptclasses.h" + namespace { using namespace funcexp; @@ -136,18 +138,27 @@ int64_t Func_round::getIntVal(Row& row, FunctionParm& parm, bool& isNull, uint64_t Func_round::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - uint64_t x; - if (UNLIKELY(op_ct.colDataType == execplan::CalpontSystemCatalog::DATE)) + IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); + + if (!op_ct.isWideDecimalType()) { - IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); - x = static_cast(d.value); + if (x.scale > 0) + { + while (x.scale-- > 0) + x.value /= 10; + } + else + { + while (x.scale++ < 0) + x.value *= 10; + } + + return x.value; } else { - x = parm[0]->data()->getUintVal(row, isNull); + return static_cast(x.getIntegralPart()); } - - return x; } double Func_round::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, @@ -434,10 +445,11 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull { uint64_t x = parm[0]->data()->getUintVal(row, isNull); - if (x > (uint64_t)helpers::maxNumber_c[18]) - { - x = helpers::maxNumber_c[18]; - } + // why it is here at all??? + // if (x > (uint64_t)helpers::maxNumber_c[18]) + //{ + // x = helpers::maxNumber_c[18]; + //} decimal.value = x; decimal.scale = 0; diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 1287cfd64..37657b0f3 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -651,6 +651,8 @@ class Func_monthname : public Func_Str int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntValInternal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); double getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); diff --git a/utils/loggingcpp/exceptclasses.h b/utils/loggingcpp/exceptclasses.h index 477a9c35e..456436066 100644 --- a/utils/loggingcpp/exceptclasses.h +++ b/utils/loggingcpp/exceptclasses.h @@ -305,4 +305,3 @@ class ProtocolError : public std::logic_error } while (0) } // namespace logging - diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 95d280e18..0180242af 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -2820,16 +2820,24 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co e->HWM = 0; e->status = EXTENTUNAVAILABLE; // mark extent as in process - // Partition, segment, and blockOffset 0 represents new table or column. - // When DDL creates a table, we can mark the first extent as VALID, since - // the table has no data. Marking as VALID enables cpimport to update - // the CP min/max for the first import. - // If DDL is adding a column to an existing table, setting to VALID won't - // hurt, because DDL resets to INVALID after the extent is created. - if ((e->partitionNum == 0) && (e->segmentNum == 0) && (e->blockOffset == 0)) - e->partition.cprange.isValid = CP_VALID; - else - e->partition.cprange.isValid = CP_INVALID; +#if 0 // XXX: sergueyz: I'll leave these under conditional flag for a while because it appears a huge change. + // Partition, segment, and blockOffset 0 represents new table or column. + // When DDL creates a table, we can mark the first extent as VALID, since + // the table has no data. Marking as VALID enables cpimport to update + // the CP min/max for the first import. + // If DDL is adding a column to an existing table, setting to VALID won't + // hurt, because DDL resets to INVALID after the extent is created. + // XXX: the comment above is out of date. bulk set of extents ranges + // works differently right now. + if ((e->partitionNum == 0) && + (e->segmentNum == 0) && + (e->blockOffset == 0)) + e->partition.cprange.isValid = CP_VALID; + else + e->partition.cprange.isValid = CP_INVALID; +#else + e->partition.cprange.isValid = CP_INVALID; +#endif partitionNum = e->partitionNum; segmentNum = e->segmentNum; @@ -3029,16 +3037,22 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, uint32_t e->HWM = 0; } - // Partition, segment, and blockOffset 0 represents new table or column. - // When DDL creates a table, we can mark the first extent as VALID, since - // the table has no data. Marking as VALID enables cpimport to update - // the CP min/max for the first import. - // If DDL is adding a column to an existing table, setting to VALID won't - // hurt, because DDL resets to INVALID after the extent is created. - if ((e->partitionNum == 0) && (e->segmentNum == 0) && (e->blockOffset == 0)) - e->partition.cprange.isValid = CP_VALID; - else - e->partition.cprange.isValid = CP_INVALID; +#if 0 // XXX: sergueyz: I'll leave these under conditional flag for a while because it appears a huge change. + // Partition, segment, and blockOffset 0 represents new table or column. + // When DDL creates a table, we can mark the first extent as VALID, since + // the table has no data. Marking as VALID enables cpimport to update + // the CP min/max for the first import. + // If DDL is adding a column to an existing table, setting to VALID won't + // hurt, because DDL resets to INVALID after the extent is created. + if ((e->partitionNum == 0) && + (e->segmentNum == 0) && + (e->blockOffset == 0)) + e->partition.cprange.isValid = CP_VALID; + else + e->partition.cprange.isValid = CP_INVALID; +#else + e->partition.cprange.isValid = CP_INVALID; +#endif startBlockOffset = e->blockOffset; diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index ebec84dc2..7e51018ef 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -739,7 +739,7 @@ int BulkLoad::preProcess(Job& job, int tableNo, TableInfo* tableInfo) // Setup import to start loading into starting HWM DB file RETURN_ON_ERROR(info->setupInitialColumnExtent(dbRoot, partition, segment, job.jobTableList[tableNo].tblName, lbid, oldHwm, hwm, - bSkippedToNewExtent, false)); + bSkippedToNewExtent, bSkippedToNewExtent || oldHwm < 1)); } tableInfo->addColumn(info); diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index ef1792a09..106689eaf 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -1717,7 +1717,7 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) lastInputRowInExtent += columnInfo.rowsPerExtent(); - if (isUnsigned(columnInfo.column.dataType) || isCharType(columnInfo.column.dataType)) + if (isUnsigned(columnInfo.column.dataType)) { if (columnInfo.column.width <= 8) { diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index 9aa32cc1e..2abc40e63 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -54,7 +54,7 @@ class BLBufferStats }; BLBufferStats(ColDataType colDataType) : satCount(0) { - if (isUnsigned(colDataType) || isCharType(colDataType)) + if (isUnsigned(colDataType)) { minBufferVal = static_cast(MAX_UBIGINT); maxBufferVal = static_cast(MIN_UBIGINT); diff --git a/writeengine/bulk/we_colextinf.cpp b/writeengine/bulk/we_colextinf.cpp index 1551cd822..74271c2f4 100644 --- a/writeengine/bulk/we_colextinf.cpp +++ b/writeengine/bulk/we_colextinf.cpp @@ -104,7 +104,7 @@ void ColExtInf::addOrUpdateEntryTemplate(RID lastInputRow, T minVal, T maxVal, C } else // Update the range { - if (isUnsigned(colDataType) || isCharType(colDataType)) + if (isUnsigned(colDataType)) { if (width <= 8) { diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 3970e6c1e..6e8217bb1 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -241,6 +241,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = column.oid; } else @@ -656,9 +657,10 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err dctnryStruct.fCompressionType = 2; } - if (colStruct.tokenFlag) + if (colStruct.tokenFlag) // TODO: XXX: this is copied aplenty. NEED TO REFACTOR. { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = column.oid; } else @@ -1046,6 +1048,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = column.oid; } else @@ -2442,6 +2445,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -2846,6 +2850,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -3087,6 +3092,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -3273,6 +3279,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -4258,6 +4265,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -4545,6 +4553,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column1.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column1.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else @@ -4756,6 +4765,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = column5.colType.ddn.dictOID; + dctnryStruct.fCharsetNumber = column5.colType.charsetNumber; dctnryStruct.columnOid = colStruct.dataOid; } else diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index ec4aff7bc..e44844c1c 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -181,7 +181,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: colStruct.colDataType = colType.colDataType; - if (colStruct.tokenFlag) + dctnryStruct.fCharsetNumber = colType.charsetNumber; + + if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = colType.ddn.dictOID; dctnryStruct.columnOid = colStruct.dataOid; @@ -1037,6 +1039,8 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: colStruct.colDataType = colType.colDataType; + dctnryStruct.fCharsetNumber = colType.charsetNumber; + if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = colType.ddn.dictOID; @@ -1619,6 +1623,8 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, colStruct.colDataType = colType.colDataType; + dctnryStruct.fCharsetNumber = colType.charsetNumber; + if (colStruct.tokenFlag) { dctnryStruct.dctnryOid = colType.ddn.dictOID; @@ -2830,6 +2836,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin dctnryStruct.dctnryOid = colType.ddn.dictOID; dctnryStruct.columnOid = colStruct.dataOid; dctnryStruct.fCompressionType = colType.compressionType; + dctnryStruct.fCharsetNumber = colType.charsetNumber; dctnryStruct.colWidth = colType.colWidth; if (NO_ERROR != (error = fWEWrapper.openDctnry(txnId, dctnryStruct, false))) // @bug 5572 HDFS tmp file @@ -4445,6 +4452,8 @@ uint8_t WE_DMLCommandProc::processFixRows(messageqcpp::ByteStream& bs, std::stri dctnryStruct.fCompressionType = colStruct.fCompressionType; dctnryStruct.dctnryOid = 0; + dctnryStruct.fCharsetNumber = colType.charsetNumber; + if (colType.colWidth > 8) // token { colStruct.colWidth = 8; diff --git a/writeengine/server/we_server.cpp b/writeengine/server/we_server.cpp index b7c9fc194..1e0497319 100644 --- a/writeengine/server/we_server.cpp +++ b/writeengine/server/we_server.cpp @@ -149,11 +149,10 @@ int ServiceWriteEngine::setupResources() return -3; } - if (rlim.rlim_cur != 65536) + if (rlim.rlim_cur < 65536) { return -4; } - #endif return 0; } diff --git a/writeengine/shared/we_brm.h b/writeengine/shared/we_brm.h index cb5916a88..27a7e2a93 100644 --- a/writeengine/shared/we_brm.h +++ b/writeengine/shared/we_brm.h @@ -24,6 +24,7 @@ #pragma once #include +#include #include #include #include @@ -49,17 +50,19 @@ namespace WriteEngine // forward reference class DbFileOp; -/** @brief Extended CPInfo - with type handler for all type-related information */ +/** @brief Extended CPInfo - with all type-related information and associated range data */ struct ExtCPInfo { execplan::CalpontSystemCatalog::ColDataType fColType; int fColWidth; BRM::CPInfo fCPInfo; + std::shared_ptr> fStringsPrefixes; ExtCPInfo(execplan::CalpontSystemCatalog::ColDataType colType, int colWidth) : fColType(colType), fColWidth(colWidth) { fCPInfo.isBinaryColumn = (unsigned int)colWidth > datatypes::MAXLEGACYWIDTH; } + void toInvalid() { auto mm = datatypes::MinMaxInfo::invalidRange(fColType); @@ -68,7 +71,22 @@ struct ExtCPInfo fCPInfo.bigMax = mm.int128Max; fCPInfo.bigMin = mm.int128Min; } - + void addStringPrefix(int64_t strPrefix) + { + if (!fStringsPrefixes) + { + fStringsPrefixes.reset(new std::vector()); + } + fStringsPrefixes->push_back(strPrefix); + } + bool hasStringsPrefixes() const + { + return fStringsPrefixes.get() != nullptr; + } + int64_t* stringsPrefixes() const + { + return hasStringsPrefixes() ? fStringsPrefixes->data() : nullptr; + } bool isInvalid() { datatypes::MinMaxInfo mm; diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index e54f03b59..7bd11f892 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -344,6 +344,7 @@ struct DctnryStruct /** @brief Dctnry Interface Struct*/ uint16_t fColSegment; /** @brief Segment for column file */ uint16_t fColDbRoot; /** @brief DBRoot for column file */ int fCompressionType; /** @brief Compression tpye for column file */ + int fCharsetNumber; /** @brief Charset number to account for collation when computing string prefixes */ DctnryStruct() : dctnryOid(0) , columnOid(0) @@ -353,6 +354,7 @@ struct DctnryStruct /** @brief Dctnry Interface Struct*/ , fColSegment(0) , fColDbRoot(0) , fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) + , fCharsetNumber(8) { } }; diff --git a/writeengine/splitter/we_brmupdater.cpp b/writeengine/splitter/we_brmupdater.cpp index f6ea954e8..a7be2c0df 100644 --- a/writeengine/splitter/we_brmupdater.cpp +++ b/writeengine/splitter/we_brmupdater.cpp @@ -214,6 +214,7 @@ int WEBrmUpdater::updateHighWaterMarkInBRM() int WEBrmUpdater::updateCPAndHWMInBRM() { int rc = 0; + size_t i; // BUG 4232. some imports may not contain CP but HWM if ((fCPInfo.size() > 0) || (fHWMInfo.size() > 0)) @@ -227,6 +228,13 @@ int WEBrmUpdater::updateCPAndHWMInBRM() const std::vector & mergeCPDataArgs, VER_t transID = 0) DBRM_THROW; */ + for (i = 0; i < fCPInfo.size(); i++) + { + if (fCPInfo[i].newExtent) + { + fCPInfo[i].seqNum = 0; // to be in sync with DBRM. + } + } rc = fpBrm->bulkSetHWMAndCP(fHWMInfo, fCPInfoData, fCPInfo, 0); // rc = fpBrm->mergeExtentsMaxMin(fCPInfo); diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 414714725..9e87e8be5 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -21,6 +21,10 @@ /** @writeengine.cpp * A wrapper class for the write engine to write information to files */ + +// XXX: a definition to switch off computations for token columns. +//#define XXX_WRITEENGINE_TOKENS_RANGES_XXX + #include #include #include @@ -59,6 +63,7 @@ using namespace execplan; #include "MonitorProcMem.h" using namespace idbdatafile; #include "dataconvert.h" +#include "string_prefixes.h" #ifdef _MSC_VER #define isnan _isnan @@ -362,6 +367,9 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ case WR_UINT: case WR_ULONGLONG: case WR_CHAR: +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + case WR_TOKEN: +#endif { isUnsigned = true; break; @@ -385,6 +393,13 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ maxMin->fromToChars(); } } +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + if (colType == WR_TOKEN) + { + oldValArrayVoid = nullptr; // no old values for tokens, sadly. + valArrayVoid = (void*)maxMin->stringsPrefixes(); + } +#endif size_t i; for (i = 0; i < totalOldRow; i++) { @@ -435,6 +450,9 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); break; } +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + case WR_TOKEN: +#endif case WR_ULONGLONG: { fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, @@ -449,12 +467,11 @@ void WriteEngineWrapper::updateMaxMinRange(const size_t totalNewRow, const size_ } case WR_CHAR: { - fetchNewOldValues(uvalue, oldUValue, valArrayVoid, oldValArrayVoid, i, - totalNewRow); + fetchNewOldValues(value, oldValue, valArrayVoid, oldValArrayVoid, i, totalNewRow); // for characters (strings, actually), we fetched then in LSB order, on x86, at the very least. // this means most significant byte of the string, which is first, is now in LSB of uvalue/oldValue. // we must perform a conversion. - uvalue = uint64ToStr(uvalue); + value = uint64ToStr(uvalue); oldValue = uint64ToStr(oldValue); break; } @@ -576,6 +593,7 @@ void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColT curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); memcpy(value, curStr.c_str(), curStr.length()); + break; case WriteEngine::WR_FLOAT: @@ -1179,10 +1197,17 @@ static void log_this(const char *message, #endif /** @brief Determine whether we may update a column's ranges (by type) and return nullptr if we can't */ -static ExtCPInfo* getCPInfoToUpdateForUpdatableType(const ColStruct& colStruct, ExtCPInfo* currentCPInfo) +static ExtCPInfo* getCPInfoToUpdateForUpdatableType(const ColStruct& colStruct, ExtCPInfo* currentCPInfo, + OpType optype) { if (colStruct.tokenFlag) { +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + if (currentCPInfo && currentCPInfo->hasStringsPrefixes() && optype == INSERT) + { + return currentCPInfo; + } +#endif return nullptr; } switch (colStruct.colType) @@ -1689,10 +1714,16 @@ int WriteEngineWrapper::insertColumnRecs( for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) { +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + int64_t strPrefix; +#endif if (dctStr_iter->length() == 0) { Token nullToken; col_iter->data = nullToken; +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + strPrefix = (int64_t)joblist::UBIGINTNULL; // the string prefixes are signed long ints. +#endif } else { @@ -1702,6 +1733,10 @@ int WriteEngineWrapper::insertColumnRecs( DctnryTuple dctTuple; dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); dctTuple.sigSize = dctStr_iter->length(); +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + strPrefix = encodeStringPrefix_check_null(dctTuple.sigValue, dctTuple.sigSize, + dctnryStructList[i].fCharsetNumber); +#endif dctTuple.isNull = false; rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType); @@ -1717,6 +1752,9 @@ int WriteEngineWrapper::insertColumnRecs( col_iter->data = dctTuple.token; } +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + maxMins[i].fSplitMaxMinInfo[0].addStringPrefix(strPrefix); +#endif dctStr_iter++; col_iter++; } @@ -1744,10 +1782,16 @@ int WriteEngineWrapper::insertColumnRecs( for (uint32_t rows = 0; rows < rowsLeft; rows++) { +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + int64_t strPrefix; +#endif if (dctStr_iter->length() == 0) { Token nullToken; col_iter->data = nullToken; +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + strPrefix = joblist::UBIGINTNULL; // string prefixes are signed long ints. +#endif } else { @@ -1757,6 +1801,10 @@ int WriteEngineWrapper::insertColumnRecs( DctnryTuple dctTuple; dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str(); dctTuple.sigSize = dctStr_iter->length(); +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + strPrefix = encodeStringPrefix_check_null(dctTuple.sigValue, dctTuple.sigSize, + dctnryStructList[i].fCharsetNumber); +#endif dctTuple.isNull = false; rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType); @@ -1772,6 +1820,9 @@ int WriteEngineWrapper::insertColumnRecs( col_iter->data = dctTuple.token; } +#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX) + maxMins[i].fSplitMaxMinInfo[1].addStringPrefix(strPrefix); +#endif dctStr_iter++; col_iter++; } @@ -1938,7 +1989,7 @@ int WriteEngineWrapper::insertColumnRecs( if (isFirstBatchPm && (totalRow == rowsLeft)) { - // in this particular case we already marked extents as invalid up there. + // in this particular case we already marked extents as invalid above. } else { @@ -1950,7 +2001,7 @@ int WriteEngineWrapper::insertColumnRecs( if (firstHalfCount) { ExtCPInfo* cpInfoP = - getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[0]); + getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[0], m_opType); RID thisRid = rowsLeft ? lastRid : lastRidNew; successFlag = colOp->calculateRowId(thisRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); @@ -1966,7 +2017,7 @@ int WriteEngineWrapper::insertColumnRecs( if (rowsLeft) { ExtCPInfo* cpInfoP = - getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[1]); + getCPInfoToUpdateForUpdatableType(colStructList[i], &maxMins[i].fSplitMaxMinInfo[1], m_opType); if (cpInfoP) { RETURN_ON_ERROR(GetLBIDRange(newExtentsStartingLbids[i], colStructList[i], *cpInfoP)); @@ -4446,11 +4497,6 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, const vector currentExtentRangesPtrs(colStructList.size(), NULL); // pointers for each extent. + if (m_opType != DELETE) + m_opType = UPDATE; + for (unsigned j = 0; j < colStructList.size(); j++) { colOp = m_colOp[op(colStructList[j].fCompressionType)]; ExtCPInfo* cpInfoP = &(currentExtentRanges[j]); - cpInfoP = getCPInfoToUpdateForUpdatableType(colStructList[j], cpInfoP); + cpInfoP = getCPInfoToUpdateForUpdatableType(colStructList[j], cpInfoP, m_opType); currentExtentRangesPtrs[j] = cpInfoP; - if (colStructList[j].tokenFlag) - continue; + // XXX: highly dubious. + // if (!colStructList[j].tokenFlag) + // continue; width = colOp->getCorrectRowWidth(colStructList[j].colDataType, colStructList[j].colWidth); successFlag = colOp->calculateRowId(aRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); @@ -4550,9 +4600,6 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, const vectorsetExtentsMaxMin(infosToDrop); setInvalidCPInfosSpecialMarks(infosToUpdate); rc = BRMWrapper::getInstance()->setExtentsMaxMin(infosToUpdate); @@ -4611,12 +4659,9 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, const CSCTypesList& colOp = m_colOp[op(colExtentsStruct[j].fCompressionType)]; ExtCPInfo* cpInfoP = &(infosToUpdate[j]); - cpInfoP = getCPInfoToUpdateForUpdatableType(colExtentsStruct[j], cpInfoP); + cpInfoP = getCPInfoToUpdateForUpdatableType(colExtentsStruct[j], cpInfoP, m_opType); pointersToInfos.push_back(cpInfoP); - if (colExtentsStruct[j].tokenFlag) - continue; - width = colOp->getCorrectRowWidth(colExtentsStruct[j].colDataType, colExtentsStruct[j].colWidth); successFlag = colOp->calculateRowId(aRid, BYTE_PER_BLOCK / width, width, curFbo, curBio); @@ -4964,7 +5009,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const CSCTypesList& c allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( - colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL); + colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL, m_opType); if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. { @@ -5109,7 +5154,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const CSCTypesList& c } ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( - newColStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[1] : NULL); + newColStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[1] : NULL, m_opType); allocateValArray(valArray, totalRow2, newColStructList[i].colType, newColStructList[i].colWidth); if (m_opType != INSERT && cpInfo != NULL) // we allocate space for old values only when we need them. @@ -5190,7 +5235,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const CSCTypesList& c ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; ExtCPInfo* cpInfo = getCPInfoToUpdateForUpdatableType( - colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL); + colStructList[i], maxMins ? ((*maxMins)[i]).fSplitMaxMinInfoPtrs[0] : NULL, m_opType); // set params colOp->initColumn(curCol); From ba0306e5ce8a05b1c5205e6d82238c41f14f7546 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Wed, 16 Mar 2022 15:43:02 +0000 Subject: [PATCH 24/55] Fix build with Server 10.7 and newer. Its kinda hack, but works, can be reverted, when server fix thier code --- utils/common/collation.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/common/collation.h b/utils/common/collation.h index 951a2e03c..5d43df31f 100644 --- a/utils/common/collation.h +++ b/utils/common/collation.h @@ -88,6 +88,8 @@ typedef double pfloat; /* Mixed prototypes can't take float */ typedef const struct charset_info_st CHARSET_INFO; extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO* default_charset_info; +#define HAVE_PSI_INTERFACE + #include "m_ctype.h" #undef FALSE From 14c4840d53097a77cb7e5ce67d8c399f217fe32c Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Mon, 21 Mar 2022 09:54:39 -0500 Subject: [PATCH 25/55] MCOL-4576: remove S3 options from cpimport. (#2307) --- writeengine/bulk/cpimport.cpp | 17 ++++++++--------- writeengine/splitter/we_cmdargs.cpp | 11 +++-------- writeengine/splitter/we_filereadthread.cpp | 9 +++++++++ 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/writeengine/bulk/cpimport.cpp b/writeengine/bulk/cpimport.cpp index 4b1fbac8d..5e934c1a3 100644 --- a/writeengine/bulk/cpimport.cpp +++ b/writeengine/bulk/cpimport.cpp @@ -172,12 +172,11 @@ void printUsage() << " -T Timezone used for TIMESTAMP datatype" << endl << " Possible values: \"SYSTEM\" (default)" << endl << " : Offset in the form +/-HH:MM" << endl - << endl - << " -y S3 Authentication Key (for S3 imports)" << endl - << " -K S3 Authentication Secret (for S3 imports)" << endl - << " -t S3 Bucket (for S3 imports)" << endl - << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl - << " -g S3 Regions (for S3 imports)" << endl +// << " -y S3 Authentication Key (for S3 imports)" << endl +// << " -K S3 Authentication Secret (for S3 imports)" << endl +// << " -t S3 Bucket (for S3 imports)" << endl +// << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl +// << " -g S3 Regions (for S3 imports)" << endl << " -U username of new data files owner. Default is mysql" << endl; cout << " Example1:" << endl @@ -310,7 +309,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob BulkModeType bulkMode = BULK_MODE_LOCAL; std::string jobUUID; - while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:U:")) != + while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:U:")) != EOF) { switch (option) @@ -677,7 +676,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob break; } - case 'y': +/* case 'y': { curJob.setS3Key(optarg); break; @@ -706,7 +705,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob curJob.setS3Region(optarg); break; } - +*/ case 'U': { curJob.setUsername(optarg); diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index 27efb1d37..5d6adac51 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -561,11 +561,6 @@ void WECmdArgs::usage() << "\t-T\tTimezone used for TIMESTAMP datatype.\n" << "\t\tPossible values: \"SYSTEM\" (default)\n" << "\t\t : Offset in the form +/-HH:MM\n" - << "\t-y\tS3 Authentication Key (for S3 imports)\n" - << "\t-K\tS3 Authentication Secret (for S3 imports)\n" - << "\t-t\tS3 Bucket (for S3 imports)\n" - << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" - << "\t-g\tS3 Region (for S3 imports)\n" << "\t-L\tDirectory for the output .err and .bad files.\n" << "\t\tDefault is " << string(MCSLOGDIR); @@ -598,7 +593,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) if (argc > 0) fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; // argv[0] is splitter but we need cpimport - while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:")) != EOF) + while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:N:U:L:")) != EOF) { switch (aCh) { @@ -906,7 +901,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fConsoleOutput = false; break; } - +/* case 'y': //-y S3 Key { fS3Key = optarg; @@ -936,7 +931,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fS3Region = optarg; break; } - +*/ case 'U': //-U username of the files owner { fUsername = optarg; diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index a21d5ef1f..eb975cd26 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -481,6 +481,15 @@ void WEFileReadThread::openInFile() use ms3 lib to d/l data into mem use boost::iostreams to wrap the mem in a stream interface point infile's stream buffer to it. + MCOL-4576: The options to setup S3 with cpimport have been removed and this + code is unreachable. However we may need to resurrect it at some point in some form. + Performance issues with extremely large data files as well as the fact files larger + than system memory will cause an OOM error. Multipart downloads/uploads need to be + implemented or more likely a different streaming solution developed with external API tools + + MCOL-4576 work around is to use 3rd party CLI tools and pipe data file from S3 bucket + into cpimport stdin. 3rd party tooling for large object downloads will be more efficient. + */ if (fSdh.getDebugLvl()) From 29679e91ecefc22a45372fac159b48f0c3a2a10c Mon Sep 17 00:00:00 2001 From: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com> Date: Mon, 21 Mar 2022 21:19:55 +0300 Subject: [PATCH 26/55] Clang warnfixes (#2310) --- dbcon/joblist/columncommand-jl.h | 14 +++++++------- primitives/linux-port/column.cpp | 10 ---------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/dbcon/joblist/columncommand-jl.h b/dbcon/joblist/columncommand-jl.h index 4a864a253..7f700766e 100644 --- a/dbcon/joblist/columncommand-jl.h +++ b/dbcon/joblist/columncommand-jl.h @@ -45,13 +45,13 @@ class ColumnCommandJL : public CommandJL ColumnCommandJL(const ColumnCommandJL&, const DictStepJL&); virtual ~ColumnCommandJL(); - virtual void createCommand(messageqcpp::ByteStream& bs) const; - virtual void runCommand(messageqcpp::ByteStream& bs) const; - void setLBID(uint64_t rid, uint32_t dbroot); - uint8_t getTableColumnType(); - virtual std::string toString(); - uint16_t getWidth(); - CommandType getCommandType() + virtual void createCommand(messageqcpp::ByteStream& bs) const override; + virtual void runCommand(messageqcpp::ByteStream& bs) const override; + void setLBID(uint64_t rid, uint32_t dbroot) override; + uint8_t getTableColumnType() override; + virtual std::string toString() override; + uint16_t getWidth() override; + CommandType getCommandType() override { return COLUMN_COMMAND; } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index ff74406a9..974a277da 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -539,16 +539,6 @@ inline bool isNullValue(const T val, const T NULL_VALUE) return val == NULL_VALUE; } -template <> -inline bool isNullValue(const int64_t val, const int64_t NULL_VALUE) -{ - //@bug 339 might be a token here - // TODO: what's up with the alternative NULL here? - constexpr const int64_t ALT_NULL_VALUE = 0xFFFFFFFFFFFFFFFELL; - - return (val == NULL_VALUE || val == ALT_NULL_VALUE); -} - // // FILTER A COLUMN VALUE // From 42f0dae81bf17fc6776a52fa437e4636d9c442aa Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Tue, 22 Mar 2022 11:54:17 +0000 Subject: [PATCH 27/55] Current branch was not always correct --- build/bootstrap_mcs.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index e7f90f73a..2e6381562 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -62,7 +62,10 @@ select_branch() git config submodule.storage/columnstore/columnstore.update none cd - fi + + cd $SCRIPT_LOCATION CURRENT_BRANCH=$(git branch --show-current) + cd - message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch" } From 2b4946f53a9242e0df54166a4b46deb237f88d24 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 23 Mar 2022 12:33:23 +0000 Subject: [PATCH 28/55] Revert "MCOL-4576: remove S3 options from cpimport. (#2307)" This reverts commit 14c4840d53097a77cb7e5ce67d8c399f217fe32c. --- writeengine/bulk/cpimport.cpp | 17 +++++++++-------- writeengine/splitter/we_cmdargs.cpp | 11 ++++++++--- writeengine/splitter/we_filereadthread.cpp | 9 --------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/writeengine/bulk/cpimport.cpp b/writeengine/bulk/cpimport.cpp index 5e934c1a3..4b1fbac8d 100644 --- a/writeengine/bulk/cpimport.cpp +++ b/writeengine/bulk/cpimport.cpp @@ -172,11 +172,12 @@ void printUsage() << " -T Timezone used for TIMESTAMP datatype" << endl << " Possible values: \"SYSTEM\" (default)" << endl << " : Offset in the form +/-HH:MM" << endl -// << " -y S3 Authentication Key (for S3 imports)" << endl -// << " -K S3 Authentication Secret (for S3 imports)" << endl -// << " -t S3 Bucket (for S3 imports)" << endl -// << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl -// << " -g S3 Regions (for S3 imports)" << endl + << endl + << " -y S3 Authentication Key (for S3 imports)" << endl + << " -K S3 Authentication Secret (for S3 imports)" << endl + << " -t S3 Bucket (for S3 imports)" << endl + << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl + << " -g S3 Regions (for S3 imports)" << endl << " -U username of new data files owner. Default is mysql" << endl; cout << " Example1:" << endl @@ -309,7 +310,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob BulkModeType bulkMode = BULK_MODE_LOCAL; std::string jobUUID; - while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:U:")) != + while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:U:")) != EOF) { switch (option) @@ -676,7 +677,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob break; } -/* case 'y': + case 'y': { curJob.setS3Key(optarg); break; @@ -705,7 +706,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob curJob.setS3Region(optarg); break; } -*/ + case 'U': { curJob.setUsername(optarg); diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index 5d6adac51..27efb1d37 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -561,6 +561,11 @@ void WECmdArgs::usage() << "\t-T\tTimezone used for TIMESTAMP datatype.\n" << "\t\tPossible values: \"SYSTEM\" (default)\n" << "\t\t : Offset in the form +/-HH:MM\n" + << "\t-y\tS3 Authentication Key (for S3 imports)\n" + << "\t-K\tS3 Authentication Secret (for S3 imports)\n" + << "\t-t\tS3 Bucket (for S3 imports)\n" + << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" + << "\t-g\tS3 Region (for S3 imports)\n" << "\t-L\tDirectory for the output .err and .bad files.\n" << "\t\tDefault is " << string(MCSLOGDIR); @@ -593,7 +598,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) if (argc > 0) fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; // argv[0] is splitter but we need cpimport - while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:N:U:L:")) != EOF) + while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:")) != EOF) { switch (aCh) { @@ -901,7 +906,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fConsoleOutput = false; break; } -/* + case 'y': //-y S3 Key { fS3Key = optarg; @@ -931,7 +936,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fS3Region = optarg; break; } -*/ + case 'U': //-U username of the files owner { fUsername = optarg; diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index eb975cd26..a21d5ef1f 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -481,15 +481,6 @@ void WEFileReadThread::openInFile() use ms3 lib to d/l data into mem use boost::iostreams to wrap the mem in a stream interface point infile's stream buffer to it. - MCOL-4576: The options to setup S3 with cpimport have been removed and this - code is unreachable. However we may need to resurrect it at some point in some form. - Performance issues with extremely large data files as well as the fact files larger - than system memory will cause an OOM error. Multipart downloads/uploads need to be - implemented or more likely a different streaming solution developed with external API tools - - MCOL-4576 work around is to use 3rd party CLI tools and pipe data file from S3 bucket - into cpimport stdin. 3rd party tooling for large object downloads will be more efficient. - */ if (fSdh.getDebugLvl()) From fbd043b0360506b700ecfdb50590c4e0aa21b268 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Tue, 22 Mar 2022 13:50:34 +0000 Subject: [PATCH 29/55] Fixing alightment for clang tests of rowgroup --- datatypes/mcs_decimal.h | 8 +++--- tests/rowgroup-tests.cpp | 55 +++++++++++++++++++------------------ utils/rowgroup/rowgroup.cpp | 3 +- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 621e0aed4..91bdbe55f 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -318,12 +318,12 @@ class TDecimal128 : public TSInt128 static constexpr int128_t minInt128 = TFloat128::minInt128; static constexpr int128_t maxInt128 = TFloat128::maxInt128; - static inline bool isWideDecimalNullValue(const int128_t& val) + static inline bool isWideDecimalNullValue(const int128_t val) { return (val == TSInt128::NullValue); } - static inline bool isWideDecimalEmptyValue(const int128_t& val) + static inline bool isWideDecimalEmptyValue(const int128_t val) { return (val == TSInt128::EmptyValue); } @@ -342,10 +342,10 @@ class TDecimal128 : public TSInt128 TDecimal128() { } - explicit TDecimal128(const int128_t& val) : TSInt128(val) + explicit TDecimal128(const int128_t val) : TSInt128(val) { } - explicit TDecimal128(const TSInt128& val) : TSInt128(val) + explicit TDecimal128(const TSInt128 val) : TSInt128(val) { } explicit TDecimal128(const int128_t* valPtr) : TSInt128(valPtr) diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index b1c6d5fa5..5073702ae 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -27,6 +27,7 @@ #define INITIAL_ROW_OFFSET 2 using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; +using datatypes::TSInt128; class RowDecimalTest : public ::testing::Test { @@ -196,15 +197,15 @@ TEST_F(RowDecimalTest, InitToNullAndIsNullValueCheck) TEST_F(RowDecimalTest, GetBinaryFieldCheck) { rg.getRow(0, &r); - int128_t* a128Value; - int128_t* s128Value; + TSInt128 a128Value; + TSInt128 s128Value; for (size_t i = 0; i < sValueVector.size(); i++) { - s128Value = r.getBinaryField(0); - EXPECT_EQ(sValueVector[i], *s128Value); - a128Value = r.getBinaryField(1); - EXPECT_EQ(anotherValueVector[i], *a128Value); + s128Value = r.getTSInt128Field(0); + EXPECT_EQ(sValueVector[i], s128Value.getValue()); + a128Value = r.getTSInt128Field(1); + EXPECT_EQ(anotherValueVector[i], a128Value.getValue()); // EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); // EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); // EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); @@ -245,22 +246,24 @@ TEST_F(RowDecimalTest, ApplyMappingCheck) int mapping[] = {0, 1, -1, -1, -1, -1}; rg.getRow(1, &r); rg.getRow(2, &rOutMappingCheck); - int128_t* s128Value = rOutMappingCheck.getBinaryField(0); - int128_t* a128Value = rOutMappingCheck.getBinaryField(1); - EXPECT_NE(sValueVector[1], *s128Value); - EXPECT_NE(anotherValueVector[1], *a128Value); + + auto s128Value = rOutMappingCheck.getTSInt128Field(0); + auto a128Value = rOutMappingCheck.getTSInt128Field(1); + EXPECT_NE(s128Value.getValue(), sValueVector[1]); + EXPECT_NE(a128Value.getValue(), anotherValueVector[1]); + applyMapping(mapping, r, &rOutMappingCheck); - s128Value = rOutMappingCheck.getBinaryField(0); - a128Value = rOutMappingCheck.getBinaryField(1); - EXPECT_EQ(sValueVector[1], *s128Value); - EXPECT_EQ(anotherValueVector[1], *a128Value); + s128Value = rOutMappingCheck.getTSInt128Field(0); + a128Value = rOutMappingCheck.getTSInt128Field(1); + EXPECT_EQ(sValueVector[1], s128Value.getValue()); + EXPECT_EQ(anotherValueVector[1], a128Value.getValue()); } TEST_F(RowDecimalTest, CopyBinaryFieldCheck) { int128_t constVal = 1; - int128_t *col1Out, *col1In; - int128_t *col2Out, *col2In; + TSInt128 col1Out, col1In; + TSInt128 col2Out, col2In; rgOut.getRow(0, &rOut); for (size_t i = 0; i < sValueVector.size(); i++) @@ -276,18 +279,18 @@ TEST_F(RowDecimalTest, CopyBinaryFieldCheck) for (size_t i = 0; i < sValueVector.size(); i++) { - col1In = r.getBinaryField(0); - col1Out = rOut.getBinaryField(0); - col2In = r.getBinaryField(1); - col2Out = rOut.getBinaryField(1); - EXPECT_NE(*col1In, *col1Out); - EXPECT_NE(*col2In, *col2Out); + col1In = r.getTSInt128Field(0); + col1Out = rOut.getTSInt128Field(0); + col2In = r.getTSInt128Field(1); + col2Out = rOut.getTSInt128Field(1); + EXPECT_NE(col1In, col1Out); + EXPECT_NE(col2In, col2Out); r.copyBinaryField(rOut, 0, 0); r.copyBinaryField(rOut, 1, 1); - col1Out = rOut.getBinaryField(0); - col2Out = rOut.getBinaryField(1); - EXPECT_EQ(*col1In, *col1Out); - EXPECT_EQ(*col2In, *col2Out); + col1Out = rOut.getTSInt128Field(0); + col2Out = rOut.getTSInt128Field(1); + EXPECT_EQ(col1In.getValue(), col1Out.getValue()); + EXPECT_EQ(col2In.getValue(), col2Out.getValue()); r.nextRow(rowSize); rOut.nextRow(rowSize); } diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 7c297f8fb..7aa41477b 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -800,7 +800,8 @@ template <> inline bool Row::isNullValue_offset(uint32_t offset) const { const int128_t* intPtr = reinterpret_cast(&data[offset]); - return datatypes::Decimal::isWideDecimalNullValue(*intPtr); + const datatypes::TSInt128 value(intPtr); + return datatypes::Decimal::isWideDecimalNullValue(value.getValue()); } template <> From f108fe81ebce1027b0f4c9d9af8a40f4fbb4e169 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Wed, 23 Mar 2022 17:01:15 +0000 Subject: [PATCH 30/55] Bootstrap rockylinux support tested --- build/bootstrap_mcs.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 2e6381562..02ac3477c 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -20,7 +20,7 @@ fi message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") -DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "openSUSE") +DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "openSUSE" "Rocky") BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE @@ -37,7 +37,7 @@ if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then MCS_BUILD_TYPE=$selectedChoice fi -if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "Centos" ]]; then +if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "CentOS" ]]; then detect_distro fi @@ -78,7 +78,7 @@ install_deps() libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev - elif [ $OS = 'CentOS' ]; then + elif [[ $OS = 'CentOS' || $OS = 'Rocky' ]]; then yum -y install epel-release \ && yum -y groupinstall "Development Tools" \ && yum config-manager --set-enabled powertools \ @@ -90,6 +90,9 @@ install_deps() else yum -y install cmake fi + if [ $OS = 'Rocky' ]; then + yum install -y checkpolicy + fi elif [ $OS = 'openSUSE' ]; then zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \ && zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel \ @@ -183,6 +186,8 @@ build() MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" elif [ $OS = 'CentOS' ]; then MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" + elif [ $OS = 'Rocky' ]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" elif [ $OS = 'openSUSE' ]; then MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15" fi From ed97d73112fcab76cdeb12b8d4d1c2801aee23dc Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Thu, 24 Mar 2022 19:56:23 +0300 Subject: [PATCH 31/55] Use clang for openSuse --- .drone.jsonnet | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 69cea1e4e..a1e99830f 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -45,13 +45,14 @@ local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yu local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; local ubuntu18_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; local debian10_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; -local opensuse_build_deps = 'zypper install -y liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + gcc_update_alternatives; +local opensuse_build_deps = 'zypper install -y clang12 liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + clang12_update_alternatives; local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 git && ' + gcc_update_alternatives; -local platformMap(platform) = +local platformMap(platform, arch) = + local clang_force = if (arch == 'arm64') then " && export CXX=/usr/bin/clang++ && export CC=/usr/bin/clang " else ''; local platform_map = { - 'opensuse/leap:15': opensuse_build_deps + ' && zypper ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=sles15 && make -j$(nproc) package', + 'opensuse/leap:15': opensuse_build_deps + ' && zypper ' + rpm_build_deps + clang_force + ' && cmake ' + cmakeflags + ' -DRPM=sles15 && make -j$(nproc) package', 'centos:7': centos7_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos7 && make -j$(nproc) package', 'centos:8': centos8_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos8 && make -j$(nproc) package', 'debian:10': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=buster' debian/autobake-deb.sh", @@ -399,7 +400,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { // Workaround till upstream removes 4535 workaround (workaround for workaround!) "sed -i '/MCOL-4535/,/^$/d' debian/autobake-deb.sh", testPreparation(platform), - platformMap(platform), + platformMap(platform, arch), if (pkg_format == 'rpm') then 'createrepo .' else 'dpkg-scanpackages ../ | gzip > ../Packages.gz', ], }, From c847f6ce25eb9bbc0997db775994220f9c0751f5 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Fri, 25 Mar 2022 13:08:58 +0000 Subject: [PATCH 32/55] Fix segfault for vector scan tests on clang --- tests/primitives_column_scan_and_filter.cpp | 93 +++++++++++++++------ 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/tests/primitives_column_scan_and_filter.cpp b/tests/primitives_column_scan_and_filter.cpp index eeed651ee..f017892d0 100644 --- a/tests/primitives_column_scan_and_filter.cpp +++ b/tests/primitives_column_scan_and_filter.cpp @@ -15,9 +15,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include #include #include +#include "mcs_basic_types.h" #include "utils/common/columnwidth.h" #include "datatypes/mcs_datatype.h" #include "datatypes/mcs_int128.h" @@ -175,8 +177,10 @@ TEST_F(ColumnScanFilterTest, ColumnScan1ByteVectorized) for (i = 8034; i < 8160; ++i) EXPECT_EQ(results[i], i % 255 + 1); - EXPECT_EQ(out->Max, __col1block_cdf_umax); - EXPECT_EQ(out->Min, __col1block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + EXPECT_EQ(expectedMax.getValue(), __col1block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col1block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan2Bytes) @@ -200,8 +204,10 @@ TEST_F(ColumnScanFilterTest, ColumnScan2Bytes) for (i = 0; i < out->NVALS; i++) EXPECT_EQ(results[i], i); - EXPECT_EQ(out->Max, __col2block_cdf_umax); - EXPECT_EQ(out->Min, __col2block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + EXPECT_EQ(expectedMax.getValue(), __col2block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col2block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan4Bytes) @@ -224,8 +230,12 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes) for (i = 0; i < out->NVALS; i++) EXPECT_EQ(results[i], (uint32_t)i); - EXPECT_EQ(out->Max, __col4block_cdf_umax); - EXPECT_EQ(out->Min, __col4block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col4block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col4block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes) @@ -248,8 +258,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes) for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], (uint32_t)i); - EXPECT_EQ(out->Max, __col8block_cdf_umax); - EXPECT_EQ(out->Min, __col8block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + EXPECT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan2Bytes1EqFilter) @@ -280,8 +293,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan2Bytes1EqFilter) ASSERT_EQ(out->NVALS, 51); for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], i); - EXPECT_EQ(out->Max, __col2block_cdf_umax); - EXPECT_EQ(out->Min, __col2block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + EXPECT_EQ(expectedMax.getValue(), __col2block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col2block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan1ByteUsingRID) @@ -432,8 +448,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes2Filters) ASSERT_EQ(results[i], 11 + (uint32_t)i); } - EXPECT_EQ(out->Max, __col4block_cdf_umax); - EXPECT_EQ(out->Min, __col4block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col4block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col4block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes1EqFilter) @@ -465,8 +484,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes1EqFilter) for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], i); - EXPECT_EQ(out->Max, __col8block_cdf_umax); - EXPECT_EQ(out->Min, __col8block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8BytesUsingMultipleRIDs) @@ -533,8 +555,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2CompFilters) for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], (uint32_t)(i < 10 ? i : i - 10 + 1001)); - EXPECT_EQ(out->Max, __col8block_cdf_umax); - EXPECT_EQ(out->Min, __col8block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFilters) @@ -569,8 +594,10 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFilters) ASSERT_EQ(out->NVALS, 2); ASSERT_EQ(results[0], 10); ASSERT_EQ(results[1], 1000); - ASSERT_EQ(out->Max, __col8block_cdf_umax); - ASSERT_EQ(out->Min, __col8block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + ASSERT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + ASSERT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRID) @@ -642,8 +669,11 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2FiltersRIDOutputRid) for (i = 0; i < out->NVALS; i++) ASSERT_EQ(results[i], (i < 10 ? i : i - 10 + 1001)); - ASSERT_EQ(out->Max, __col8block_cdf_umax); - ASSERT_EQ(out->Min, __col8block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + ASSERT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + ASSERT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRIDOutputBoth) @@ -680,8 +710,12 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRIDOutputBoth) ASSERT_EQ(resultRid[i], (i < 10 ? i : i - 10 + 1001)); ASSERT_EQ(resultVal[i], (i < 10 ? i : i - 10 + 1001)); } - ASSERT_EQ(out->Max, __col8block_cdf_umax); - ASSERT_EQ(out->Min, __col8block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + ASSERT_EQ(expectedMax.getValue(), __col8block_cdf_umax); + ASSERT_EQ(expectedMin.getValue(), __col8block_cdf_umin); } // void p_Col_12() @@ -926,8 +960,12 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes) ASSERT_EQ(results[0], NullValue); for (i = 1; i < out->NVALS; ++i) ASSERT_EQ(results[i], i + 1); - EXPECT_EQ(out->Max, __col16block_cdf_umax); - EXPECT_EQ(out->Min, __col16block_cdf_umin); + + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col16block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col16block_cdf_umin); } TEST_F(ColumnScanFilterTest, ColumnScan16Bytes2CompFilters) @@ -962,6 +1000,9 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes2CompFilters) ASSERT_EQ(results[0], 10); ASSERT_EQ(results[1], 510); - EXPECT_EQ(out->Max, __col16block_cdf_umax); - EXPECT_EQ(out->Min, __col16block_cdf_umin); + datatypes::TSInt128 expectedMax(&(out->Max)); + datatypes::TSInt128 expectedMin(&(out->Min)); + + EXPECT_EQ(expectedMax.getValue(), __col16block_cdf_umax); + EXPECT_EQ(expectedMin.getValue(), __col16block_cdf_umin); } From 7c294ca30cdb5b731fff7788adf385e953e29f15 Mon Sep 17 00:00:00 2001 From: mariadb-RomanNavrotskiy Date: Sun, 27 Mar 2022 23:30:32 +0200 Subject: [PATCH 33/55] ci: centos8 stream, rockylinux, deb upstream fmt dependency --- .drone.jsonnet | 54 ++++++++--------- debian/autobake-deb.sh | 130 ----------------------------------------- 2 files changed, 23 insertions(+), 161 deletions(-) delete mode 100755 debian/autobake-deb.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index a1e99830f..96d70316e 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,27 +1,24 @@ local events = ['pull_request', 'cron']; local platforms = { - develop: ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], - 'develop-7': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], - 'develop-6': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], - 'develop-5': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], + develop: ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], + 'develop-6': ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], + 'develop-5': ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], }; local platforms_arm = { - develop: ['centos:8'], - 'develop-7': ['centos:8'], - 'develop-6': ['centos:8'], + develop: ['rockylinux:8'], + 'develop-6': ['rockylinux:8'], }; local any_branch = '**'; -local platforms_custom = ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04']; +local platforms_custom = ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04']; local platforms_arm_custom = ['centos:8']; local platforms_mtr = ['centos:7', 'centos:8', 'ubuntu:20.04']; local server_ref_map = { develop: '10.8', - 'develop-7': '10.7', 'develop-6': '10.6-enterprise', 'develop-5': '10.5', '**': '10.8', @@ -39,10 +36,10 @@ local gcc_update_alternatives = 'update-alternatives --install /usr/bin/gcc gcc local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 '; -local yum_vault_mirror = "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-*; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-*; dnf update -y; "; local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo '; local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; -local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; +local centos8_build_deps = 'dnf install -y gcc-toolset-10 libarchive cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable '; +local rockylinux8_powertools = "dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools "; local ubuntu18_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; local debian10_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; local opensuse_build_deps = 'zypper install -y clang12 liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + clang12_update_alternatives; @@ -50,11 +47,12 @@ local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recom local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 git && ' + gcc_update_alternatives; local platformMap(platform, arch) = - local clang_force = if (arch == 'arm64') then " && export CXX=/usr/bin/clang++ && export CC=/usr/bin/clang " else ''; + local clang_force = if (arch == 'arm64') then ' && export CXX=/usr/bin/clang++ && export CC=/usr/bin/clang ' else ''; local platform_map = { 'opensuse/leap:15': opensuse_build_deps + ' && zypper ' + rpm_build_deps + clang_force + ' && cmake ' + cmakeflags + ' -DRPM=sles15 && make -j$(nproc) package', 'centos:7': centos7_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos7 && make -j$(nproc) package', - 'centos:8': centos8_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos8 && make -j$(nproc) package', + 'centos:8': centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos8 && make -j$(nproc) package', + 'rockylinux:8': rockylinux8_powertools + ' && ' + centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && make -j$(nproc) package', 'debian:10': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=buster' debian/autobake-deb.sh", 'ubuntu:18.04': ubuntu18_04_deps + ' && ' + deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=bionic' debian/autobake-deb.sh", 'ubuntu:20.04': ubuntu20_04_deps + ' && ' + deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh", @@ -67,6 +65,7 @@ local testRun(platform) = 'opensuse/leap:15': 'ctest -R columnstore: -j $(nproc) --output-on-failure', 'centos:7': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', 'centos:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', + 'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', 'debian:10': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', 'ubuntu:18.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', 'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', @@ -78,7 +77,8 @@ local testPreparation(platform) = local platform_map = { 'opensuse/leap:15': 'zypper install -y git boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake', 'centos:7': 'yum -y install epel-release && yum install -y git cppunit-devel cmake3 boost-devel snappy-devel', - 'centos:8': yum_vault_mirror + ' yum install -y dnf-plugins-core libarchive && yum config-manager --set-enabled powertools && yum install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', + 'centos:8': 'dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', + 'rockylinux:8': rockylinux8_powertools + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', 'debian:10': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', 'ubuntu:18.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake g++', 'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', @@ -87,12 +87,12 @@ local testPreparation(platform) = local Pipeline(branch, platform, event, arch='amd64') = { - local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'opensuse/leap') then 'rpm' else 'deb', + local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'opensuse/leap' || std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb', local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd', local mtr_path = if (pkg_format == 'rpm') then '/usr/share/mysql-test' else '/usr/share/mysql/mysql-test', local socket_path = if (pkg_format == 'rpm') then '/var/lib/mysql/mysql.sock' else '/run/mysqld/mysqld.sock', local config_path_prefix = if (pkg_format == 'rpm') then '/etc/my.cnf.d/' else '/etc/mysql/mariadb.conf.d/50-', - local img = if (std.split(platform, ':')[0] == 'centos') then platform else 'romcheck/' + std.strReplace(platform, '/', '-'), + local img = if (platform == 'centos:7' || std.split(platform, ':')[0] == 'rockylinux') then platform else 'romcheck/' + std.strReplace(platform, '/', '-'), local regression_ref = if (std.split(branch, '-')[0] == 'develop') then branch else 'develop-6', local branchp = if (branch == '**') then '' else branch, @@ -136,9 +136,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { commands: [ 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker cp result smoke$${DRONE_BUILD_NUMBER}:/', - if (platform == 'centos:8') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "' + yum_vault_mirror + '"' else '', - if (std.split(platform, ':')[0] == 'centos') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release which rsyslog hostname && yum install -y /result/*.' + pkg_format + '"' else '', - if (platform == 'centos:8') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "sed -i s/enabled=0/enabled=1/ /etc/yum.repos.d/*PowerTools.repo && yum install -y gtest"' else '', + if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release which rsyslog hostname procps-ng && yum install -y /result/*.' + pkg_format + '"' else '', if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname && apt install -y -f /result/*.' + pkg_format + '"' else '', if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '', @@ -161,9 +159,8 @@ local Pipeline(branch, platform, event, arch='amd64') = { commands: [ 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env MYSQL_TEST_DIR=' + mtr_path + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker cp result mtr$${DRONE_BUILD_NUMBER}:/', - if (platform == 'centos:8') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "' + yum_vault_mirror + '"' else '', if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog patch perl-Data-Dumper-Concise perl-Memoize-ExpireLRU && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '', - if (std.split(platform, ':')[0] == 'centos') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils which rsyslog hostname patch perl-Data-Dumper perl-Getopt-Long perl-Memoize perl-Time-HiRes cracklib-dicts && yum install -y /result/*.' + pkg_format + '"' else '', + if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils which rsyslog hostname patch perl-Data-Dumper perl-Getopt-Long perl-Memoize perl-Time-HiRes cracklib-dicts procps-ng && yum install -y /result/*.' + pkg_format + '"' else '', if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname patch && apt install -y -f /result/*.' + pkg_format + '"' else '', 'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/', @@ -224,8 +221,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { 'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/', // check storage-manager unit test binary file 'docker exec -t regression$${DRONE_BUILD_NUMBER} ls -l /storage-manager', - if (platform == 'centos:8') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "' + yum_vault_mirror + '"' else '', - if (std.split(platform, ':')[0] == 'centos') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils tar lz4 wget which rsyslog hostname && yum install -y /result/*.' + pkg_format + '"' else '', + if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils tar lz4 wget which rsyslog hostname procps-ng && yum install -y /result/*.' + pkg_format + '"' else '', if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y tar liblz4-tool wget rsyslog hostname && apt install -y -f /result/*.' + pkg_format + '"' else '', if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "zypper install -y gzip tar lz4 wget which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '', @@ -362,7 +358,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { }, { name: 'build', - image: platform, + image: img, volumes: [pipeline._volumes.mdb], environment: { DEBIAN_FRONTEND: 'noninteractive', @@ -391,14 +387,10 @@ local Pipeline(branch, platform, event, arch='amd64') = { // Leave test package for mtr "sed -i '/(mariadb|mysql)-test/d;/-test/d' debian/autobake-deb.sh", "sed -i '/test-embedded/d' debian/mariadb-test.install", - // From Debian Bullseye/Ubuntu Groovy, liburing replaces libaio - "apt-cache madison liburing-dev | grep 'liburing-dev' || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules", - // From Debian Buster/Ubuntu Focal onwards libpmem-dev is available - "apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules", + // Deb dependencies from server scripts + if (pkg_format == 'deb') then "apt-cache madison liburing-dev | grep liburing-dev || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules && apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules && sed '/libfmt-dev/d' -i debian/control" else '', // Change plugin_maturity level // "sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt", - // Workaround till upstream removes 4535 workaround (workaround for workaround!) - "sed -i '/MCOL-4535/,/^$/d' debian/autobake-deb.sh", testPreparation(platform), platformMap(platform, arch), if (pkg_format == 'rpm') then 'createrepo .' else 'dpkg-scanpackages ../ | gzip > ../Packages.gz', @@ -406,7 +398,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { }, { name: 'unittests', - image: platform, + image: img, volumes: [pipeline._volumes.mdb], environment: { DEBIAN_FRONTEND: 'noninteractive', diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh deleted file mode 100755 index 75df5e2bb..000000000 --- a/debian/autobake-deb.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# -# Build MariaDB .deb packages for test and release at mariadb.org -# -# Purpose of this script: -# Always keep the actual packaging as up-to-date as possible following the latest -# Debian policy and targeting Debian Sid. Then case-by-case run in autobake-deb.sh -# tests for backwards compatibility and strip away parts on older builders or -# specfic build environments. - -# Exit immediately on any error -set -e - -# This file is invocated from Buildbot and Travis-CI to build deb packages. -# As both of those CI systems have many parallel jobs that include different -# parts of the test suite, we don't need to run the mysql-test-run at all when -# building the deb packages here. -export DEB_BUILD_OPTIONS="nocheck $DEB_BUILD_OPTIONS" - -if [[ -d storage/columnstore/columnstore/debian ]]; then - cp -v storage/columnstore/columnstore/debian/mariadb-plugin-columnstore.* debian/ - echo >> debian/control - cat storage/columnstore/columnstore/debian/control >> debian/control -fi - -# General CI optimizations to keep build output smaller -if [[ $TRAVIS ]] || [[ $GITLAB_CI ]] -then - # On both Travis and Gitlab the output log must stay under 4MB so make the - # build less verbose - sed '/Add support for verbose builds/,/^$/d' -i debian/rules - - # MCOL-4149: ColumnStore builds are so slow and big that they must be skipped on - # both Travis-CI and Gitlab-CI - sed 's|-DPLUGIN_COLUMNSTORE=YES|-DPLUGIN_COLUMNSTORE=NO|' -i debian/rules - sed "/Package: mariadb-plugin-columnstore/,/^$/d" -i debian/control -fi - -# Don't build or try to put files in a package for selected plugins and compontents on Travis-CI -# in order to keep build small (in both duration and disk space) -if [[ $TRAVIS ]] -then - # Test suite package not relevant on Travis-CI - sed 's|DINSTALL_MYSQLTESTDIR=share/mysql/mysql-test|DINSTALL_MYSQLTESTDIR=false|' -i debian/rules - sed '/Package: mariadb-test-data/,/^$/d' -i debian/control - sed '/Package: mariadb-test$/,/^$/d' -i debian/control - - # Extra plugins such as Mroonga, Spider, OQgraph, Sphinx and the embedded build can safely be skipped - sed 's|-DDEB|-DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_SPHINX=NO -DWITH_EMBEDDED_SERVER=OFF -DDEB|' -i debian/rules - sed "/Package: mariadb-plugin-mroonga/,/^$/d" -i debian/control - sed "/Package: mariadb-plugin-rocksdb/,/^$/d" -i debian/control - sed "/Package: mariadb-plugin-spider/,/^$/d" -i debian/control - sed "/Package: mariadb-plugin-oqgraph/,/^$/d" -i debian/control - sed "/ha_sphinx.so/d" -i debian/mariadb-server-10.*.install - sed "/Package: libmariadbd19/,/^$/d" -i debian/control - sed "/Package: libmariadbd-dev/,/^$/d" -i debian/control -fi - -if [[ $(arch) =~ i[346]86 ]] -then - sed "/Package: mariadb-plugin-rocksdb/,/^$/d" -i debian/control -fi - -# If rocksdb-tools is not available (before Debian Buster and Ubuntu Disco) -# remove the dependency from the RocksDB plugin so it can install properly -# and instead ship the one built from MariaDB sources -if ! apt-cache madison rocksdb-tools | grep 'rocksdb-tools' >/dev/null 2>&1 -then - sed '/rocksdb-tools/d' -i debian/control - sed '/sst_dump/d' -i debian/not-installed - echo "usr/bin/sst_dump" >> debian/mariadb-plugin-rocksdb.install -fi - -# From Debian Buster/Ubuntu Bionic, libcurl4 replaces libcurl3. -if ! apt-cache madison libcurl4 | grep 'libcurl4' >/dev/null 2>&1 -then - sed 's/libcurl4/libcurl3/g' -i debian/control -fi - -# Adjust changelog, add new version -echo "Incrementing changelog and starting build scripts" - -# Find major.minor version -source ./VERSION -UPSTREAM="${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}" -PATCHLEVEL="+maria" -LOGSTRING="MariaDB build" -CODENAME="$(lsb_release -sc)" -EPOCH="1:" - -dch -b -D "${CODENAME}" -v "${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automatic build with ${LOGSTRING}." - -echo "Creating package version ${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME} ... " - -# On Travis CI and Gitlab-CI, use -b to build binary only packages as there is -# no need to waste time on generating the source package. -if [[ $TRAVIS ]] -then - BUILDPACKAGE_FLAGS="-b" -fi - -# Use eatmydata is available to build faster with less I/O, skipping fsync() -# during the entire build process (safe because a build can always be restarted) -if which eatmydata > /dev/null -then - BUILDPACKAGE_PREPEND=eatmydata -fi - -# Build the package -# Pass -I so that .git and other unnecessary temporary and source control files -# will be ignored by dpkg-source when creating the tar.gz source package. -fakeroot $BUILDPACKAGE_PREPEND dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS || sleep 600 - -# If the step above fails due to missing dependencies, you can manually run -# sudo mk-build-deps debian/control -r -i - -# Don't log package contents on Travis-CI or Gitlab-CI to save time and log size -if [[ ! $TRAVIS ]] && [[ ! $GITLAB_CI ]] -then - echo "List package contents ..." - cd .. - for package in *.deb - do - echo "$package" | cut -d '_' -f 1 - dpkg-deb -c "$package" | awk '{print $1 " " $2 " " $6 " " $7 " " $8}' | sort -k 3 - echo "------------------------------------------------" - done -fi - -echo "Build complete" From 65252df4f66c64e7f0ec2cb80c091cf379ecabb6 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Tue, 1 Feb 2022 15:54:05 +0000 Subject: [PATCH 34/55] C++20 fixes --- .drone.jsonnet | 13 +- CMakeLists.txt | 7 +- dbcon/ddlpackageproc/altertableprocessor.cpp | 10 +- dbcon/execplan/predicateoperator.h | 1 - dbcon/execplan/treenode.h | 2 +- dbcon/joblist/jobstep.h | 2 +- dbcon/joblist/primitivestep.h | 4 +- dbcon/joblist/resourcemanager.cpp | 3 - dbcon/mysql/ha_autoi.cpp | 38 +- dbcon/mysql/ha_mcs_ddl.cpp | 34 +- dbcon/mysql/ha_mcs_execplan.cpp | 1 - dbcon/mysql/ha_mcs_impl.cpp | 1 - oamapps/postConfigure/mycnfUpgrade.cpp | 50 +- primitives/linux-port/primitiveprocessor.h | 2 +- primitives/primproc/primproc.cpp | 2 - storage-manager/src/Config.cpp | 28 +- storage-manager/src/IOCoordinator.cpp | 21 +- storage-manager/src/MetadataFile.cpp | 107 +- storage-manager/src/MetadataFile.h | 4 +- storage-manager/src/Replicator.cpp | 16 +- storage-manager/src/S3Storage.cpp | 20 +- tools/passwd/CMakeLists.txt | 2 +- tools/passwd/secrets.cpp | 40 +- utils/common/cgroupconfigurator.cpp | 7 +- utils/common/stlpoolallocator.h | 4 +- utils/dataconvert/dataconvert.h | 9 +- utils/funcexp/func_regexp.cpp | 6 +- utils/funcexp/funchelpers.h | 1 - utils/json/json.hpp | 22109 +++++++++++++++++ utils/loggingcpp/format.h | 92 + utils/loggingcpp/idberrorinfo.cpp | 41 +- utils/loggingcpp/message.cpp | 41 +- utils/rwlock/rwlock.h | 15 +- utils/winport/grepit.cpp | 6 +- writeengine/bulk/we_tableinfo.h | 2 +- writeengine/server/we_dataloader.cpp | 2 +- 36 files changed, 22433 insertions(+), 310 deletions(-) create mode 100644 utils/json/json.hpp create mode 100644 utils/loggingcpp/format.h diff --git a/.drone.jsonnet b/.drone.jsonnet index 96d70316e..c0ebd5e06 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,9 +1,10 @@ local events = ['pull_request', 'cron']; local platforms = { - develop: ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], - 'develop-6': ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], - 'develop-5': ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04'], + develop: ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], + 'develop-6': ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], + 'develop-5': ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], + }; local platforms_arm = { @@ -12,7 +13,7 @@ local platforms_arm = { }; local any_branch = '**'; -local platforms_custom = ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:18.04', 'ubuntu:20.04']; +local platforms_custom = ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:20.04']; local platforms_arm_custom = ['centos:8']; local platforms_mtr = ['centos:7', 'centos:8', 'ubuntu:20.04']; @@ -33,7 +34,7 @@ local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DPLUGIN_COLUMNSTORE=YES - local gcc_update_alternatives = 'update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 '; -local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 '; +local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 '; local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo '; @@ -54,7 +55,6 @@ local platformMap(platform, arch) = 'centos:8': centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos8 && make -j$(nproc) package', 'rockylinux:8': rockylinux8_powertools + ' && ' + centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && make -j$(nproc) package', 'debian:10': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=buster' debian/autobake-deb.sh", - 'ubuntu:18.04': ubuntu18_04_deps + ' && ' + deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=bionic' debian/autobake-deb.sh", 'ubuntu:20.04': ubuntu20_04_deps + ' && ' + deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh", }; platform_map[platform]; @@ -67,7 +67,6 @@ local testRun(platform) = 'centos:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', 'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', 'debian:10': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', - 'ubuntu:18.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', 'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', }; platform_map[platform]; diff --git a/CMakeLists.txt b/CMakeLists.txt index 067f1c7e5..1be5206fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ endif() INCLUDE(ExternalProject) INCLUDE(CheckCXXSourceCompiles) -SET(CMAKE_CXX_STANDARD 17) +SET(CMAKE_CXX_STANDARD 20) SET(CMAKE_CXX_STANDARD_REQUIRED TRUE) SET(CMAKE_CXX_EXTENSIONS FALSE) SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) @@ -214,6 +214,11 @@ IF (MASK_LONGDOUBLE) MY_CHECK_AND_SET_COMPILER_FLAG("-DMASK_LONGDOUBLE") ENDIF() +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-enum-enum-conversion -Wno-register") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-register") +endif() MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall") SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index c09c79b48..86457fb57 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -22,12 +22,12 @@ #include #include +#include #include #include using namespace std; #include -#include #include #include "altertableprocessor.h" @@ -2083,14 +2083,14 @@ void AlterTableProcessor::tableComment(uint32_t sessionID, execplan::CalpontSyst CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); boost::algorithm::to_upper(ataTableComment.fTableComment); - boost::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", boost::regex_constants::extended); - boost::match_results what; + std::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", std::regex_constants::extended); + std::match_results what; std::string::const_iterator start, end; start = ataTableComment.fTableComment.begin(); end = ataTableComment.fTableComment.end(); - boost::match_flag_type flags = boost::match_default; + std::regex_constants::match_flag_type flags = std::regex_constants::match_default; - if (boost::regex_search(start, end, what, compat, flags) && what[0].matched) + if (std::regex_search(start, end, what, compat, flags) && what[0].matched) { std::string params(&(*(what[0].second))); char* ep = NULL; diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index da82e0726..b732a2825 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -36,7 +36,6 @@ #endif #include #include -#include #include "expressionparser.h" #include "returnedcolumn.h" diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index afc0625c9..f8351669b 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -63,7 +63,7 @@ typedef datatypes::Decimal IDB_Decimal; #ifdef POSIX_REGEX typedef regex_t IDB_Regex; #else -typedef boost::regex IDB_Regex; +typedef std::regex IDB_Regex; #endif typedef IDB_Regex CNX_Regex; diff --git a/dbcon/joblist/jobstep.h b/dbcon/joblist/jobstep.h index 79d1b350d..28bd4f994 100644 --- a/dbcon/joblist/jobstep.h +++ b/dbcon/joblist/jobstep.h @@ -479,7 +479,7 @@ class JobStep bool fDelivery; bool fOnClauseFilter; volatile bool fDie; - volatile uint32_t fWaitToRunStepCnt; + uint32_t fWaitToRunStepCnt; std::string fExtendedInfo; std::string fMiniInfo; diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index d08580b4f..e68837c6c 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -1398,8 +1398,8 @@ class TupleBPS : public BatchPrimitive, public TupleDeliveryStep SP_LBIDList lbidList; uint64_t ridsRequested; uint64_t totalMsgs; - volatile uint64_t msgsSent; - volatile uint64_t msgsRecvd; + uint64_t msgsSent; + uint64_t msgsRecvd; volatile bool finishedSending; bool firstRead; bool sendWaiting; diff --git a/dbcon/joblist/resourcemanager.cpp b/dbcon/joblist/resourcemanager.cpp index 92740a4c0..da041efa6 100644 --- a/dbcon/joblist/resourcemanager.cpp +++ b/dbcon/joblist/resourcemanager.cpp @@ -29,9 +29,6 @@ #include using namespace std; -#include -using namespace boost; - #include "resourcemanager.h" #include "jl_logger.h" diff --git a/dbcon/mysql/ha_autoi.cpp b/dbcon/mysql/ha_autoi.cpp index 4b9a2ae3b..ac71d0f80 100644 --- a/dbcon/mysql/ha_autoi.cpp +++ b/dbcon/mysql/ha_autoi.cpp @@ -18,38 +18,42 @@ /* * $Id$ */ +#include +#include +#include +#include bool parseAutoincrementTableComment(std::string comment, uint64_t& startValue, std::string& columnName) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", regex_constants::extended); + boost::algorithm::to_upper(comment); + std::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*=[[:space:]]*", std::regex_constants::extended); bool autoincrement = false; columnName = ""; - boost::match_results what; + std::match_results what; std::string::const_iterator start, end; start = comment.begin(); end = comment.end(); - boost::match_flag_type flags = boost::match_default; + std::regex_constants::match_flag_type flags = std::regex_constants::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (std::regex_search(start, end, what, compat, flags)) { if (what[0].matched) { // string params (what[0].first, what[0].second); - string params(&(*(what[0].second))); + std::string params(&(*(what[0].second))); unsigned i = params.find_first_of(","); if (i <= params.length()) { // check whether there is more autoincrement column - string restComment = params.substr(i + 1, params.length()); + std::string restComment = params.substr(i + 1, params.length()); start = restComment.begin(); end = restComment.end(); - if (boost::regex_search(start, end, what, compat, flags)) + if (std::regex_search(start, end, what, compat, flags)) { if (what[0].matched) - throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)); + throw runtime_error(logging::IDBErrorInfo::instance()->errorMsg(ERR_INVALID_NUMBER_AUTOINCREMENT)); } columnName = params.substr(0, i); @@ -112,25 +116,25 @@ bool parseAutoincrementTableComment(std::string comment, uint64_t& startValue, s bool parseAutoincrementColumnComment(std::string comment, uint64_t& startValue) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*", regex_constants::extended); + boost::algorithm::to_upper(comment); + std::regex compat("[[:space:]]*AUTOINCREMENT[[:space:]]*", std::regex_constants::extended); bool autoincrement = false; - boost::match_results what; + std::match_results what; std::string::const_iterator start, end; start = comment.begin(); end = comment.end(); - boost::match_flag_type flags = boost::match_default; + std::regex_constants::match_flag_type flags = std::regex_constants::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (std::regex_search(start, end, what, compat, flags)) { if (what[0].matched) { - string params(&(*(what[0].second))); + std::string params(&(*(what[0].second))); unsigned i = params.find_first_of(","); if (i <= params.length()) { - string startVal = params.substr(i + 1, params.length()); + std::string startVal = params.substr(i + 1, params.length()); // get rid of possible empty space i = startVal.find_first_not_of(" "); @@ -160,7 +164,7 @@ bool parseAutoincrementColumnComment(std::string comment, uint64_t& startValue) // (no digits) || (more chars) || (other errors & value = 0) if ((ep == str) || (*ep != '\0') || (errno != 0)) { - throw runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); + throw runtime_error(logging::IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); } } } diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 2efba1efd..92c9c9ceb 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -25,27 +25,19 @@ #include #include #include -#ifdef _MSC_VER -#include -#else #include -#endif #include #include #include #include -#ifdef _MSC_VER -#include -#else +#include #include -#endif #include #include using namespace std; #include #include -#include #include using namespace boost; @@ -142,16 +134,16 @@ CalpontSystemCatalog::ColDataType convertDataType(const ddlpackage::ColumnType& int parseCompressionComment(std::string comment) { - algorithm::to_upper(comment); - regex compat("[[:space:]]*COMPRESSION[[:space:]]*=[[:space:]]*", regex_constants::extended); + boost::algorithm::to_upper(comment); + std::regex compat("[[:space:]]*COMPRESSION[[:space:]]*=[[:space:]]*", std::regex_constants::extended); int compressiontype = 0; - boost::match_results what; + std::match_results what; std::string::const_iterator start, end; start = comment.begin(); end = comment.end(); - boost::match_flag_type flags = boost::match_default; + std::regex_constants::match_flag_type flags = std::regex_constants::match_default; - if (boost::regex_search(start, end, what, compat, flags)) + if (std::regex_search(start, end, what, compat, flags)) { // Find the pattern, now get the compression type string compType(&(*(what[0].second))); @@ -1389,10 +1381,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl if (comment.length() > 0) { //@Bug 3782 This is for synchronization after calonlinealter to use - algorithm::to_upper(comment); - regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", regex_constants::extended); + boost::algorithm::to_upper(comment); + std::regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", std::regex_constants::extended); - if (regex_search(comment, pat)) + if (std::regex_search(comment, pat)) { return 0; } @@ -2358,14 +2350,14 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea bool schemaSyncOnly = false; bool isCreate = true; - regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", regex_constants::extended); + std::regex pat("[[:space:]]*SCHEMA[[:space:]]+SYNC[[:space:]]+ONLY", std::regex_constants::extended); - if (regex_search(tablecomment, pat)) + if (std::regex_search(tablecomment, pat)) { schemaSyncOnly = true; pat = createpatstr; - if (!regex_search(stmt, pat)) + if (!std::regex_search(stmt, pat)) { isCreate = false; } @@ -2395,7 +2387,7 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea pat = alterpatstr; - if (regex_search(stmt, pat)) + if (std::regex_search(stmt, pat)) { ci.isAlter = true; ci.alterTableState = cal_connection_info::ALTER_FIRST_RENAME; diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 9651413f8..c9975947c 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -45,7 +45,6 @@ using namespace std; #include #include #include -#include #include #include "errorids.h" diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 9e7292949..853da476f 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -54,7 +54,6 @@ using namespace std; #include #include -#include #include #include "mcs_basic_types.h" diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index a5b51dec0..ae79eb1bc 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -26,28 +26,30 @@ * @file */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include #include +#include +#include +#include +#include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + #include "liboamcpp.h" #include "installdir.h" #include "mcsconfig.h" @@ -178,7 +180,7 @@ int main(int argc, char* argv[]) { includeArg = line; - boost::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*="); + std::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*="); // see if in columnstore.cnf.rpmsave ifstream mycnfsavefile(mycnfsaveFile.c_str()); char line[200]; @@ -188,7 +190,7 @@ int main(int argc, char* argv[]) { oldbuf = line; - if (boost::regex_search(oldbuf.begin(), oldbuf.end(), icludeArgRegEx)) + if (std::regex_search(oldbuf.begin(), oldbuf.end(), icludeArgRegEx)) { // found in columnstore.cnf.rpmsave, check if this is commented out if (line[0] != '#') @@ -205,7 +207,7 @@ int main(int argc, char* argv[]) { newbuf = line1; - if (boost::regex_search(newbuf.begin(), newbuf.end(), icludeArgRegEx)) + if (std::regex_search(newbuf.begin(), newbuf.end(), icludeArgRegEx)) { newbuf = oldbuf; cout << "Updated argument: " << includeArg << endl; @@ -240,9 +242,9 @@ int main(int argc, char* argv[]) while (mycnffile.getline(line1, 200)) { newbuf = line1; - boost::regex mysqldSectionRegEx("\\[mysqld\\]"); + std::regex mysqldSectionRegEx("\\[mysqld\\]"); - if (boost::regex_search(newbuf.begin(), newbuf.end(), mysqldSectionRegEx)) + if (std::regex_search(newbuf.begin(), newbuf.end(), mysqldSectionRegEx)) { lines.push_back(newbuf); newbuf = oldbuf; diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index f6c50b49d..98bdd9fee 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -40,7 +40,7 @@ #ifdef POSIX_REGEX #include #else -#include +#include #endif #include #include diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 943e65595..b65846533 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -44,8 +44,6 @@ using namespace std; #include -#include -#include using namespace boost; #include "configcpp.h" diff --git a/storage-manager/src/Config.cpp b/storage-manager/src/Config.cpp index f634ececb..fa9873484 100644 --- a/storage-manager/src/Config.cpp +++ b/storage-manager/src/Config.cpp @@ -23,7 +23,6 @@ #include #include #include -#include #include #include @@ -31,6 +30,7 @@ #include #include #include +#include #include "SMLogging.h" @@ -165,13 +165,13 @@ bool Config::reload() return rtn; } -string use_envvar(const boost::smatch& envvar) +string use_envvar(const std::smatch& envvar) { char* env = getenv(envvar[1].str().c_str()); return (env ? env : ""); } -string expand_numbers(const boost::smatch& match) +string expand_numbers(const std::smatch& match) { long long num = stol(match[1].str()); char suffix = (char)::tolower(match[2].str()[0]); @@ -187,6 +187,20 @@ string expand_numbers(const boost::smatch& match) return ::to_string(num); } +std::string regex_replace_with_format(const std::string& input, + const std::regex& regex, + std::function format) +{ + + std::ostringstream output; + std::sregex_iterator begin(input.begin(), input.end(), regex), end; + for(; begin != end; begin++){ + output << begin->prefix() << format(*begin); + } + output << input.substr(input.size() - begin->position()); + return output.str(); +} + string Config::getValue(const string& section, const string& key) const { // if we care, move this envvar substition stuff to where the file is loaded @@ -202,15 +216,15 @@ string Config::getValue(const string& section, const string& key) const } s.unlock(); - boost::regex re("\\$\\{(.+)\\}"); + std::regex re("\\$\\{(.+)\\}"); - ret = boost::regex_replace(ret, re, use_envvar); + ret = regex_replace_with_format(ret, re, use_envvar); // do the numeric substitutions. ex, the suffixes m, k, g // ehhhhh. going to end up turning a string to a number, to a string, and then to a number again // don't like that. OTOH who cares. - boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended); - ret = boost::regex_replace(ret, num_re, expand_numbers); + std::regex num_re("^([[:digit:]]+)([mMkKgG])$", std::regex::extended); + ret = regex_replace_with_format(ret, num_re, expand_numbers); return ret; } diff --git a/storage-manager/src/IOCoordinator.cpp b/storage-manager/src/IOCoordinator.cpp index b17d502b2..52c1567f6 100644 --- a/storage-manager/src/IOCoordinator.cpp +++ b/storage-manager/src/IOCoordinator.cpp @@ -25,8 +25,6 @@ #include #include #include -#define BOOST_SPIRIT_THREADSAFE -#include #include #include "checks.h" #include "vlarray.h" @@ -1266,9 +1264,10 @@ boost::shared_array IOCoordinator::mergeJournal(const char* object, con boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); + + nlohmann::json header = nlohmann::json::parse(ss); + + assert(header["version"] == 1); // start processing the entries while (1) @@ -1353,9 +1352,9 @@ int IOCoordinator::mergeJournalInMem(boost::shared_array& objData, size boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); + + nlohmann::json header = nlohmann::json::parse(ss); + assert(header["version"] == 1); // read the journal file into memory size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead; @@ -1433,9 +1432,9 @@ int IOCoordinator::mergeJournalInMem_bigJ(boost::shared_array& objData, boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - boost::property_tree::ptree header; - boost::property_tree::json_parser::read_json(ss, header); - assert(header.get("version") == 1); + + nlohmann::json header = nlohmann::json::parse(ss); + assert(header["version"] == 1); // start processing the entries while (1) diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index e45c95173..5386beae5 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -20,20 +20,17 @@ */ #include "MetadataFile.h" #include -#define BOOST_SPIRIT_THREADSAFE -#include -#include -#include #include #include #include +#include #include +#include #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) using namespace std; -namespace bpt = boost::property_tree; namespace bf = boost::filesystem; namespace @@ -120,12 +117,13 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename) { if (boost::filesystem::exists(mFilename)) { - jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); + std::ifstream i(mFilename.string()); + jsontree.reset(new nlohmann::json); + i >> *jsontree; jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; - mRevision = jsontree->get("revision"); + mRevision = (*jsontree)["revision"]; } else { @@ -140,7 +138,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename) { s.unlock(); mVersion = 1; - mRevision = jsontree->get("revision"); + mRevision = (*jsontree)["revision"];; } ++metadataFilesAccessed; } @@ -162,12 +160,13 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, if (boost::filesystem::exists(mFilename)) { _exists = true; - jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); + jsontree.reset(new nlohmann::json); + std::ifstream i(mFilename.string()); + i >> *jsontree; jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; - mRevision = jsontree->get("revision"); + mRevision = (*jsontree)["revision"]; } else { @@ -182,7 +181,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, s.unlock(); _exists = true; mVersion = 1; - mRevision = jsontree->get("revision"); + mRevision = (*jsontree)["revision"]; } ++metadataFilesAccessed; } @@ -193,11 +192,10 @@ MetadataFile::~MetadataFile() void MetadataFile::makeEmptyJsonTree() { - jsontree.reset(new bpt::ptree()); - boost::property_tree::ptree objs; - jsontree->put("version", mVersion); - jsontree->put("revision", mRevision); - jsontree->add_child("objects", objs); + jsontree.reset(new nlohmann::json); + (*jsontree)["version"] = mVersion; + (*jsontree)["revision"] = mRevision; + (*jsontree)["objects"] = nlohmann::json::array(); } void MetadataFile::printKPIs() @@ -219,11 +217,11 @@ size_t MetadataFile::getLength() const { size_t totalSize = 0; - auto& objects = jsontree->get_child("objects"); + auto &objects = (*jsontree)["objects"]; if (!objects.empty()) { - auto& lastObject = objects.back().second; - totalSize = lastObject.get("offset") + lastObject.get("length"); + auto& lastObject = objects.back(); + totalSize = lastObject["offset"].get() + lastObject["length"].get(); } return totalSize; } @@ -243,10 +241,9 @@ vector MetadataFile::metadataRead(off_t offset, size_t length) c rather than write a new alg. */ set mObjects; - BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + for(const auto &v : (*jsontree)["objects"]) { - mObjects.insert(metadataObject(v.second.get("offset"), v.second.get("length"), - v.second.get("key"))); + mObjects.insert(metadataObject(v["offset"], v["length"], v["key"])); } if (mObjects.size() == 0) @@ -288,20 +285,20 @@ metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& fi // metadataObject addObject; - auto& objects = jsontree->get_child("objects"); + auto& objects = (*jsontree)["objects"]; if (!objects.empty()) { - auto& lastObject = objects.back().second; - addObject.offset = lastObject.get("offset") + mpConfig->mObjectSize; + auto& lastObject = objects.back(); + addObject.offset = lastObject["offset"].get() + mpConfig->mObjectSize; } addObject.length = length; addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length); - boost::property_tree::ptree object; - object.put("offset", addObject.offset); - object.put("length", addObject.length); - object.put("key", addObject.key); - objects.push_back(make_pair("", object)); + nlohmann::json object = nlohmann::json::object(); + object["offset"] = addObject.offset; + object["length"] = addObject.length; + object["key"] = addObject.key; + objects.push_back(object); return addObject; } @@ -312,7 +309,8 @@ int MetadataFile::writeMetadata() if (!boost::filesystem::exists(mFilename.parent_path())) boost::filesystem::create_directories(mFilename.parent_path()); - write_json(mFilename.string(), *jsontree); + std::ofstream o(mFilename.c_str()); + o << *jsontree; _exists = true; boost::unique_lock s(jsonCache.getMutex()); @@ -324,13 +322,13 @@ int MetadataFile::writeMetadata() bool MetadataFile::getEntry(off_t offset, metadataObject* out) const { metadataObject addObject; - BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + for(auto &v: (*jsontree)["objects"]) { - if (v.second.get("offset") == offset) + if (v["offset"].get() == offset) { out->offset = offset; - out->length = v.second.get("length"); - out->key = v.second.get("key"); + out->length = v["length"].get(); + out->key = v["key"]; return true; } } @@ -339,10 +337,10 @@ bool MetadataFile::getEntry(off_t offset, metadataObject* out) const void MetadataFile::removeEntry(off_t offset) { - bpt::ptree& objects = jsontree->get_child("objects"); - for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it) + auto& objects = (*jsontree)["objects"]; + for (auto it = objects.begin(); it != objects.end(); ++it) { - if (it->second.get("offset") == offset) + if ((*it)["offset"].get() == offset) { objects.erase(it); break; @@ -352,7 +350,7 @@ void MetadataFile::removeEntry(off_t offset) void MetadataFile::removeAllEntries() { - jsontree->get_child("objects").clear(); + (*jsontree)["objects"] = nlohmann::json::array(); } void MetadataFile::deletedMeta(const bf::path& p) @@ -456,21 +454,21 @@ void MetadataFile::setLengthInKey(string& key, size_t newLength) void MetadataFile::printObjects() const { - BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) + for (auto& v : (*jsontree)["objects"]) { - printf("Name: %s Length: %zu Offset: %lld\n", v.second.get("key").c_str(), - v.second.get("length"), (long long)v.second.get("offset")); + printf("Name: %s Length: %zu Offset: %lld\n", v["key"].get().c_str(), + v["length"].get(), (long long)v["offset"].get()); } } void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLength) { - for (auto& v : jsontree->get_child("objects")) + for (auto& v : (*jsontree)["objects"]) { - if (v.second.get("offset") == offset) + if (v["offset"].get() == offset) { - v.second.put("key", newName); - v.second.put("length", newLength); + v["key"] = newName; + v["length"] = newLength; return; } } @@ -482,11 +480,11 @@ void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLe void MetadataFile::updateEntryLength(off_t offset, size_t newLength) { - for (auto& v : jsontree->get_child("objects")) + for (auto& v : (*jsontree)["objects"]) { - if (v.second.get("offset") == offset) + if (v["offset"].get() == offset) { - v.second.put("length", newLength); + v["length"] = newLength; return; } } @@ -498,11 +496,12 @@ void MetadataFile::updateEntryLength(off_t offset, size_t newLength) off_t MetadataFile::getMetadataNewObjectOffset() { - auto& objects = jsontree->get_child("objects"); + auto& objects = (*jsontree)["objects"]; if (objects.empty()) return 0; - auto& lastObject = jsontree->get_child("objects").back().second; - return lastObject.get("offset") + lastObject.get("length"); + + auto& lastObject = objects.back(); + return lastObject["offset"].get() + lastObject["length"].get(); } metadataObject::metadataObject() : offset(0), length(0) diff --git a/storage-manager/src/MetadataFile.h b/storage-manager/src/MetadataFile.h index 5c302c618..94d65c404 100644 --- a/storage-manager/src/MetadataFile.h +++ b/storage-manager/src/MetadataFile.h @@ -28,6 +28,8 @@ #include #include +#include + namespace storagemanager { struct metadataObject @@ -110,7 +112,7 @@ class MetadataFile static void printKPIs(); - typedef boost::shared_ptr Jsontree_t; + typedef boost::shared_ptr Jsontree_t; private: MetadataConfig* mpConfig; diff --git a/storage-manager/src/Replicator.cpp b/storage-manager/src/Replicator.cpp index 10fc07002..2a9241306 100644 --- a/storage-manager/src/Replicator.cpp +++ b/storage-manager/src/Replicator.cpp @@ -27,12 +27,12 @@ #include #include #include -#define BOOST_SPIRIT_THREADSAFE -#include #include #include #include +#include + using namespace std; namespace @@ -279,12 +279,14 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u stringstream ss; ss << headertxt.get(); headerRollback = headertxt.get(); - boost::property_tree::ptree header; + nlohmann::json header; + try { - boost::property_tree::json_parser::read_json(ss, header); + header = nlohmann::json::parse(ss); } - catch (const boost::property_tree::json_parser::json_parser_error& e) + + catch (const nlohmann::json::exception& e) { mpLogger->log(LOG_CRIT, "%s", e.what()); errno = EIO; @@ -296,8 +298,8 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u errno = EIO; return -1; } - assert(header.get("version") == 1); - uint64_t currentMaxOffset = header.get("max_offset"); + assert(header["version"] == 1); + uint64_t currentMaxOffset = header["max_offset"]; if (thisEntryMaxOffset > currentMaxOffset) { bHeaderChanged = true; diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index 758d9b2e6..9f14e1562 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -26,9 +26,9 @@ #include #include #include -#define BOOST_SPIRIT_THREADSAFE -#include -#include + +#include "utils/json/json.hpp" + #include "Utilities.h" using namespace std; @@ -258,12 +258,12 @@ bool S3Storage::getCredentialsFromMetadataEC2() logger->log(LOG_ERR, "CURL fail %u", curl_res); return false; } - stringstream credentials(readBuffer); - boost::property_tree::ptree pt; - boost::property_tree::read_json(credentials, pt); - key = pt.get("AccessKeyId"); - secret = pt.get("SecretAccessKey"); - token = pt.get("Token"); + + nlohmann::json pt = nlohmann::json::parse(readBuffer); + key = pt["AccessKeyId"]; + secret = pt["SecretAccessKey"]; + token = pt["Token"]; + // logger->log(LOG_INFO, "S3Storage: key = %s secret = %s token = // %s",key.c_str(),secret.c_str(),token.c_str()); @@ -626,7 +626,7 @@ int S3Storage::copyObject(const string& _sourceKey, const string& _destKey) #if 0 // no s3-s3 copy yet. get & put for now. - + int err; boost::shared_array data; size_t len; diff --git a/tools/passwd/CMakeLists.txt b/tools/passwd/CMakeLists.txt index 836b2219a..b4a72c057 100644 --- a/tools/passwd/CMakeLists.txt +++ b/tools/passwd/CMakeLists.txt @@ -1,5 +1,5 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ) +include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_COMMON_INCLUDE} ) ########### next target ############### set(cspasswd_SRCS cspasswd.cpp secrets.cpp) diff --git a/tools/passwd/secrets.cpp b/tools/passwd/secrets.cpp index aac32ffa5..179d38025 100644 --- a/tools/passwd/secrets.cpp +++ b/tools/passwd/secrets.cpp @@ -23,9 +23,8 @@ #include #include -#define BOOST_SPIRIT_THREADSAFE -#include -#include +#include "utils/json/json.hpp" + #include "idberrorinfo.h" #include "logger.h" #include "mcsconfig.h" @@ -393,16 +392,16 @@ ReadKeyResult secrets_readkeys(const string& filepath) { // File contents should be json. // json_error_t err; - boost::property_tree::ptree jsontree; + nlohmann::json jsontree; try { - boost::property_tree::read_json(filepath, jsontree); + std::ifstream i(filepath); + jsontree = nlohmann::json::parse(i); } - catch (boost::property_tree::json_parser::json_parser_error& je) + catch (const nlohmann::json::exception& je) { - std::cout << "Error reading JSON from secrets file: " << je.filename() << " on line: " << je.line() - << std::endl; - std::cout << je.message() << std::endl; + std::cout << "Error reading JSON from secrets file: " << filepath << std::endl; + std::cout << je.what() << std::endl; } catch (...) { @@ -410,8 +409,8 @@ ReadKeyResult secrets_readkeys(const string& filepath) strerror(errno)); } // json_t* obj = json_load_file(filepathc, 0, &err); - string enc_cipher = jsontree.get(field_cipher); - string enc_key = jsontree.get(field_key); + string enc_cipher = jsontree[field_cipher]; + string enc_key = jsontree[field_key]; // const char* enc_cipher = json_string_value(json_object_get(obj, field_cipher)); // const char* enc_key = json_string_value(json_object_get(obj, field_key)); bool cipher_ok = !enc_cipher.empty() && (enc_cipher == CIPHER_NAME); @@ -648,23 +647,24 @@ bool secrets_write_keys(const ByteVec& key, const string& filepath, const string utils::VLArray key_hex(2 * keylen + 1); bin2hex(key.data(), keylen, key_hex.data()); - boost::property_tree::ptree jsontree; - jsontree.put(field_desc, desc); - jsontree.put(field_version, columnstore_version.c_str()); - jsontree.put(field_cipher, CIPHER_NAME); - jsontree.put(field_key, (const char*)key_hex.data()); + nlohmann::json jsontree; + jsontree[field_desc] = desc; + jsontree[field_version] = columnstore_version; + jsontree[field_cipher] = CIPHER_NAME; + jsontree[field_key] = (const char*)key_hex.data(); auto filepathc = filepath.c_str(); bool write_ok = false; errno = 0; try { - write_json(filepathc, jsontree); + std::ofstream o(filepath); + o << jsontree; } - catch (boost::property_tree::json_parser::json_parser_error& je) + catch (const nlohmann::json::exception& je) { - std::cout << "Write to secrets file: " << je.filename() << " on line: " << je.line() << std::endl; - std::cout << je.message() << std::endl; + std::cout << "Write to secrets file: " << filepath << std::endl; + std::cout << je.what() << std::endl; } catch (...) { diff --git a/utils/common/cgroupconfigurator.cpp b/utils/common/cgroupconfigurator.cpp index 65b8b1139..45dcc1e26 100644 --- a/utils/common/cgroupconfigurator.cpp +++ b/utils/common/cgroupconfigurator.cpp @@ -20,13 +20,8 @@ #include "logger.h" #include #include -#include -#ifdef _MSC_VER -#include "unistd.h" -#include "sysinfo.h" -#else #include -#endif + using namespace boost; using namespace std; diff --git a/utils/common/stlpoolallocator.h b/utils/common/stlpoolallocator.h index 46c9bc1ac..1ca6f0880 100644 --- a/utils/common/stlpoolallocator.h +++ b/utils/common/stlpoolallocator.h @@ -72,7 +72,7 @@ class STLPoolAllocator void usePoolAllocator(boost::shared_ptr b); boost::shared_ptr getPoolAllocator(); - pointer allocate(size_type, const void* hint = 0); + pointer allocate(size_type, typename STLPoolAllocator::const_pointer hint = 0); void deallocate(pointer p, size_type n); size_type max_size() const throw(); inline uint64_t getMemUsage() const @@ -131,7 +131,7 @@ boost::shared_ptr STLPoolAllocator::getPoolAllocator() template typename STLPoolAllocator::pointer STLPoolAllocator::allocate( - typename STLPoolAllocator::size_type s, typename std::allocator::const_pointer hint) + typename STLPoolAllocator::size_type s, typename STLPoolAllocator::const_pointer hint) { return (pointer)pa->allocate(s * sizeof(T)); } diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 5653f59a7..908961caa 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -28,13 +28,8 @@ #include #include #include -#ifdef _MSC_VER -#include -#include -#include -#else + #include -#endif #ifdef __linux__ #define POSIX_REGEX @@ -43,7 +38,7 @@ #ifdef POSIX_REGEX #include #else -#include +#include #endif #include "mcs_datatype.h" diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 60e1459de..eef59d784 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -28,7 +28,7 @@ using namespace std; #ifdef __linux__ #include #else -#include +#include using namespace boost; #endif @@ -226,8 +226,8 @@ inline bool getBool(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull, return false; #else - regex pat(pattern.c_str()); - return regex_search(expr.c_str(), pat); + std::regex pat(pattern.c_str()); + return std::regex_search(expr.c_str(), pat); #endif } diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index 73586aef6..ec0e6f22c 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -32,7 +32,6 @@ #include #include -#include #include #include "dataconvert.h" diff --git a/utils/json/json.hpp b/utils/json/json.hpp new file mode 100644 index 000000000..e40e3b05b --- /dev/null +++ b/utils/json/json.hpp @@ -0,0 +1,22109 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.10.5 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2022 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file doc/README.md. * +\****************************************************************************/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 10 +#define NLOHMANN_JSON_VERSION_PATCH 5 + +#include // all_of, find, for_each +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#ifndef JSON_NO_IO + #include // istream, ostream +#endif // JSON_NO_IO +#include // random_access_iterator_tag +#include // unique_ptr +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include + + +#include +#include + +// #include + + +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include + + +#include // exception +#include // runtime_error +#include // to_string +#include // vector + +// #include + + +#include // array +#include // size_t +#include // uint8_t +#include // string + +namespace nlohmann +{ +namespace detail +{ +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : std::uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + binary, ///< binary array (ordered collection of bytes) + discarded ///< discarded by the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string < binary +- furthermore, each type is not smaller than itself +- discarded values are not comparable +- binary is represented as a b"" string in python and directly comparable to a + string; however, making a binary array directly comparable with a string would + be surprising behavior in a JSON file. + +@since version 1.0.0 +*/ +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + static constexpr std::array order = {{ + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */, + 6 /* binary */ + } + }; + + const auto l_index = static_cast(lhs); + const auto r_index = static_cast(rhs); + return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index]; +} +} // namespace detail +} // namespace nlohmann + +// #include + + +#include +// #include + + +#include // declval, pair +// #include + + +/* Hedley - https://nemequ.github.io/hedley + * Created by Evan Nemerson + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to + * the public domain worldwide. This software is distributed without + * any warranty. + * + * For details, see . + * SPDX-License-Identifier: CC0-1.0 + */ + +#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15) +#if defined(JSON_HEDLEY_VERSION) + #undef JSON_HEDLEY_VERSION +#endif +#define JSON_HEDLEY_VERSION 15 + +#if defined(JSON_HEDLEY_STRINGIFY_EX) + #undef JSON_HEDLEY_STRINGIFY_EX +#endif +#define JSON_HEDLEY_STRINGIFY_EX(x) #x + +#if defined(JSON_HEDLEY_STRINGIFY) + #undef JSON_HEDLEY_STRINGIFY +#endif +#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) + +#if defined(JSON_HEDLEY_CONCAT_EX) + #undef JSON_HEDLEY_CONCAT_EX +#endif +#define JSON_HEDLEY_CONCAT_EX(a,b) a##b + +#if defined(JSON_HEDLEY_CONCAT) + #undef JSON_HEDLEY_CONCAT +#endif +#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b) + +#if defined(JSON_HEDLEY_CONCAT3_EX) + #undef JSON_HEDLEY_CONCAT3_EX +#endif +#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c + +#if defined(JSON_HEDLEY_CONCAT3) + #undef JSON_HEDLEY_CONCAT3 +#endif +#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c) + +#if defined(JSON_HEDLEY_VERSION_ENCODE) + #undef JSON_HEDLEY_VERSION_ENCODE +#endif +#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) + #undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) + #undef JSON_HEDLEY_VERSION_DECODE_MINOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) + #undef JSON_HEDLEY_VERSION_DECODE_REVISION +#endif +#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) + +#if defined(JSON_HEDLEY_GNUC_VERSION) + #undef JSON_HEDLEY_GNUC_VERSION +#endif +#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#elif defined(__GNUC__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#endif + +#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) + #undef JSON_HEDLEY_GNUC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION) + #undef JSON_HEDLEY_MSVC_VERSION +#endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) +#elif defined(_MSC_FULL_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#elif defined(_MSC_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) + #undef JSON_HEDLEY_MSVC_VERSION_CHECK +#endif +#if !defined(JSON_HEDLEY_MSVC_VERSION) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#elif defined(_MSC_VER) && (_MSC_VER >= 1200) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#else + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION) + #undef JSON_HEDLEY_INTEL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#elif defined(__INTEL_COMPILER) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_VERSION) + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #undef JSON_HEDLEY_INTEL_CL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL) + #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION) + #undef JSON_HEDLEY_PGI_VERSION +#endif +#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) + #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION_CHECK) + #undef JSON_HEDLEY_PGI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PGI_VERSION) + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #undef JSON_HEDLEY_SUNPRO_VERSION +#endif +#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#elif defined(__SUNPRO_C) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) +#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) +#elif defined(__SUNPRO_CC) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) + #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#endif +#if defined(__EMSCRIPTEN__) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION) + #undef JSON_HEDLEY_ARM_VERSION +#endif +#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) +#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION_CHECK) + #undef JSON_HEDLEY_ARM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_ARM_VERSION) + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION) + #undef JSON_HEDLEY_IBM_VERSION +#endif +#if defined(__ibmxl__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#elif defined(__xlC__) && defined(__xlC_ver__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#elif defined(__xlC__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION_CHECK) + #undef JSON_HEDLEY_IBM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IBM_VERSION) + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION) + #undef JSON_HEDLEY_TI_VERSION +#endif +#if \ + defined(__TI_COMPILER_VERSION__) && \ + ( \ + defined(__TMS470__) || defined(__TI_ARM__) || \ + defined(__MSP430__) || \ + defined(__TMS320C2000__) \ + ) +#if (__TI_COMPILER_VERSION__ >= 16000000) + #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif +#endif + +#if defined(JSON_HEDLEY_TI_VERSION_CHECK) + #undef JSON_HEDLEY_TI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_VERSION) + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #undef JSON_HEDLEY_TI_CL2000_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__) + #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #undef JSON_HEDLEY_TI_CL430_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__) + #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #undef JSON_HEDLEY_TI_ARMCL_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__)) + #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK) + #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #undef JSON_HEDLEY_TI_CL6X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__) + #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #undef JSON_HEDLEY_TI_CL7X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__) + #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #undef JSON_HEDLEY_TI_CLPRU_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__) + #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION) + #undef JSON_HEDLEY_CRAY_VERSION +#endif +#if defined(_CRAYC) + #if defined(_RELEASE_PATCHLEVEL) + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) + #else + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) + #undef JSON_HEDLEY_CRAY_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_CRAY_VERSION) + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION) + #undef JSON_HEDLEY_IAR_VERSION +#endif +#if defined(__IAR_SYSTEMS_ICC__) + #if __VER__ > 1000 + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) + #else + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION_CHECK) + #undef JSON_HEDLEY_IAR_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IAR_VERSION) + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION) + #undef JSON_HEDLEY_TINYC_VERSION +#endif +#if defined(__TINYC__) + #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) + #undef JSON_HEDLEY_TINYC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION) + #undef JSON_HEDLEY_DMC_VERSION +#endif +#if defined(__DMC__) + #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION_CHECK) + #undef JSON_HEDLEY_DMC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_DMC_VERSION) + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #undef JSON_HEDLEY_COMPCERT_VERSION +#endif +#if defined(__COMPCERT_VERSION__) + #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) + #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION) + #undef JSON_HEDLEY_PELLES_VERSION +#endif +#if defined(__POCC__) + #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) + #undef JSON_HEDLEY_PELLES_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PELLES_VERSION) + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #undef JSON_HEDLEY_MCST_LCC_VERSION +#endif +#if defined(__LCC__) && defined(__LCC_MINOR__) + #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK) + #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION) + #undef JSON_HEDLEY_GCC_VERSION +#endif +#if \ + defined(JSON_HEDLEY_GNUC_VERSION) && \ + !defined(__clang__) && \ + !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_ARM_VERSION) && \ + !defined(JSON_HEDLEY_CRAY_VERSION) && \ + !defined(JSON_HEDLEY_TI_VERSION) && \ + !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL430_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \ + !defined(__COMPCERT__) && \ + !defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_ATTRIBUTE +#endif +#if \ + defined(__has_attribute) && \ + ( \ + (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \ + ) +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#else +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#endif +#if \ + defined(__has_cpp_attribute) && \ + defined(__cplusplus) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#endif +#if !defined(__cplusplus) || !defined(__has_cpp_attribute) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#elif \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ + (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_BUILTIN) + #undef JSON_HEDLEY_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#else + #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) + #undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) + #undef JSON_HEDLEY_GCC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_FEATURE) + #undef JSON_HEDLEY_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#else + #define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) + #undef JSON_HEDLEY_GNUC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_FEATURE) + #undef JSON_HEDLEY_GCC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_EXTENSION) + #undef JSON_HEDLEY_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#else + #define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) + #undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) + #undef JSON_HEDLEY_GCC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_WARNING) + #undef JSON_HEDLEY_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#else + #define JSON_HEDLEY_HAS_WARNING(warning) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_WARNING) + #undef JSON_HEDLEY_GNUC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_WARNING) + #undef JSON_HEDLEY_GCC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) + #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#else + #define JSON_HEDLEY_PRAGMA(value) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) + #undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#endif +#if defined(JSON_HEDLEY_DIAGNOSTIC_POP) + #undef JSON_HEDLEY_DIAGNOSTIC_POP +#endif +#if defined(__clang__) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) + #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_PUSH + #define JSON_HEDLEY_DIAGNOSTIC_POP +#endif + +/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat") +# if JSON_HEDLEY_HAS_WARNING("-Wc++17-extensions") +# if JSON_HEDLEY_HAS_WARNING("-Wc++1z-extensions") +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# endif +#endif +#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x +#endif + +#if defined(JSON_HEDLEY_CONST_CAST) + #undef JSON_HEDLEY_CONST_CAST +#endif +#if defined(__cplusplus) +# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif \ + JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_REINTERPRET_CAST) + #undef JSON_HEDLEY_REINTERPRET_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#else + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_STATIC_CAST) + #undef JSON_HEDLEY_STATIC_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#else + #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_CPP_CAST) + #undef JSON_HEDLEY_CPP_CAST +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wold-style-cast") +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \ + ((T) (expr)) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0) +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("diag_suppress=Pe137") \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr)) +# endif +#else +# define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-attributes") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunused-function") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif + +#if defined(JSON_HEDLEY_DEPRECATED) + #undef JSON_HEDLEY_DEPRECATED +#endif +#if defined(JSON_HEDLEY_DEPRECATED_FOR) + #undef JSON_HEDLEY_DEPRECATED_FOR +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif \ + (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif defined(__cplusplus) && (__cplusplus >= 201402L) + #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]]) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#else + #define JSON_HEDLEY_DEPRECATED(since) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#endif + +#if defined(JSON_HEDLEY_UNAVAILABLE) + #undef JSON_HEDLEY_UNAVAILABLE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#else + #define JSON_HEDLEY_UNAVAILABLE(available_since) +#endif + +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT +#endif +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__)) +#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) +#elif defined(_Check_return_) /* SAL */ + #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_ +#else + #define JSON_HEDLEY_WARN_UNUSED_RESULT + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) +#endif + +#if defined(JSON_HEDLEY_SENTINEL) + #undef JSON_HEDLEY_SENTINEL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#else + #define JSON_HEDLEY_SENTINEL(position) +#endif + +#if defined(JSON_HEDLEY_NO_RETURN) + #undef JSON_HEDLEY_NO_RETURN +#endif +#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NO_RETURN __noreturn +#elif \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L + #define JSON_HEDLEY_NO_RETURN _Noreturn +#elif defined(__cplusplus) && (__cplusplus >= 201103L) + #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_NO_RETURN _Pragma("does_not_return") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#else + #define JSON_HEDLEY_NO_RETURN +#endif + +#if defined(JSON_HEDLEY_NO_ESCAPE) + #undef JSON_HEDLEY_NO_ESCAPE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape) + #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__)) +#else + #define JSON_HEDLEY_NO_ESCAPE +#endif + +#if defined(JSON_HEDLEY_UNREACHABLE) + #undef JSON_HEDLEY_UNREACHABLE +#endif +#if defined(JSON_HEDLEY_UNREACHABLE_RETURN) + #undef JSON_HEDLEY_UNREACHABLE_RETURN +#endif +#if defined(JSON_HEDLEY_ASSUME) + #undef JSON_HEDLEY_ASSUME +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) + #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #if defined(__cplusplus) + #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) + #else + #define JSON_HEDLEY_ASSUME(expr) _nassert(expr) + #endif +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif defined(JSON_HEDLEY_ASSUME) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif +#if !defined(JSON_HEDLEY_ASSUME) + #if defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1))) + #else + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr) + #endif +#endif +#if defined(JSON_HEDLEY_UNREACHABLE) + #if \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value)) + #else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() + #endif +#else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value) +#endif +#if !defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif + +JSON_HEDLEY_DIAGNOSTIC_PUSH +#if JSON_HEDLEY_HAS_WARNING("-Wpedantic") + #pragma clang diagnostic ignored "-Wpedantic" +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus) + #pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif +#if JSON_HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0) + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wvariadic-macros" + #elif defined(JSON_HEDLEY_GCC_VERSION) + #pragma GCC diagnostic ignored "-Wvariadic-macros" + #endif +#endif +#if defined(JSON_HEDLEY_NON_NULL) + #undef JSON_HEDLEY_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#else + #define JSON_HEDLEY_NON_NULL(...) +#endif +JSON_HEDLEY_DIAGNOSTIC_POP + +#if defined(JSON_HEDLEY_PRINTF_FORMAT) + #undef JSON_HEDLEY_PRINTF_FORMAT +#endif +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(format) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) +#else + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) +#endif + +#if defined(JSON_HEDLEY_CONSTEXPR) + #undef JSON_HEDLEY_CONSTEXPR +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr) + #endif +#endif +#if !defined(JSON_HEDLEY_CONSTEXPR) + #define JSON_HEDLEY_CONSTEXPR +#endif + +#if defined(JSON_HEDLEY_PREDICT) + #undef JSON_HEDLEY_PREDICT +#endif +#if defined(JSON_HEDLEY_LIKELY) + #undef JSON_HEDLEY_LIKELY +#endif +#if defined(JSON_HEDLEY_UNLIKELY) + #undef JSON_HEDLEY_UNLIKELY +#endif +#if defined(JSON_HEDLEY_UNPREDICTABLE) + #undef JSON_HEDLEY_UNPREDICTABLE +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) + #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr)) +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability)) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 ) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 ) +#elif \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ + })) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ + })) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#else +# define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#endif +#if !defined(JSON_HEDLEY_UNPREDICTABLE) + #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#endif + +#if defined(JSON_HEDLEY_MALLOC) + #undef JSON_HEDLEY_MALLOC +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_MALLOC _Pragma("returns_new_memory") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_MALLOC __declspec(restrict) +#else + #define JSON_HEDLEY_MALLOC +#endif + +#if defined(JSON_HEDLEY_PURE) + #undef JSON_HEDLEY_PURE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) +# define JSON_HEDLEY_PURE _Pragma("does_not_write_global_data") +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \ + ) +# define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#else +# define JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_CONST) + #undef JSON_HEDLEY_CONST +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(const) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_CONST __attribute__((__const__)) +#elif \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_CONST _Pragma("no_side_effect") +#else + #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_RESTRICT) + #undef JSON_HEDLEY_RESTRICT +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT restrict +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + defined(__clang__) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT _Restrict +#else + #define JSON_HEDLEY_RESTRICT +#endif + +#if defined(JSON_HEDLEY_INLINE) + #undef JSON_HEDLEY_INLINE +#endif +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + (defined(__cplusplus) && (__cplusplus >= 199711L)) + #define JSON_HEDLEY_INLINE inline +#elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0) + #define JSON_HEDLEY_INLINE __inline__ +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_INLINE __inline +#else + #define JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_ALWAYS_INLINE) + #undef JSON_HEDLEY_ALWAYS_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) +# define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \ + ) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#else +# define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_NEVER_INLINE) + #undef JSON_HEDLEY_NEVER_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#else + #define JSON_HEDLEY_NEVER_INLINE +#endif + +#if defined(JSON_HEDLEY_PRIVATE) + #undef JSON_HEDLEY_PRIVATE +#endif +#if defined(JSON_HEDLEY_PUBLIC) + #undef JSON_HEDLEY_PUBLIC +#endif +#if defined(JSON_HEDLEY_IMPORT) + #undef JSON_HEDLEY_IMPORT +#endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC __declspec(dllexport) +# define JSON_HEDLEY_IMPORT __declspec(dllimport) +#else +# if \ + JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + ( \ + defined(__TI_EABI__) && \ + ( \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \ + ) \ + ) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +# define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +# else +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC +# endif +# define JSON_HEDLEY_IMPORT extern +#endif + +#if defined(JSON_HEDLEY_NO_THROW) + #undef JSON_HEDLEY_NO_THROW +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#else + #define JSON_HEDLEY_NO_THROW +#endif + +#if defined(JSON_HEDLEY_FALL_THROUGH) + #undef JSON_HEDLEY_FALL_THROUGH +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]]) +#elif defined(__fallthrough) /* SAL */ + #define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else + #define JSON_HEDLEY_FALL_THROUGH +#endif + +#if defined(JSON_HEDLEY_RETURNS_NON_NULL) + #undef JSON_HEDLEY_RETURNS_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#elif defined(_Ret_notnull_) /* SAL */ + #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#else + #define JSON_HEDLEY_RETURNS_NON_NULL +#endif + +#if defined(JSON_HEDLEY_ARRAY_PARAM) + #undef JSON_HEDLEY_ARRAY_PARAM +#endif +#if \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#else + #define JSON_HEDLEY_ARRAY_PARAM(name) +#endif + +#if defined(JSON_HEDLEY_IS_CONSTANT) + #undef JSON_HEDLEY_IS_CONSTANT +#endif +#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) + #undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#endif +/* JSON_HEDLEY_IS_CONSTEXPR_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #undef JSON_HEDLEY_IS_CONSTEXPR_ +#endif +#if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#endif +#if !defined(__cplusplus) +# if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) +#endif +# elif \ + ( \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION)) || \ + (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) +#endif +# elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + defined(JSON_HEDLEY_INTEL_VERSION) || \ + defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \ + defined(JSON_HEDLEY_TI_CL2000_VERSION) || \ + defined(JSON_HEDLEY_TI_CL6X_VERSION) || \ + defined(JSON_HEDLEY_TI_CL7X_VERSION) || \ + defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \ + defined(__clang__) +# define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \ + sizeof(void) != \ + sizeof(*( \ + 1 ? \ + ((void*) ((expr) * 0L) ) : \ +((struct { char v[sizeof(void) * 2]; } *) 1) \ + ) \ + ) \ + ) +# endif +#endif +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1)) +#else + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) (0) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#endif + +#if defined(JSON_HEDLEY_BEGIN_C_DECLS) + #undef JSON_HEDLEY_BEGIN_C_DECLS +#endif +#if defined(JSON_HEDLEY_END_C_DECLS) + #undef JSON_HEDLEY_END_C_DECLS +#endif +#if defined(JSON_HEDLEY_C_DECL) + #undef JSON_HEDLEY_C_DECL +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { + #define JSON_HEDLEY_END_C_DECLS } + #define JSON_HEDLEY_C_DECL extern "C" +#else + #define JSON_HEDLEY_BEGIN_C_DECLS + #define JSON_HEDLEY_END_C_DECLS + #define JSON_HEDLEY_C_DECL +#endif + +#if defined(JSON_HEDLEY_STATIC_ASSERT) + #undef JSON_HEDLEY_STATIC_ASSERT +#endif +#if \ + !defined(__cplusplus) && ( \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ + (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + defined(_Static_assert) \ + ) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif \ + (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message)) +#else +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#endif + +#if defined(JSON_HEDLEY_NULL) + #undef JSON_HEDLEY_NULL +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr) + #elif defined(NULL) + #define JSON_HEDLEY_NULL NULL + #else + #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0) + #endif +#elif defined(NULL) + #define JSON_HEDLEY_NULL NULL +#else + #define JSON_HEDLEY_NULL ((void*) 0) +#endif + +#if defined(JSON_HEDLEY_MESSAGE) + #undef JSON_HEDLEY_MESSAGE +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_WARNING) + #undef JSON_HEDLEY_WARNING +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE) + #undef JSON_HEDLEY_REQUIRE +#endif +#if defined(JSON_HEDLEY_REQUIRE_MSG) + #undef JSON_HEDLEY_REQUIRE_MSG +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) +# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +# define JSON_HEDLEY_REQUIRE(expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), #expr, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error"))) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error"))) +# endif +#else +# define JSON_HEDLEY_REQUIRE(expr) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) +#endif + +#if defined(JSON_HEDLEY_FLAGS) + #undef JSON_HEDLEY_FLAGS +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion")) + #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#else + #define JSON_HEDLEY_FLAGS +#endif + +#if defined(JSON_HEDLEY_FLAGS_CAST) + #undef JSON_HEDLEY_FLAGS_CAST +#endif +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0) +# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)") \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#endif + +#if defined(JSON_HEDLEY_EMPTY_BASES) + #undef JSON_HEDLEY_EMPTY_BASES +#endif +#if \ + (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases) +#else + #define JSON_HEDLEY_EMPTY_BASES +#endif + +/* Remaining macros are deprecated. */ + +#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#endif +#if defined(__clang__) + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) +#else + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) + #undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#endif +#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) + +#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) + #undef JSON_HEDLEY_CLANG_HAS_FEATURE +#endif +#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) + +#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) + #undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#endif +#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) + +#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_WARNING) + #undef JSON_HEDLEY_CLANG_HAS_WARNING +#endif +#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) + +#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ + +// #include + + +#include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; +} // namespace detail +} // namespace nlohmann + + +// https://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template class Op, class... Args> +using is_detected = typename detector::value_t; + +template class Op, class... Args> +struct is_detected_lazy : is_detected { }; + +template class Op, class... Args> +using detected_t = typename detector::type; + +template class Op, class... Args> +using detected_or = detector; + +template class Op, class... Args> +using detected_or_t = typename detected_or::type; + +template class Op, class... Args> +using is_detected_exact = std::is_same>; + +template class Op, class... Args> +using is_detected_convertible = + std::is_convertible, To>; +} // namespace detail +} // namespace nlohmann + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +// if the user manually specified the used c++ version this is skipped +#if !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11) + #if (defined(__cplusplus) && __cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 + #endif + // the cpp 11 flag is always specified because it is the minimal required version + #define JSON_HAS_CPP_11 +#endif + +#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM) + #ifdef JSON_HAS_CPP_17 + #if defined(__cpp_lib_filesystem) + #define JSON_HAS_FILESYSTEM 1 + #elif defined(__cpp_lib_experimental_filesystem) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif !defined(__has_include) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #endif + + // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/ + #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__clang_major__) && __clang_major__ < 7 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support + #if defined(_MSC_VER) && _MSC_VER < 1914 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before iOS 13 + #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before macOS Catalina + #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + #endif +#endif + +#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_FILESYSTEM + #define JSON_HAS_FILESYSTEM 0 +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdocumentation" + #pragma clang diagnostic ignored "-Wdocumentation-unknown-command" +#endif + +// allow disabling exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// allow overriding assert +#if !defined(JSON_ASSERT) + #include // assert + #define JSON_ASSERT(x) assert(x) +#endif + +// allow to access some private functions (needed by the test suite) +#if defined(JSON_TESTS_PRIVATE) + #define JSON_PRIVATE_UNLESS_TESTED public +#else + #define JSON_PRIVATE_UNLESS_TESTED private +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [&j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer, \ + class BinaryType> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// Macros to simplify conversion from/to types + +#define NLOHMANN_JSON_EXPAND( x ) x +#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME +#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ + NLOHMANN_JSON_PASTE64, \ + NLOHMANN_JSON_PASTE63, \ + NLOHMANN_JSON_PASTE62, \ + NLOHMANN_JSON_PASTE61, \ + NLOHMANN_JSON_PASTE60, \ + NLOHMANN_JSON_PASTE59, \ + NLOHMANN_JSON_PASTE58, \ + NLOHMANN_JSON_PASTE57, \ + NLOHMANN_JSON_PASTE56, \ + NLOHMANN_JSON_PASTE55, \ + NLOHMANN_JSON_PASTE54, \ + NLOHMANN_JSON_PASTE53, \ + NLOHMANN_JSON_PASTE52, \ + NLOHMANN_JSON_PASTE51, \ + NLOHMANN_JSON_PASTE50, \ + NLOHMANN_JSON_PASTE49, \ + NLOHMANN_JSON_PASTE48, \ + NLOHMANN_JSON_PASTE47, \ + NLOHMANN_JSON_PASTE46, \ + NLOHMANN_JSON_PASTE45, \ + NLOHMANN_JSON_PASTE44, \ + NLOHMANN_JSON_PASTE43, \ + NLOHMANN_JSON_PASTE42, \ + NLOHMANN_JSON_PASTE41, \ + NLOHMANN_JSON_PASTE40, \ + NLOHMANN_JSON_PASTE39, \ + NLOHMANN_JSON_PASTE38, \ + NLOHMANN_JSON_PASTE37, \ + NLOHMANN_JSON_PASTE36, \ + NLOHMANN_JSON_PASTE35, \ + NLOHMANN_JSON_PASTE34, \ + NLOHMANN_JSON_PASTE33, \ + NLOHMANN_JSON_PASTE32, \ + NLOHMANN_JSON_PASTE31, \ + NLOHMANN_JSON_PASTE30, \ + NLOHMANN_JSON_PASTE29, \ + NLOHMANN_JSON_PASTE28, \ + NLOHMANN_JSON_PASTE27, \ + NLOHMANN_JSON_PASTE26, \ + NLOHMANN_JSON_PASTE25, \ + NLOHMANN_JSON_PASTE24, \ + NLOHMANN_JSON_PASTE23, \ + NLOHMANN_JSON_PASTE22, \ + NLOHMANN_JSON_PASTE21, \ + NLOHMANN_JSON_PASTE20, \ + NLOHMANN_JSON_PASTE19, \ + NLOHMANN_JSON_PASTE18, \ + NLOHMANN_JSON_PASTE17, \ + NLOHMANN_JSON_PASTE16, \ + NLOHMANN_JSON_PASTE15, \ + NLOHMANN_JSON_PASTE14, \ + NLOHMANN_JSON_PASTE13, \ + NLOHMANN_JSON_PASTE12, \ + NLOHMANN_JSON_PASTE11, \ + NLOHMANN_JSON_PASTE10, \ + NLOHMANN_JSON_PASTE9, \ + NLOHMANN_JSON_PASTE8, \ + NLOHMANN_JSON_PASTE7, \ + NLOHMANN_JSON_PASTE6, \ + NLOHMANN_JSON_PASTE5, \ + NLOHMANN_JSON_PASTE4, \ + NLOHMANN_JSON_PASTE3, \ + NLOHMANN_JSON_PASTE2, \ + NLOHMANN_JSON_PASTE1)(__VA_ARGS__)) +#define NLOHMANN_JSON_PASTE2(func, v1) func(v1) +#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2) +#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3) +#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4) +#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5) +#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6) +#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7) +#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8) +#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9) +#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10) +#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) +#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) +#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) +#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) +#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) +#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) +#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) +#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) +#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) +#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) +#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) +#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) +#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) +#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) +#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) +#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) +#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) +#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) +#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) +#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) +#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) +#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) +#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) +#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) +#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) +#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) +#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) +#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) +#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) +#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) +#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) +#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) +#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) +#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) +#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) +#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) +#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) +#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) +#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) +#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) +#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) +#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) +#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) +#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) +#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) +#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) +#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) +#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) +#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) +#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) +#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) +#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) +#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) + +#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1; +#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1); +#define NLOHMANN_JSON_FROM_WITH_DEFAULT(v1) nlohmann_json_t.v1 = nlohmann_json_j.value(#v1, nlohmann_json_default_obj.v1); + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + + +// inspired from https://stackoverflow.com/a/26745591 +// allows to call any std function as if (e.g. with begin): +// using std::begin; begin(x); +// +// it allows using the detected idiom to retrieve the return type +// of such an expression +#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name) \ + namespace detail { \ + using std::std_name; \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + } \ + \ + namespace detail2 { \ + struct std_name##_tag \ + { \ + }; \ + \ + template \ + std_name##_tag std_name(T&&...); \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + \ + template \ + struct would_call_std_##std_name \ + { \ + static constexpr auto const value = ::nlohmann::detail:: \ + is_detected_exact::value; \ + }; \ + } /* namespace detail2 */ \ + \ + template \ + struct would_call_std_##std_name : detail2::would_call_std_##std_name \ + { \ + } + +#ifndef JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_USE_IMPLICIT_CONVERSIONS 1 +#endif + +#if JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_EXPLICIT +#else + #define JSON_EXPLICIT explicit +#endif + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 0 +#endif + + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief replace all occurrences of a substring by another string + +@param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t +@param[in] f the substring to replace with @a t +@param[in] t the string to replace @a f + +@pre The search string @a f must not be empty. **This precondition is +enforced with an assertion.** + +@since version 2.0.0 +*/ +inline void replace_substring(std::string& s, const std::string& f, + const std::string& t) +{ + JSON_ASSERT(!f.empty()); + for (auto pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find next occurrence of f + {} +} + +/*! + * @brief string escaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to escape + * @return escaped string + * + * Note the order of escaping "~" to "~0" and "/" to "~1" is important. + */ +inline std::string escape(std::string s) +{ + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; +} + +/*! + * @brief string unescaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to unescape + * @return unescaped string + * + * Note the order of escaping "~1" to "/" and "~0" to "~" is important. + */ +static void unescape(std::string& s) +{ + replace_substring(s, "~1", "/"); + replace_substring(s, "~0", "~"); +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +//////////////// +// exceptions // +//////////////// + +/// @brief general exception of the @ref basic_json class +/// @sa https://json.nlohmann.me/api/basic_json/exception/ +class exception : public std::exception +{ + public: + /// returns the explanatory string + const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes) + + protected: + JSON_HEDLEY_NON_NULL(3) + exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} // NOLINT(bugprone-throw-keyword-missing) + + static std::string name(const std::string& ename, int id_) + { + return "[json.exception." + ename + "." + std::to_string(id_) + "] "; + } + + template + static std::string diagnostics(const BasicJsonType& leaf_element) + { +#if JSON_DIAGNOSTICS + std::vector tokens; + for (const auto* current = &leaf_element; current->m_parent != nullptr; current = current->m_parent) + { + switch (current->m_parent->type()) + { + case value_t::array: + { + for (std::size_t i = 0; i < current->m_parent->m_value.array->size(); ++i) + { + if (¤t->m_parent->m_value.array->operator[](i) == current) + { + tokens.emplace_back(std::to_string(i)); + break; + } + } + break; + } + + case value_t::object: + { + for (const auto& element : *current->m_parent->m_value.object) + { + if (&element.second == current) + { + tokens.emplace_back(element.first.c_str()); + break; + } + } + break; + } + + case value_t::null: // LCOV_EXCL_LINE + case value_t::string: // LCOV_EXCL_LINE + case value_t::boolean: // LCOV_EXCL_LINE + case value_t::number_integer: // LCOV_EXCL_LINE + case value_t::number_unsigned: // LCOV_EXCL_LINE + case value_t::number_float: // LCOV_EXCL_LINE + case value_t::binary: // LCOV_EXCL_LINE + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + break; // LCOV_EXCL_LINE + } + } + + if (tokens.empty()) + { + return ""; + } + + return "(" + std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + detail::escape(b); + }) + ") "; +#else + static_cast(leaf_element); + return ""; +#endif + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; + +/// @brief exception indicating a parse error +/// @sa https://json.nlohmann.me/api/basic_json/parse_error/ +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id_ the id of the exception + @param[in] pos the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + template + static parse_error create(int id_, const position_t& pos, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + exception::diagnostics(context) + what_arg; + return {id_, pos.chars_read_total, w.c_str()}; + } + + template + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + + ": " + exception::diagnostics(context) + what_arg; + return {id_, byte_, w.c_str()}; + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character and + n+1 is the index of the terminating null byte or the end of file. + This also holds true when reading a byte vector (CBOR or MessagePack). + */ + const std::size_t byte; + + private: + parse_error(int id_, std::size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } +}; + +/// @brief exception indicating errors with iterators +/// @sa https://json.nlohmann.me/api/basic_json/invalid_iterator/ +class invalid_iterator : public exception +{ + public: + template + static invalid_iterator create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("invalid_iterator", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) {} +}; + +/// @brief exception indicating executing a member function with a wrong type +/// @sa https://json.nlohmann.me/api/basic_json/type_error/ +class type_error : public exception +{ + public: + template + static type_error create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("type_error", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/// @brief exception indicating access out of the defined range +/// @sa https://json.nlohmann.me/api/basic_json/out_of_range/ +class out_of_range : public exception +{ + public: + template + static out_of_range create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("out_of_range", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/// @brief exception indicating other library errors +/// @sa https://json.nlohmann.me/api/basic_json/other_error/ +class other_error : public exception +{ + public: + template + static other_error create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("other_error", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type +#include // index_sequence, make_index_sequence, index_sequence_for + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +template +using uncvref_t = typename std::remove_cv::type>::type; + +#ifdef JSON_HAS_CPP_14 + +// the following utilities are natively available in C++14 +using std::enable_if_t; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; + +#else + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h +// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0. + +//// START OF CODE FROM GOOGLE ABSEIL + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `absl::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence()); +// } +template +struct integer_sequence +{ + using value_type = T; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `absl::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template +using index_sequence = integer_sequence; + +namespace utility_internal +{ + +template +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template +struct Extend, SeqSize, 0> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >; +}; + +template +struct Extend, SeqSize, 1> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >; +}; + +// Recursion helper for 'make_integer_sequence'. +// 'Gen::type' is an alias for 'integer_sequence'. +template +struct Gen +{ + using type = + typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type; +}; + +template +struct Gen +{ + using type = integer_sequence; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template +using make_integer_sequence = typename utility_internal::Gen::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template +using make_index_sequence = make_integer_sequence; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template +using index_sequence_for = make_index_sequence; + +//// END OF CODE FROM GOOGLE ABSEIL + +#endif + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; // NOLINT(readability-redundant-declaration) + +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +// dispatching helper struct +template struct identity_tag {}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval +#include // tuple + +// #include + + +// #include + + +#include // random_access_iterator_tag + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +// #include + + +namespace nlohmann +{ +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); +} // namespace nlohmann + +// #include + + +// #include + + +namespace nlohmann +{ +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); +} // namespace nlohmann + +// #include + +// #include + +// #include +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +/// a class to store JSON values +/// @sa https://json.nlohmann.me/api/basic_json/ +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector> +class basic_json; + +/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document +/// @sa https://json.nlohmann.me/api/json_pointer/ +template +class json_pointer; + +/*! +@brief default specialization +@sa https://json.nlohmann.me/api/json/ +*/ +using json = basic_json<>; + +/// @brief a minimal map-like container that preserves insertion order +/// @sa https://json.nlohmann.me/api/ordered_map/ +template +struct ordered_map; + +/// @brief specialization that maintains the insertion order of object keys +/// @sa https://json.nlohmann.me/api/ordered_json/ +using ordered_json = basic_json; + +} // namespace nlohmann + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + +namespace nlohmann +{ +/*! +@brief detail namespace with internal helper functions + +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. + +@since version 2.1.0 +*/ +namespace detail +{ +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json : std::true_type {}; + +////////////////////// +// json_ref helpers // +////////////////////// + +template +class json_ref; + +template +struct is_json_ref : std::false_type {}; + +template +struct is_json_ref> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template +using mapped_type_t = typename T::mapped_type; + +template +using key_type_t = typename T::key_type; + +template +using value_type_t = typename T::value_type; + +template +using difference_type_t = typename T::difference_type; + +template +using pointer_t = typename T::pointer; + +template +using reference_t = typename T::reference; + +template +using iterator_category_t = typename T::iterator_category; + +template +using to_json_function = decltype(T::to_json(std::declval()...)); + +template +using from_json_function = decltype(T::from_json(std::declval()...)); + +template +using get_template_function = decltype(std::declval().template get()); + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json : std::false_type {}; + +// trait checking if j.get is valid +// use this trait instead of std::is_constructible or std::is_convertible, +// both rely on, or make use of implicit conversions, and thus fail when T +// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958) +template +struct is_getable +{ + static constexpr bool value = is_detected::value; +}; + +template +struct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json : std::false_type {}; + +template +struct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template +struct has_to_json : std::false_type {}; + +template +struct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + + +/////////////////// +// is_ functions // +/////////////////// + +// https://en.cppreference.com/w/cpp/types/conjunction +template struct conjunction : std::true_type { }; +template struct conjunction : B1 { }; +template +struct conjunction +: std::conditional, B1>::type {}; + +// https://en.cppreference.com/w/cpp/types/negation +template struct negation : std::integral_constant < bool, !B::value > { }; + +// Reimplementation of is_constructible and is_default_constructible, due to them being broken for +// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367). +// This causes compile errors in e.g. clang 3.5 or gcc 4.9. +template +struct is_default_constructible : std::is_default_constructible {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + + +template +struct is_constructible : std::is_constructible {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + + +template +struct is_iterator_traits : std::false_type {}; + +template +struct is_iterator_traits> +{ + private: + using traits = iterator_traits; + + public: + static constexpr auto value = + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value; +}; + +template +struct is_range +{ + private: + using t_ref = typename std::add_lvalue_reference::type; + + using iterator = detected_t; + using sentinel = detected_t; + + // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator + // and https://en.cppreference.com/w/cpp/iterator/sentinel_for + // but reimplementing these would be too much work, as a lot of other concepts are used underneath + static constexpr auto is_iterator_begin = + is_iterator_traits>::value; + + public: + static constexpr bool value = !std::is_same::value && !std::is_same::value && is_iterator_begin; +}; + +template +using iterator_t = enable_if_t::value, result_of_begin())>>; + +template +using range_value_t = value_type_t>>; + +// The following implementation of is_complete_type is taken from +// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/ +// and is written by Xiang Fan who agreed to using it in this library. + +template +struct is_complete_type : std::false_type {}; + +template +struct is_complete_type : std::true_type {}; + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + is_constructible::value && + is_constructible::value; +}; + +template +struct is_compatible_object_type + : is_compatible_object_type_impl {}; + +template +struct is_constructible_object_type_impl : std::false_type {}; + +template +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (is_default_constructible::value && + (std::is_move_assignable::value || + std::is_copy_assignable::value) && + (is_constructible::value && + std::is_same < + typename object_t::mapped_type, + typename ConstructibleObjectType::mapped_type >::value)) || + (has_from_json::value || + has_non_default_from_json < + BasicJsonType, + typename ConstructibleObjectType::mapped_type >::value); +}; + +template +struct is_constructible_object_type + : is_constructible_object_type_impl {}; + +template +struct is_compatible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_constructible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_compatible_array_type_impl : std::false_type {}; + +template +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t < + is_detected::value&& + is_iterator_traits>>::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 + !std::is_same>::value >> +{ + static constexpr bool value = + is_constructible>::value; +}; + +template +struct is_compatible_array_type + : is_compatible_array_type_impl {}; + +template +struct is_constructible_array_type_impl : std::false_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value >> + : std::true_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t < !std::is_same::value&& + !is_compatible_string_type::value&& + is_default_constructible::value&& +(std::is_move_assignable::value || + std::is_copy_assignable::value)&& +is_detected::value&& +is_iterator_traits>>::value&& +is_detected::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 +!std::is_same>::value&& + is_complete_type < + detected_t>::value >> +{ + using value_type = range_value_t; + + static constexpr bool value = + std::is_same::value || + has_from_json::value || + has_non_default_from_json < + BasicJsonType, + value_type >::value; +}; + +template +struct is_constructible_array_type + : is_constructible_array_type_impl {}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t < std::is_integral::value&& + std::is_integral::value&& + !std::is_same::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + is_constructible::value && + CompatibleLimits::is_integer && + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type + : is_compatible_integer_type_impl {}; + +template +struct is_compatible_type_impl: std::false_type {}; + +template +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t::value >> +{ + static constexpr bool value = + has_to_json::value; +}; + +template +struct is_compatible_type + : is_compatible_type_impl {}; + +template +struct is_constructible_tuple : std::false_type {}; + +template +struct is_constructible_tuple> : conjunction...> {}; + +// a naive helper to check if a type is an ordered_map (exploits the fact that +// ordered_map inherits capacity() from std::vector) +template +struct is_ordered_map +{ + using one = char; + + struct two + { + char x[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + }; + + template static one test( decltype(&C::capacity) ) ; + template static two test(...); + + enum { value = sizeof(test(nullptr)) == sizeof(char) }; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) +}; + +// to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) +template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > +T conditional_static_cast(U value) +{ + return static_cast(value); +} + +template::value, int> = 0> +T conditional_static_cast(U value) +{ + return value; +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#if JSON_HAS_EXPERIMENTAL_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::experimental::filesystem; +} // namespace nlohmann::detail +#elif JSON_HAS_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::filesystem; +} // namespace nlohmann::detail +#endif + +namespace nlohmann +{ +namespace detail +{ +template +void from_json(const BasicJsonType& j, typename std::nullptr_t& n) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_null())) + { + JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name()), j)); + } + n = nullptr; +} + +// overloads for basic_json template parameters +template < typename BasicJsonType, typename ArithmeticType, + enable_if_t < std::is_arithmetic::value&& + !std::is_same::value, + int > = 0 > +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::boolean: + case value_t::binary: + case value_t::discarded: + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()), j)); + } +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_boolean())) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()), j)); + } + b = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + s = *j.template get_ptr(); +} + +template < + typename BasicJsonType, typename ConstructibleStringType, + enable_if_t < + is_constructible_string_type::value&& + !std::is_same::value, + int > = 0 > +void from_json(const BasicJsonType& j, ConstructibleStringType& s) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + + s = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type::type val; + get_arithmetic_value(j, val); + e = static_cast(val); +} + +// forward_list doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list& l) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + l.clear(); + std::transform(j.rbegin(), j.rend(), + std::front_inserter(l), [](const BasicJsonType & i) + { + return i.template get(); + }); +} + +// valarray doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::valarray& l) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + l.resize(j.size()); + std::transform(j.begin(), j.end(), std::begin(l), + [](const BasicJsonType & elem) + { + return elem.template get(); + }); +} + +template +auto from_json(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +-> decltype(j.template get(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get(); + } +} + +template +void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/) +{ + arr = *j.template get_ptr(); +} + +template +auto from_json_array_impl(const BasicJsonType& j, std::array& arr, + priority_tag<2> /*unused*/) +-> decltype(j.template get(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get(); + } +} + +template::value, + int> = 0> +auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) +-> decltype( + arr.reserve(std::declval()), + j.template get(), + void()) +{ + using std::end; + + ConstructibleArrayType ret; + ret.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(ret, end(ret)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); + arr = std::move(ret); +} + +template::value, + int> = 0> +void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, + priority_tag<0> /*unused*/) +{ + using std::end; + + ConstructibleArrayType ret; + std::transform( + j.begin(), j.end(), std::inserter(ret, end(ret)), + [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); + arr = std::move(ret); +} + +template < typename BasicJsonType, typename ConstructibleArrayType, + enable_if_t < + is_constructible_array_type::value&& + !is_constructible_object_type::value&& + !is_constructible_string_type::value&& + !std::is_same::value&& + !is_basic_json::value, + int > = 0 > +auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr) +-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}), +j.template get(), +void()) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + from_json_array_impl(j, arr, priority_tag<3> {}); +} + +template < typename BasicJsonType, typename T, std::size_t... Idx > +std::array from_json_inplace_array_impl(BasicJsonType&& j, + identity_tag> /*unused*/, index_sequence /*unused*/) +{ + return { { std::forward(j).at(Idx).template get()... } }; +} + +template < typename BasicJsonType, typename T, std::size_t N > +auto from_json(BasicJsonType&& j, identity_tag> tag) +-> decltype(from_json_inplace_array_impl(std::forward(j), tag, make_index_sequence {})) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + return from_json_inplace_array_impl(std::forward(j), tag, make_index_sequence {}); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::binary_t& bin) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_binary())) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(j.type_name()), j)); + } + + bin = *j.template get_ptr(); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_object())) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()), j)); + } + + ConstructibleObjectType ret; + const auto* inner_object = j.template get_ptr(); + using value_type = typename ConstructibleObjectType::value_type; + std::transform( + inner_object->begin(), inner_object->end(), + std::inserter(ret, ret.begin()), + [](typename BasicJsonType::object_t::value_type const & p) + { + return value_type(p.first, p.second.template get()); + }); + obj = std::move(ret); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template < typename BasicJsonType, typename ArithmeticType, + enable_if_t < + std::is_arithmetic::value&& + !std::is_same::value&& + !std::is_same::value&& + !std::is_same::value&& + !std::is_same::value, + int > = 0 > +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::boolean: + { + val = static_cast(*j.template get_ptr()); + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::binary: + case value_t::discarded: + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()), j)); + } +} + +template +std::tuple from_json_tuple_impl_base(BasicJsonType&& j, index_sequence /*unused*/) +{ + return std::make_tuple(std::forward(j).at(Idx).template get()...); +} + +template < typename BasicJsonType, class A1, class A2 > +std::pair from_json_tuple_impl(BasicJsonType&& j, identity_tag> /*unused*/, priority_tag<0> /*unused*/) +{ + return {std::forward(j).at(0).template get(), + std::forward(j).at(1).template get()}; +} + +template +void from_json_tuple_impl(BasicJsonType&& j, std::pair& p, priority_tag<1> /*unused*/) +{ + p = from_json_tuple_impl(std::forward(j), identity_tag> {}, priority_tag<0> {}); +} + +template +std::tuple from_json_tuple_impl(BasicJsonType&& j, identity_tag> /*unused*/, priority_tag<2> /*unused*/) +{ + return from_json_tuple_impl_base(std::forward(j), index_sequence_for {}); +} + +template +void from_json_tuple_impl(BasicJsonType&& j, std::tuple& t, priority_tag<3> /*unused*/) +{ + t = from_json_tuple_impl_base(std::forward(j), index_sequence_for {}); +} + +template +auto from_json(BasicJsonType&& j, TupleRelated&& t) +-> decltype(from_json_tuple_impl(std::forward(j), std::forward(t), priority_tag<3> {})) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + return from_json_tuple_impl(std::forward(j), std::forward(t), priority_tag<3> {}); +} + +template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t < !std::is_constructible < + typename BasicJsonType::string_t, Key >::value >> +void from_json(const BasicJsonType& j, std::map& m) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + m.clear(); + for (const auto& p : j) + { + if (JSON_HEDLEY_UNLIKELY(!p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()), j)); + } + m.emplace(p.at(0).template get(), p.at(1).template get()); + } +} + +template < typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, + typename = enable_if_t < !std::is_constructible < + typename BasicJsonType::string_t, Key >::value >> +void from_json(const BasicJsonType& j, std::unordered_map& m) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + m.clear(); + for (const auto& p : j) + { + if (JSON_HEDLEY_UNLIKELY(!p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()), j)); + } + m.emplace(p.at(0).template get(), p.at(1).template get()); + } +} + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template +void from_json(const BasicJsonType& j, std_fs::path& p) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + p = *j.template get_ptr(); +} +#endif + +struct from_json_fn +{ + template + auto operator()(const BasicJsonType& j, T&& val) const + noexcept(noexcept(from_json(j, std::forward(val)))) + -> decltype(from_json(j, std::forward(val))) + { + return from_json(j, std::forward(val)); + } +}; +} // namespace detail + +/// namespace to hold default `from_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces) +{ +constexpr const auto& from_json = detail::static_const::value; // NOLINT(misc-definitions-in-headers) +} // namespace +} // namespace nlohmann + +// #include + + +#include // copy +#include // begin, end +#include // string +#include // tuple, get +#include // is_same, is_constructible, is_floating_point, is_enum, underlying_type +#include // move, forward, declval, pair +#include // valarray +#include // vector + +// #include + +// #include + + +#include // size_t +#include // input_iterator_tag +#include // string, to_string +#include // tuple_size, get, tuple_element +#include // move + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +void int_to_string( string_type& target, std::size_t value ) +{ + // For ADL + using std::to_string; + target = to_string(value); +} +template class iteration_proxy_value +{ + public: + using difference_type = std::ptrdiff_t; + using value_type = iteration_proxy_value; + using pointer = value_type * ; + using reference = value_type & ; + using iterator_category = std::input_iterator_tag; + using string_type = typename std::remove_cv< typename std::remove_reference().key() ) >::type >::type; + + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + std::size_t array_index = 0; + /// last stringified array index + mutable std::size_t array_index_last = 0; + /// a string representation of the array index + mutable string_type array_index_str = "0"; + /// an empty string (to return a reference for primitive values) + const string_type empty_str{}; + + public: + explicit iteration_proxy_value(IteratorType it) noexcept + : anchor(std::move(it)) + {} + + /// dereference operator (needed for range-based for) + iteration_proxy_value& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_value& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// equality operator (needed for InputIterator) + bool operator==(const iteration_proxy_value& o) const + { + return anchor == o.anchor; + } + + /// inequality operator (needed for range-based for) + bool operator!=(const iteration_proxy_value& o) const + { + return anchor != o.anchor; + } + + /// return key of the iterator + const string_type& key() const + { + JSON_ASSERT(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + if (array_index != array_index_last) + { + int_to_string( array_index_str, array_index ); + array_index_last = array_index; + } + return array_index_str; + } + + // use key from the object + case value_t::object: + return anchor.key(); + + // use an empty key for all primitive types + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return empty_str; + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } +}; + +/// proxy class for the items() function +template class iteration_proxy +{ + private: + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) noexcept + : container(cont) {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_value begin() noexcept + { + return iteration_proxy_value(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_value end() noexcept + { + return iteration_proxy_value(container.end()); + } +}; +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template = 0> +auto get(const nlohmann::detail::iteration_proxy_value& i) -> decltype(i.key()) +{ + return i.key(); +} +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template = 0> +auto get(const nlohmann::detail::iteration_proxy_value& i) -> decltype(i.value()) +{ + return i.value(); +} +} // namespace detail +} // namespace nlohmann + +// The Addition to the STD Namespace is required to add +// Structured Bindings Support to the iteration_proxy_value class +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +namespace std +{ +#if defined(__clang__) + // Fix: https://github.com/nlohmann/json/issues/1401 + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wmismatched-tags" +#endif +template +class tuple_size<::nlohmann::detail::iteration_proxy_value> + : public std::integral_constant {}; + +template +class tuple_element> +{ + public: + using type = decltype( + get(std::declval < + ::nlohmann::detail::iteration_proxy_value> ())); +}; +#if defined(__clang__) + #pragma clang diagnostic pop +#endif +} // namespace std + +// #include + +// #include + +// #include + + +#if JSON_HAS_EXPERIMENTAL_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::experimental::filesystem; +} // namespace nlohmann::detail +#elif JSON_HAS_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::filesystem; +} // namespace nlohmann::detail +#endif + +namespace nlohmann +{ +namespace detail +{ +////////////////// +// constructors // +////////////////// + +/* + * Note all external_constructor<>::construct functions need to call + * j.m_value.destroy(j.m_type) to avoid a memory leak in case j contains an + * allocated value (e.g., a string). See bug issue + * https://github.com/nlohmann/json/issues/2865 for more information. + */ + +template struct external_constructor; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::boolean; + j.m_value = b; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value = s; + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value = std::move(s); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleStringType, + enable_if_t < !std::is_same::value, + int > = 0 > + static void construct(BasicJsonType& j, const CompatibleStringType& str) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value.string = j.template create(str); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::binary; + j.m_value = typename BasicJsonType::binary_t(b); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::binary; + j.m_value = typename BasicJsonType::binary_t(std::move(b)); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_float; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_unsigned; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_integer; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = arr; + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = std::move(arr); + j.set_parents(); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < !std::is_same::value, + int > = 0 > + static void construct(BasicJsonType& j, const CompatibleArrayType& arr) + { + using std::begin; + using std::end; + + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value.array = j.template create(begin(arr), end(arr)); + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, const std::vector& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->reserve(arr.size()); + for (const bool x : arr) + { + j.m_value.array->push_back(x); + j.set_parent(j.m_value.array->back()); + } + j.assert_invariant(); + } + + template::value, int> = 0> + static void construct(BasicJsonType& j, const std::valarray& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->resize(arr.size()); + if (arr.size() > 0) + { + std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin()); + } + j.set_parents(); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value = obj; + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value = std::move(obj); + j.set_parents(); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleObjectType, + enable_if_t < !std::is_same::value, int > = 0 > + static void construct(BasicJsonType& j, const CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value.object = j.template create(begin(obj), end(obj)); + j.set_parents(); + j.assert_invariant(); + } +}; + +///////////// +// to_json // +///////////// + +template::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept +{ + external_constructor::construct(j, b); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleString& s) +{ + external_constructor::construct(j, s); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s) +{ + external_constructor::construct(j, std::move(s)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, FloatType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, EnumType e) noexcept +{ + using underlying_type = typename std::underlying_type::type; + external_constructor::construct(j, static_cast(e)); +} + +template +void to_json(BasicJsonType& j, const std::vector& e) +{ + external_constructor::construct(j, e); +} + +template < typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < is_compatible_array_type::value&& + !is_compatible_object_type::value&& + !is_compatible_string_type::value&& + !std::is_same::value&& + !is_basic_json::value, + int > = 0 > +void to_json(BasicJsonType& j, const CompatibleArrayType& arr) +{ + external_constructor::construct(j, arr); +} + +template +void to_json(BasicJsonType& j, const typename BasicJsonType::binary_t& bin) +{ + external_constructor::construct(j, bin); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, const std::valarray& arr) +{ + external_constructor::construct(j, std::move(arr)); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr) +{ + external_constructor::construct(j, std::move(arr)); +} + +template < typename BasicJsonType, typename CompatibleObjectType, + enable_if_t < is_compatible_object_type::value&& !is_basic_json::value, int > = 0 > +void to_json(BasicJsonType& j, const CompatibleObjectType& obj) +{ + external_constructor::construct(j, obj); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) +{ + external_constructor::construct(j, std::move(obj)); +} + +template < + typename BasicJsonType, typename T, std::size_t N, + enable_if_t < !std::is_constructible::value, // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + int > = 0 > +void to_json(BasicJsonType& j, const T(&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +{ + external_constructor::construct(j, arr); +} + +template < typename BasicJsonType, typename T1, typename T2, enable_if_t < std::is_constructible::value&& std::is_constructible::value, int > = 0 > +void to_json(BasicJsonType& j, const std::pair& p) +{ + j = { p.first, p.second }; +} + +// for https://github.com/nlohmann/json/pull/1134 +template>::value, int> = 0> +void to_json(BasicJsonType& j, const T& b) +{ + j = { {b.key(), b.value()} }; +} + +template +void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence /*unused*/) +{ + j = { std::get(t)... }; +} + +template::value, int > = 0> +void to_json(BasicJsonType& j, const T& t) +{ + to_json_tuple_impl(j, t, make_index_sequence::value> {}); +} + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template +void to_json(BasicJsonType& j, const std_fs::path& p) +{ + j = p.string(); +} +#endif + +struct to_json_fn +{ + template + auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward(val)))) + -> decltype(to_json(j, std::forward(val)), void()) + { + return to_json(j, std::forward(val)); + } +}; +} // namespace detail + +/// namespace to hold default `to_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces) +{ +constexpr const auto& to_json = detail::static_const::value; // NOLINT(misc-definitions-in-headers) +} // namespace +} // namespace nlohmann + +// #include + +// #include + + +namespace nlohmann +{ + +/// @sa https://json.nlohmann.me/api/adl_serializer/ +template +struct adl_serializer +{ + /// @brief convert a JSON value to any value type + /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/ + template + static auto from_json(BasicJsonType && j, TargetType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), val))) + -> decltype(::nlohmann::from_json(std::forward(j), val), void()) + { + ::nlohmann::from_json(std::forward(j), val); + } + + /// @brief convert a JSON value to any value type + /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/ + template + static auto from_json(BasicJsonType && j) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), detail::identity_tag {}))) + -> decltype(::nlohmann::from_json(std::forward(j), detail::identity_tag {})) + { + return ::nlohmann::from_json(std::forward(j), detail::identity_tag {}); + } + + /// @brief convert any value type to a JSON value + /// @sa https://json.nlohmann.me/api/adl_serializer/to_json/ + template + static auto to_json(BasicJsonType& j, TargetType && val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward(val)))) + -> decltype(::nlohmann::to_json(j, std::forward(val)), void()) + { + ::nlohmann::to_json(j, std::forward(val)); + } +}; +} // namespace nlohmann + +// #include + + +#include // uint8_t, uint64_t +#include // tie +#include // move + +namespace nlohmann +{ + +/// @brief an internal type for a backed binary type +/// @sa https://json.nlohmann.me/api/byte_container_with_subtype/ +template +class byte_container_with_subtype : public BinaryType +{ + public: + using container_type = BinaryType; + using subtype_type = std::uint64_t; + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype() noexcept(noexcept(container_type())) + : container_type() + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(const container_type& b) noexcept(noexcept(container_type(b))) + : container_type(b) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(container_type&& b) noexcept(noexcept(container_type(std::move(b)))) + : container_type(std::move(b)) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(const container_type& b, subtype_type subtype_) noexcept(noexcept(container_type(b))) + : container_type(b) + , m_subtype(subtype_) + , m_has_subtype(true) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(container_type&& b, subtype_type subtype_) noexcept(noexcept(container_type(std::move(b)))) + : container_type(std::move(b)) + , m_subtype(subtype_) + , m_has_subtype(true) + {} + + bool operator==(const byte_container_with_subtype& rhs) const + { + return std::tie(static_cast(*this), m_subtype, m_has_subtype) == + std::tie(static_cast(rhs), rhs.m_subtype, rhs.m_has_subtype); + } + + bool operator!=(const byte_container_with_subtype& rhs) const + { + return !(rhs == *this); + } + + /// @brief sets the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/set_subtype/ + void set_subtype(subtype_type subtype_) noexcept + { + m_subtype = subtype_; + m_has_subtype = true; + } + + /// @brief return the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/subtype/ + constexpr subtype_type subtype() const noexcept + { + return m_has_subtype ? m_subtype : static_cast(-1); + } + + /// @brief return whether the value has a subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/has_subtype/ + constexpr bool has_subtype() const noexcept + { + return m_has_subtype; + } + + /// @brief clears the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/clear_subtype/ + void clear_subtype() noexcept + { + m_subtype = 0; + m_has_subtype = false; + } + + private: + subtype_type m_subtype = 0; + bool m_has_subtype = false; +}; + +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + + +#include // uint8_t +#include // size_t +#include // hash + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +// boost::hash_combine +inline std::size_t combine(std::size_t seed, std::size_t h) noexcept +{ + seed ^= h + 0x9e3779b9 + (seed << 6U) + (seed >> 2U); + return seed; +} + +/*! +@brief hash a JSON value + +The hash function tries to rely on std::hash where possible. Furthermore, the +type of the JSON value is taken into account to have different hash values for +null, 0, 0U, and false, etc. + +@tparam BasicJsonType basic_json specialization +@param j JSON value to hash +@return hash value of j +*/ +template +std::size_t hash(const BasicJsonType& j) +{ + using string_t = typename BasicJsonType::string_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + const auto type = static_cast(j.type()); + switch (j.type()) + { + case BasicJsonType::value_t::null: + case BasicJsonType::value_t::discarded: + { + return combine(type, 0); + } + + case BasicJsonType::value_t::object: + { + auto seed = combine(type, j.size()); + for (const auto& element : j.items()) + { + const auto h = std::hash {}(element.key()); + seed = combine(seed, h); + seed = combine(seed, hash(element.value())); + } + return seed; + } + + case BasicJsonType::value_t::array: + { + auto seed = combine(type, j.size()); + for (const auto& element : j) + { + seed = combine(seed, hash(element)); + } + return seed; + } + + case BasicJsonType::value_t::string: + { + const auto h = std::hash {}(j.template get_ref()); + return combine(type, h); + } + + case BasicJsonType::value_t::boolean: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_integer: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_unsigned: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_float: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::binary: + { + auto seed = combine(type, j.get_binary().size()); + const auto h = std::hash {}(j.get_binary().has_subtype()); + seed = combine(seed, h); + seed = combine(seed, static_cast(j.get_binary().subtype())); + for (const auto byte : j.get_binary()) + { + seed = combine(seed, std::hash {}(byte)); + } + return seed; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // generate_n +#include // array +#include // ldexp +#include // size_t +#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // snprintf +#include // memcpy +#include // back_inserter +#include // numeric_limits +#include // char_traits, string +#include // make_pair, move +#include // vector + +// #include + +// #include + + +#include // array +#include // size_t +#include // strlen +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // shared_ptr, make_shared, addressof +#include // accumulate +#include // string, char_traits +#include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer +#include // pair, declval + +#ifndef JSON_NO_IO + #include // FILE * + #include // istream +#endif // JSON_NO_IO + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/// the supported input formats +enum class input_format_t { json, cbor, msgpack, ubjson, bson }; + +//////////////////// +// input adapters // +//////////////////// + +#ifndef JSON_NO_IO +/*! +Input adapter for stdio file access. This adapter read only 1 byte and do not use any + buffer. This adapter is a very low level adapter. +*/ +class file_input_adapter +{ + public: + using char_type = char; + + JSON_HEDLEY_NON_NULL(2) + explicit file_input_adapter(std::FILE* f) noexcept + : m_file(f) + {} + + // make class move-only + file_input_adapter(const file_input_adapter&) = delete; + file_input_adapter(file_input_adapter&&) noexcept = default; + file_input_adapter& operator=(const file_input_adapter&) = delete; + file_input_adapter& operator=(file_input_adapter&&) = delete; + ~file_input_adapter() = default; + + std::char_traits::int_type get_character() noexcept + { + return std::fgetc(m_file); + } + + private: + /// the file pointer to read from + std::FILE* m_file; +}; + + +/*! +Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at +beginning of input. Does not support changing the underlying std::streambuf +in mid-input. Maintains underlying std::istream and std::streambuf to support +subsequent use of standard std::istream operations to process any input +characters following those used in parsing the JSON input. Clears the +std::istream flags; any input errors (e.g., EOF) will be detected by the first +subsequent call for input from the std::istream. +*/ +class input_stream_adapter +{ + public: + using char_type = char; + + ~input_stream_adapter() + { + // clear stream flags; we use underlying streambuf I/O, do not + // maintain ifstream flags, except eof + if (is != nullptr) + { + is->clear(is->rdstate() & std::ios::eofbit); + } + } + + explicit input_stream_adapter(std::istream& i) + : is(&i), sb(i.rdbuf()) + {} + + // delete because of pointer members + input_stream_adapter(const input_stream_adapter&) = delete; + input_stream_adapter& operator=(input_stream_adapter&) = delete; + input_stream_adapter& operator=(input_stream_adapter&&) = delete; + + input_stream_adapter(input_stream_adapter&& rhs) noexcept + : is(rhs.is), sb(rhs.sb) + { + rhs.is = nullptr; + rhs.sb = nullptr; + } + + // std::istream/std::streambuf use std::char_traits::to_int_type, to + // ensure that std::char_traits::eof() and the character 0xFF do not + // end up as the same value, e.g. 0xFFFFFFFF. + std::char_traits::int_type get_character() + { + auto res = sb->sbumpc(); + // set eof manually, as we don't use the istream interface. + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) + { + is->clear(is->rdstate() | std::ios::eofbit); + } + return res; + } + + private: + /// the associated input stream + std::istream* is = nullptr; + std::streambuf* sb = nullptr; +}; +#endif // JSON_NO_IO + +// General-purpose iterator-based adapter. It might not be as fast as +// theoretically possible for some containers, but it is extremely versatile. +template +class iterator_input_adapter +{ + public: + using char_type = typename std::iterator_traits::value_type; + + iterator_input_adapter(IteratorType first, IteratorType last) + : current(std::move(first)), end(std::move(last)) + {} + + typename std::char_traits::int_type get_character() + { + if (JSON_HEDLEY_LIKELY(current != end)) + { + auto result = std::char_traits::to_int_type(*current); + std::advance(current, 1); + return result; + } + + return std::char_traits::eof(); + } + + private: + IteratorType current; + IteratorType end; + + template + friend struct wide_string_input_helper; + + bool empty() const + { + return current == end; + } +}; + + +template +struct wide_string_input_helper; + +template +struct wide_string_input_helper +{ + // UTF-32 + static void fill_buffer(BaseInputAdapter& input, + std::array::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (JSON_HEDLEY_UNLIKELY(input.empty())) + { + utf8_bytes[0] = std::char_traits::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = input.get_character(); + + // UTF-32 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = static_cast::int_type>(0xC0u | ((static_cast(wc) >> 6u) & 0x1Fu)); + utf8_bytes[1] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 2; + } + else if (wc <= 0xFFFF) + { + utf8_bytes[0] = static_cast::int_type>(0xE0u | ((static_cast(wc) >> 12u) & 0x0Fu)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 3; + } + else if (wc <= 0x10FFFF) + { + utf8_bytes[0] = static_cast::int_type>(0xF0u | ((static_cast(wc) >> 18u) & 0x07u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 4; + } + else + { + // unknown character + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + } + } +}; + +template +struct wide_string_input_helper +{ + // UTF-16 + static void fill_buffer(BaseInputAdapter& input, + std::array::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (JSON_HEDLEY_UNLIKELY(input.empty())) + { + utf8_bytes[0] = std::char_traits::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = input.get_character(); + + // UTF-16 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = static_cast::int_type>(0xC0u | ((static_cast(wc) >> 6u))); + utf8_bytes[1] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 2; + } + else if (0xD800 > wc || wc >= 0xE000) + { + utf8_bytes[0] = static_cast::int_type>(0xE0u | ((static_cast(wc) >> 12u))); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 3; + } + else + { + if (JSON_HEDLEY_UNLIKELY(!input.empty())) + { + const auto wc2 = static_cast(input.get_character()); + const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); + utf8_bytes[0] = static_cast::int_type>(0xF0u | (charcode >> 18u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (charcode & 0x3Fu)); + utf8_bytes_filled = 4; + } + else + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + } + } + } +}; + +// Wraps another input apdater to convert wide character types into individual bytes. +template +class wide_string_input_adapter +{ + public: + using char_type = char; + + wide_string_input_adapter(BaseInputAdapter base) + : base_adapter(base) {} + + typename std::char_traits::int_type get_character() noexcept + { + // check if buffer needs to be filled + if (utf8_bytes_index == utf8_bytes_filled) + { + fill_buffer(); + + JSON_ASSERT(utf8_bytes_filled > 0); + JSON_ASSERT(utf8_bytes_index == 0); + } + + // use buffer + JSON_ASSERT(utf8_bytes_filled > 0); + JSON_ASSERT(utf8_bytes_index < utf8_bytes_filled); + return utf8_bytes[utf8_bytes_index++]; + } + + private: + BaseInputAdapter base_adapter; + + template + void fill_buffer() + { + wide_string_input_helper::fill_buffer(base_adapter, utf8_bytes, utf8_bytes_index, utf8_bytes_filled); + } + + /// a buffer for UTF-8 bytes + std::array::int_type, 4> utf8_bytes = {{0, 0, 0, 0}}; + + /// index to the utf8_codes array for the next valid byte + std::size_t utf8_bytes_index = 0; + /// number of valid bytes in the utf8_codes array + std::size_t utf8_bytes_filled = 0; +}; + + +template +struct iterator_input_adapter_factory +{ + using iterator_type = IteratorType; + using char_type = typename std::iterator_traits::value_type; + using adapter_type = iterator_input_adapter; + + static adapter_type create(IteratorType first, IteratorType last) + { + return adapter_type(std::move(first), std::move(last)); + } +}; + +template +struct is_iterator_of_multibyte +{ + using value_type = typename std::iterator_traits::value_type; + enum + { + value = sizeof(value_type) > 1 + }; +}; + +template +struct iterator_input_adapter_factory::value>> +{ + using iterator_type = IteratorType; + using char_type = typename std::iterator_traits::value_type; + using base_adapter_type = iterator_input_adapter; + using adapter_type = wide_string_input_adapter; + + static adapter_type create(IteratorType first, IteratorType last) + { + return adapter_type(base_adapter_type(std::move(first), std::move(last))); + } +}; + +// General purpose iterator-based input +template +typename iterator_input_adapter_factory::adapter_type input_adapter(IteratorType first, IteratorType last) +{ + using factory_type = iterator_input_adapter_factory; + return factory_type::create(first, last); +} + +// Convenience shorthand from container to iterator +// Enables ADL on begin(container) and end(container) +// Encloses the using declarations in namespace for not to leak them to outside scope + +namespace container_input_adapter_factory_impl +{ + +using std::begin; +using std::end; + +template +struct container_input_adapter_factory {}; + +template +struct container_input_adapter_factory< ContainerType, + void_t()), end(std::declval()))>> + { + using adapter_type = decltype(input_adapter(begin(std::declval()), end(std::declval()))); + + static adapter_type create(const ContainerType& container) +{ + return input_adapter(begin(container), end(container)); +} + }; + +} // namespace container_input_adapter_factory_impl + +template +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(const ContainerType& container) +{ + return container_input_adapter_factory_impl::container_input_adapter_factory::create(container); +} + +#ifndef JSON_NO_IO +// Special cases with fast paths +inline file_input_adapter input_adapter(std::FILE* file) +{ + return file_input_adapter(file); +} + +inline input_stream_adapter input_adapter(std::istream& stream) +{ + return input_stream_adapter(stream); +} + +inline input_stream_adapter input_adapter(std::istream&& stream) +{ + return input_stream_adapter(stream); +} +#endif // JSON_NO_IO + +using contiguous_bytes_input_adapter = decltype(input_adapter(std::declval(), std::declval())); + +// Null-delimited strings, and the like. +template < typename CharT, + typename std::enable_if < + std::is_pointer::value&& + !std::is_array::value&& + std::is_integral::type>::value&& + sizeof(typename std::remove_pointer::type) == 1, + int >::type = 0 > +contiguous_bytes_input_adapter input_adapter(CharT b) +{ + auto length = std::strlen(reinterpret_cast(b)); + const auto* ptr = reinterpret_cast(b); + return input_adapter(ptr, ptr + length); +} + +template +auto input_adapter(T (&array)[N]) -> decltype(input_adapter(array, array + N)) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +{ + return input_adapter(array, array + N); +} + +// This class only handles inputs of input_buffer_adapter type. +// It's required so that expressions like {ptr, len} can be implicitly cast +// to the correct adapter. +class span_input_adapter +{ + public: + template < typename CharT, + typename std::enable_if < + std::is_pointer::value&& + std::is_integral::type>::value&& + sizeof(typename std::remove_pointer::type) == 1, + int >::type = 0 > + span_input_adapter(CharT b, std::size_t l) + : ia(reinterpret_cast(b), reinterpret_cast(b) + l) {} + + template::iterator_category, std::random_access_iterator_tag>::value, + int>::type = 0> + span_input_adapter(IteratorType first, IteratorType last) + : ia(input_adapter(first, last)) {} + + contiguous_bytes_input_adapter&& get() + { + return std::move(ia); // NOLINT(hicpp-move-const-arg,performance-move-const-arg) + } + + private: + contiguous_bytes_input_adapter ia; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include +#include // string +#include // move +#include // vector + +// #include + +// #include + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template +struct json_sax +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief a floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string value was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string value. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief a binary value was read + @param[in] val binary value + @return whether parsing should proceed + @note It is safe to move the passed binary value. + */ + virtual bool binary(binary_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] ex an exception object describing the error + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + json_sax() = default; + json_sax(const json_sax&) = default; + json_sax(json_sax&&) noexcept = default; + json_sax& operator=(const json_sax&) = default; + json_sax& operator=(json_sax&&) noexcept = default; + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + /*! + @param[in,out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + // make class move-only + json_sax_dom_parser(const json_sax_dom_parser&) = delete; + json_sax_dom_parser(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + json_sax_dom_parser& operator=(const json_sax_dom_parser&) = delete; + json_sax_dom_parser& operator=(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~json_sax_dom_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool binary(binary_t& val) + { + handle_value(std::move(val)); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive object size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.back()->set_parents(); + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive array size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool end_array() + { + ref_stack.back()->set_parents(); + ref_stack.pop_back(); + return true; + } + + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const Exception& ex) + { + errored = true; + static_cast(ex); + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template + JSON_HEDLEY_RETURNS_NON_NULL + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward(v)); + return &root; + } + + JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward(v)); + return &(ref_stack.back()->m_value.array->back()); + } + + JSON_ASSERT(ref_stack.back()->is_object()); + JSON_ASSERT(object_element); + *object_element = BasicJsonType(std::forward(v)); + return object_element; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + // make class move-only + json_sax_dom_callback_parser(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + json_sax_dom_callback_parser& operator=(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser& operator=(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~json_sax_dom_callback_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool binary(binary_t& val) + { + handle_value(std::move(val)); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive object size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep && ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back()) + { + if (!callback(static_cast(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + else + { + ref_stack.back()->set_parents(); + } + } + + JSON_ASSERT(!ref_stack.empty()); + JSON_ASSERT(!keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) + { + // remove discarded value + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive array size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (keep) + { + ref_stack.back()->set_parents(); + } + else + { + // discard array + *ref_stack.back() = discarded; + } + } + + JSON_ASSERT(!ref_stack.empty()); + JSON_ASSERT(!keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (!keep && !ref_stack.empty() && ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + + return true; + } + + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const Exception& ex) + { + errored = true; + static_cast(ex); + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template + std::pair handle_value(Value&& v, const bool skip_callback = false) + { + JSON_ASSERT(!keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (!keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward(v)); + + // check callback + const bool keep = skip_callback || callback(static_cast(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (!keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (!ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object()); + + // array + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + + // object + JSON_ASSERT(ref_stack.back()->is_object()); + // check if we should store an element for the current key + JSON_ASSERT(!key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (!store_element) + { + return {false, nullptr}; + } + + JSON_ASSERT(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack {}; + /// stack to manage which values to keep + std::vector keep_stack {}; + /// stack to manage which object keys to keep + std::vector key_keep_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + bool null() + { + return true; + } + + bool boolean(bool /*unused*/) + { + return true; + } + + bool number_integer(number_integer_t /*unused*/) + { + return true; + } + + bool number_unsigned(number_unsigned_t /*unused*/) + { + return true; + } + + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) + { + return true; + } + + bool string(string_t& /*unused*/) + { + return true; + } + + bool binary(binary_t& /*unused*/) + { + return true; + } + + bool start_object(std::size_t /*unused*/ = static_cast(-1)) + { + return true; + } + + bool key(string_t& /*unused*/) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t /*unused*/ = static_cast(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) + { + return false; + } +}; +} // namespace detail + +} // namespace nlohmann + +// #include + + +#include // array +#include // localeconv +#include // size_t +#include // snprintf +#include // strtof, strtod, strtold, strtoll, strtoull +#include // initializer_list +#include // char_traits, string +#include // move +#include // vector + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/////////// +// lexer // +/////////// + +template +class lexer_base +{ + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_unsigned, ///< an unsigned integer -- use get_number_unsigned() for actual value + value_integer, ///< a signed integer -- use get_number_integer() for actual value + value_float, ///< an floating point number -- use get_number_float() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input, ///< indicating the end of the input buffer + literal_or_value ///< a literal or the begin of a value (only for diagnostics) + }; + + /// return name of values of type token_type (only used for errors) + JSON_HEDLEY_RETURNS_NON_NULL + JSON_HEDLEY_CONST + static const char* token_type_name(const token_type t) noexcept + { + switch (t) + { + case token_type::uninitialized: + return ""; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return ""; + case token_type::end_of_input: + return "end of input"; + case token_type::literal_or_value: + return "'[', '{', or a literal"; + // LCOV_EXCL_START + default: // catch non-enum values + return "unknown token"; + // LCOV_EXCL_STOP + } + } +}; +/*! +@brief lexical analysis + +This class organizes the lexical analysis during JSON deserialization. +*/ +template +class lexer : public lexer_base +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using char_type = typename InputAdapterType::char_type; + using char_int_type = typename std::char_traits::int_type; + + public: + using token_type = typename lexer_base::token_type; + + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept + : ia(std::move(adapter)) + , ignore_comments(ignore_comments_) + , decimal_point_char(static_cast(get_decimal_point())) + {} + + // delete because of pointer members + lexer(const lexer&) = delete; + lexer(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + lexer& operator=(lexer&) = delete; + lexer& operator=(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~lexer() = default; + + private: + ///////////////////// + // locales + ///////////////////// + + /// return the locale-dependent decimal point + JSON_HEDLEY_PURE + static char get_decimal_point() noexcept + { + const auto* loc = localeconv(); + JSON_ASSERT(loc != nullptr); + return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + } + + ///////////////////// + // scan functions + ///////////////////// + + /*! + @brief get codepoint from 4 hex characters following `\u` + + For input "\u c1 c2 c3 c4" the codepoint is: + (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4 + = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0) + + Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f' + must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The + conversion is done by subtracting the offset (0x30, 0x37, and 0x57) + between the ASCII value of the character and the desired integer value. + + @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or + non-hex character) + */ + int get_codepoint() + { + // this function only makes sense after reading `\u` + JSON_ASSERT(current == 'u'); + int codepoint = 0; + + const auto factors = { 12u, 8u, 4u, 0u }; + for (const auto factor : factors) + { + get(); + + if (current >= '0' && current <= '9') + { + codepoint += static_cast((static_cast(current) - 0x30u) << factor); + } + else if (current >= 'A' && current <= 'F') + { + codepoint += static_cast((static_cast(current) - 0x37u) << factor); + } + else if (current >= 'a' && current <= 'f') + { + codepoint += static_cast((static_cast(current) - 0x57u) << factor); + } + else + { + return -1; + } + } + + JSON_ASSERT(0x0000 <= codepoint && codepoint <= 0xFFFF); + return codepoint; + } + + /*! + @brief check if the next byte(s) are inside a given range + + Adds the current byte and, for each passed range, reads a new byte and + checks if it is inside the range. If a violation was detected, set up an + error message and return false. Otherwise, return true. + + @param[in] ranges list of integers; interpreted as list of pairs of + inclusive lower and upper bound, respectively + + @pre The passed list @a ranges must have 2, 4, or 6 elements; that is, + 1, 2, or 3 pairs. This precondition is enforced by an assertion. + + @return true if and only if no range violation was detected + */ + bool next_byte_in_range(std::initializer_list ranges) + { + JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); + add(current); + + for (auto range = ranges.begin(); range != ranges.end(); ++range) + { + get(); + if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) + { + add(current); + } + else + { + error_message = "invalid string: ill-formed UTF-8 byte"; + return false; + } + } + + return true; + } + + /*! + @brief scan a string literal + + This function scans a string according to Sect. 7 of RFC 8259. While + scanning, bytes are escaped and copied into buffer token_buffer. Then the + function returns successfully, token_buffer is *not* null-terminated (as it + may contain \0 bytes), and token_buffer.size() is the number of bytes in the + string. + + @return token_type::value_string if string could be successfully scanned, + token_type::parse_error otherwise + + @note In case of errors, variable error_message contains a textual + description. + */ + token_type scan_string() + { + // reset token_buffer (ignore opening quote) + reset(); + + // we entered the function by reading an open quote + JSON_ASSERT(current == '\"'); + + while (true) + { + // get next character + switch (get()) + { + // end of file while parsing string + case std::char_traits::eof(): + { + error_message = "invalid string: missing closing quote"; + return token_type::parse_error; + } + + // closing quote + case '\"': + { + return token_type::value_string; + } + + // escapes + case '\\': + { + switch (get()) + { + // quotation mark + case '\"': + add('\"'); + break; + // reverse solidus + case '\\': + add('\\'); + break; + // solidus + case '/': + add('/'); + break; + // backspace + case 'b': + add('\b'); + break; + // form feed + case 'f': + add('\f'); + break; + // line feed + case 'n': + add('\n'); + break; + // carriage return + case 'r': + add('\r'); + break; + // tab + case 't': + add('\t'); + break; + + // unicode escapes + case 'u': + { + const int codepoint1 = get_codepoint(); + int codepoint = codepoint1; // start with codepoint1 + + if (JSON_HEDLEY_UNLIKELY(codepoint1 == -1)) + { + error_message = "invalid string: '\\u' must be followed by 4 hex digits"; + return token_type::parse_error; + } + + // check if code point is a high surrogate + if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF) + { + // expect next \uxxxx entry + if (JSON_HEDLEY_LIKELY(get() == '\\' && get() == 'u')) + { + const int codepoint2 = get_codepoint(); + + if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1)) + { + error_message = "invalid string: '\\u' must be followed by 4 hex digits"; + return token_type::parse_error; + } + + // check if codepoint2 is a low surrogate + if (JSON_HEDLEY_LIKELY(0xDC00 <= codepoint2 && codepoint2 <= 0xDFFF)) + { + // overwrite codepoint + codepoint = static_cast( + // high surrogate occupies the most significant 22 bits + (static_cast(codepoint1) << 10u) + // low surrogate occupies the least significant 15 bits + + static_cast(codepoint2) + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result, so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00u); + } + else + { + error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + return token_type::parse_error; + } + } + else + { + error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + return token_type::parse_error; + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF)) + { + error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF"; + return token_type::parse_error; + } + } + + // result of the above calculation yields a proper codepoint + JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); + + // translate codepoint into bytes + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + add(static_cast(codepoint)); + } + else if (codepoint <= 0x7FF) + { + // 2-byte characters: 110xxxxx 10xxxxxx + add(static_cast(0xC0u | (static_cast(codepoint) >> 6u))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + else if (codepoint <= 0xFFFF) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + add(static_cast(0xE0u | (static_cast(codepoint) >> 12u))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + else + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + add(static_cast(0xF0u | (static_cast(codepoint) >> 18u))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 12u) & 0x3Fu))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + + break; + } + + // other characters after escape + default: + error_message = "invalid string: forbidden character after backslash"; + return token_type::parse_error; + } + + break; + } + + // invalid control characters + case 0x00: + { + error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000"; + return token_type::parse_error; + } + + case 0x01: + { + error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001"; + return token_type::parse_error; + } + + case 0x02: + { + error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002"; + return token_type::parse_error; + } + + case 0x03: + { + error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003"; + return token_type::parse_error; + } + + case 0x04: + { + error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004"; + return token_type::parse_error; + } + + case 0x05: + { + error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005"; + return token_type::parse_error; + } + + case 0x06: + { + error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006"; + return token_type::parse_error; + } + + case 0x07: + { + error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007"; + return token_type::parse_error; + } + + case 0x08: + { + error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b"; + return token_type::parse_error; + } + + case 0x09: + { + error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t"; + return token_type::parse_error; + } + + case 0x0A: + { + error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n"; + return token_type::parse_error; + } + + case 0x0B: + { + error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B"; + return token_type::parse_error; + } + + case 0x0C: + { + error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f"; + return token_type::parse_error; + } + + case 0x0D: + { + error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r"; + return token_type::parse_error; + } + + case 0x0E: + { + error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E"; + return token_type::parse_error; + } + + case 0x0F: + { + error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F"; + return token_type::parse_error; + } + + case 0x10: + { + error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010"; + return token_type::parse_error; + } + + case 0x11: + { + error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011"; + return token_type::parse_error; + } + + case 0x12: + { + error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012"; + return token_type::parse_error; + } + + case 0x13: + { + error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013"; + return token_type::parse_error; + } + + case 0x14: + { + error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014"; + return token_type::parse_error; + } + + case 0x15: + { + error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015"; + return token_type::parse_error; + } + + case 0x16: + { + error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016"; + return token_type::parse_error; + } + + case 0x17: + { + error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017"; + return token_type::parse_error; + } + + case 0x18: + { + error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018"; + return token_type::parse_error; + } + + case 0x19: + { + error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019"; + return token_type::parse_error; + } + + case 0x1A: + { + error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A"; + return token_type::parse_error; + } + + case 0x1B: + { + error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B"; + return token_type::parse_error; + } + + case 0x1C: + { + error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C"; + return token_type::parse_error; + } + + case 0x1D: + { + error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D"; + return token_type::parse_error; + } + + case 0x1E: + { + error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E"; + return token_type::parse_error; + } + + case 0x1F: + { + error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F"; + return token_type::parse_error; + } + + // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace)) + case 0x20: + case 0x21: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + { + add(current); + break; + } + + // U+0080..U+07FF: bytes C2..DF 80..BF + case 0xC2: + case 0xC3: + case 0xC4: + case 0xC5: + case 0xC6: + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD5: + case 0xD6: + case 0xD7: + case 0xD8: + case 0xD9: + case 0xDA: + case 0xDB: + case 0xDC: + case 0xDD: + case 0xDE: + case 0xDF: + { + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + { + return token_type::parse_error; + } + break; + } + + // U+0800..U+0FFF: bytes E0 A0..BF 80..BF + case 0xE0: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF + // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xEE: + case 0xEF: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+D000..U+D7FF: bytes ED 80..9F 80..BF + case 0xED: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF + case 0xF0: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF + case 0xF1: + case 0xF2: + case 0xF3: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF + case 0xF4: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // remaining bytes (80..C1 and F5..FF) are ill-formed + default: + { + error_message = "invalid string: ill-formed UTF-8 byte"; + return token_type::parse_error; + } + } + } + } + + /*! + * @brief scan a comment + * @return whether comment could be scanned successfully + */ + bool scan_comment() + { + switch (get()) + { + // single-line comments skip input until a newline or EOF is read + case '/': + { + while (true) + { + switch (get()) + { + case '\n': + case '\r': + case std::char_traits::eof(): + case '\0': + return true; + + default: + break; + } + } + } + + // multi-line comments skip input until */ is read + case '*': + { + while (true) + { + switch (get()) + { + case std::char_traits::eof(): + case '\0': + { + error_message = "invalid comment; missing closing '*/'"; + return false; + } + + case '*': + { + switch (get()) + { + case '/': + return true; + + default: + { + unget(); + continue; + } + } + } + + default: + continue; + } + } + } + + // unexpected character after reading '/' + default: + { + error_message = "invalid comment; expecting '/' or '*' after '/'"; + return false; + } + } + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(float& f, const char* str, char** endptr) noexcept + { + f = std::strtof(str, endptr); + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(double& f, const char* str, char** endptr) noexcept + { + f = std::strtod(str, endptr); + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(long double& f, const char* str, char** endptr) noexcept + { + f = std::strtold(str, endptr); + } + + /*! + @brief scan a number literal + + This function scans a string according to Sect. 6 of RFC 8259. + + The function is realized with a deterministic finite state machine derived + from the grammar described in RFC 8259. Starting in state "init", the + input is read and used to determined the next state. Only state "done" + accepts the number. State "error" is a trap state to model errors. In the + table below, "anything" means any character but the ones listed before. + + state | 0 | 1-9 | e E | + | - | . | anything + ---------|----------|----------|----------|---------|---------|----------|----------- + init | zero | any1 | [error] | [error] | minus | [error] | [error] + minus | zero | any1 | [error] | [error] | [error] | [error] | [error] + zero | done | done | exponent | done | done | decimal1 | done + any1 | any1 | any1 | exponent | done | done | decimal1 | done + decimal1 | decimal2 | decimal2 | [error] | [error] | [error] | [error] | [error] + decimal2 | decimal2 | decimal2 | exponent | done | done | done | done + exponent | any2 | any2 | [error] | sign | sign | [error] | [error] + sign | any2 | any2 | [error] | [error] | [error] | [error] | [error] + any2 | any2 | any2 | done | done | done | done | done + + The state machine is realized with one label per state (prefixed with + "scan_number_") and `goto` statements between them. The state machine + contains cycles, but any cycle can be left when EOF is read. Therefore, + the function is guaranteed to terminate. + + During scanning, the read bytes are stored in token_buffer. This string is + then converted to a signed integer, an unsigned integer, or a + floating-point number. + + @return token_type::value_unsigned, token_type::value_integer, or + token_type::value_float if number could be successfully scanned, + token_type::parse_error otherwise + + @note The scanner is independent of the current locale. Internally, the + locale's decimal point is used instead of `.` to work with the + locale-dependent converters. + */ + token_type scan_number() // lgtm [cpp/use-of-goto] + { + // reset token_buffer to store the number's bytes + reset(); + + // the type of the parsed number; initially set to unsigned; will be + // changed if minus sign, decimal point or exponent is read + token_type number_type = token_type::value_unsigned; + + // state (init): we just found out we need to scan a number + switch (current) + { + case '-': + { + add(current); + goto scan_number_minus; + } + + case '0': + { + add(current); + goto scan_number_zero; + } + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + // all other characters are rejected outside scan_number() + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + +scan_number_minus: + // state: we just parsed a leading minus sign + number_type = token_type::value_integer; + switch (get()) + { + case '0': + { + add(current); + goto scan_number_zero; + } + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + default: + { + error_message = "invalid number; expected digit after '-'"; + return token_type::parse_error; + } + } + +scan_number_zero: + // state: we just parse a zero (maybe with a leading minus sign) + switch (get()) + { + case '.': + { + add(decimal_point_char); + goto scan_number_decimal1; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_any1: + // state: we just parsed a number 0-9 (maybe with a leading minus sign) + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + case '.': + { + add(decimal_point_char); + goto scan_number_decimal1; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_decimal1: + // state: we just parsed a decimal point + number_type = token_type::value_float; + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_decimal2; + } + + default: + { + error_message = "invalid number; expected digit after '.'"; + return token_type::parse_error; + } + } + +scan_number_decimal2: + // we just parsed at least one number after a decimal point + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_decimal2; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_exponent: + // we just parsed an exponent + number_type = token_type::value_float; + switch (get()) + { + case '+': + case '-': + { + add(current); + goto scan_number_sign; + } + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + { + error_message = + "invalid number; expected '+', '-', or digit after exponent"; + return token_type::parse_error; + } + } + +scan_number_sign: + // we just parsed an exponent sign + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + { + error_message = "invalid number; expected digit after exponent sign"; + return token_type::parse_error; + } + } + +scan_number_any2: + // we just parsed a number after the exponent or exponent sign + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + goto scan_number_done; + } + +scan_number_done: + // unget the character after the number (we only read it to know that + // we are done scanning a number) + unget(); + + char* endptr = nullptr; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + errno = 0; + + // try to parse integers first and fall back to floats + if (number_type == token_type::value_unsigned) + { + const auto x = std::strtoull(token_buffer.data(), &endptr, 10); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + if (errno == 0) + { + value_unsigned = static_cast(x); + if (value_unsigned == x) + { + return token_type::value_unsigned; + } + } + } + else if (number_type == token_type::value_integer) + { + const auto x = std::strtoll(token_buffer.data(), &endptr, 10); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + if (errno == 0) + { + value_integer = static_cast(x); + if (value_integer == x) + { + return token_type::value_integer; + } + } + } + + // this code is reached if we parse a floating-point number or if an + // integer conversion above failed + strtof(value_float, token_buffer.data(), &endptr); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + return token_type::value_float; + } + + /*! + @param[in] literal_text the literal text to expect + @param[in] length the length of the passed literal text + @param[in] return_type the token type to return on success + */ + JSON_HEDLEY_NON_NULL(2) + token_type scan_literal(const char_type* literal_text, const std::size_t length, + token_type return_type) + { + JSON_ASSERT(std::char_traits::to_char_type(current) == literal_text[0]); + for (std::size_t i = 1; i < length; ++i) + { + if (JSON_HEDLEY_UNLIKELY(std::char_traits::to_char_type(get()) != literal_text[i])) + { + error_message = "invalid literal"; + return token_type::parse_error; + } + } + return return_type; + } + + ///////////////////// + // input management + ///////////////////// + + /// reset token_buffer; current character is beginning of token + void reset() noexcept + { + token_buffer.clear(); + token_string.clear(); + token_string.push_back(std::char_traits::to_char_type(current)); + } + + /* + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a + `std::char_traits::eof()` in that case. Stores the scanned characters + for use in error messages. + + @return character read from the input + */ + char_int_type get() + { + ++position.chars_read_total; + ++position.chars_read_current_line; + + if (next_unget) + { + // just reset the next_unget variable and work with current + next_unget = false; + } + else + { + current = ia.get_character(); + } + + if (JSON_HEDLEY_LIKELY(current != std::char_traits::eof())) + { + token_string.push_back(std::char_traits::to_char_type(current)); + } + + if (current == '\n') + { + ++position.lines_read; + position.chars_read_current_line = 0; + } + + return current; + } + + /*! + @brief unget current character (read it again on next get) + + We implement unget by setting variable next_unget to true. The input is not + changed - we just simulate ungetting by modifying chars_read_total, + chars_read_current_line, and token_string. The next call to get() will + behave as if the unget character is read again. + */ + void unget() + { + next_unget = true; + + --position.chars_read_total; + + // in case we "unget" a newline, we have to also decrement the lines_read + if (position.chars_read_current_line == 0) + { + if (position.lines_read > 0) + { + --position.lines_read; + } + } + else + { + --position.chars_read_current_line; + } + + if (JSON_HEDLEY_LIKELY(current != std::char_traits::eof())) + { + JSON_ASSERT(!token_string.empty()); + token_string.pop_back(); + } + } + + /// add a character to token_buffer + void add(char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + public: + ///////////////////// + // value getters + ///////////////////// + + /// return integer value + constexpr number_integer_t get_number_integer() const noexcept + { + return value_integer; + } + + /// return unsigned integer value + constexpr number_unsigned_t get_number_unsigned() const noexcept + { + return value_unsigned; + } + + /// return floating-point value + constexpr number_float_t get_number_float() const noexcept + { + return value_float; + } + + /// return current string value (implicitly resets the token; useful only once) + string_t& get_string() + { + return token_buffer; + } + + ///////////////////// + // diagnostics + ///////////////////// + + /// return position of last read token + constexpr position_t get_position() const noexcept + { + return position; + } + + /// return the last read token (for errors only). Will never contain EOF + /// (an arbitrary value that is not a valid char value, often -1), because + /// 255 may legitimately occur. May contain NUL, which should be escaped. + std::string get_token_string() const + { + // escape control characters + std::string result; + for (const auto c : token_string) + { + if (static_cast(c) <= '\x1F') + { + // escape control characters + std::array cs{{}}; + static_cast((std::snprintf)(cs.data(), cs.size(), "", static_cast(c))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + result += cs.data(); + } + else + { + // add character as is + result.push_back(static_cast(c)); + } + } + + return result; + } + + /// return syntax error message + JSON_HEDLEY_RETURNS_NON_NULL + constexpr const char* get_error_message() const noexcept + { + return error_message; + } + + ///////////////////// + // actual scanner + ///////////////////// + + /*! + @brief skip the UTF-8 byte order mark + @return true iff there is no BOM or the correct BOM has been skipped + */ + bool skip_bom() + { + if (get() == 0xEF) + { + // check if we completely parse the BOM + return get() == 0xBB && get() == 0xBF; + } + + // the first character is not the beginning of the BOM; unget it to + // process is later + unget(); + return true; + } + + void skip_whitespace() + { + do + { + get(); + } + while (current == ' ' || current == '\t' || current == '\n' || current == '\r'); + } + + token_type scan() + { + // initially, skip the BOM + if (position.chars_read_total == 0 && !skip_bom()) + { + error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; + return token_type::parse_error; + } + + // read next character and ignore whitespace + skip_whitespace(); + + // ignore comments + while (ignore_comments && current == '/') + { + if (!scan_comment()) + { + return token_type::parse_error; + } + + // skip following whitespace + skip_whitespace(); + } + + switch (current) + { + // structural characters + case '[': + return token_type::begin_array; + case ']': + return token_type::end_array; + case '{': + return token_type::begin_object; + case '}': + return token_type::end_object; + case ':': + return token_type::name_separator; + case ',': + return token_type::value_separator; + + // literals + case 't': + { + std::array true_literal = {{static_cast('t'), static_cast('r'), static_cast('u'), static_cast('e')}}; + return scan_literal(true_literal.data(), true_literal.size(), token_type::literal_true); + } + case 'f': + { + std::array false_literal = {{static_cast('f'), static_cast('a'), static_cast('l'), static_cast('s'), static_cast('e')}}; + return scan_literal(false_literal.data(), false_literal.size(), token_type::literal_false); + } + case 'n': + { + std::array null_literal = {{static_cast('n'), static_cast('u'), static_cast('l'), static_cast('l')}}; + return scan_literal(null_literal.data(), null_literal.size(), token_type::literal_null); + } + + // string + case '\"': + return scan_string(); + + // number + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return scan_number(); + + // end of input (the null byte is needed when parsing from + // string literals) + case '\0': + case std::char_traits::eof(): + return token_type::end_of_input; + + // error + default: + error_message = "invalid literal"; + return token_type::parse_error; + } + } + + private: + /// input adapter + InputAdapterType ia; + + /// whether comments should be ignored (true) or signaled as errors (false) + const bool ignore_comments = false; + + /// the current character + char_int_type current = std::char_traits::eof(); + + /// whether the next get() call should just return current + bool next_unget = false; + + /// the start position of the current token + position_t position {}; + + /// raw input token string (for error messages) + std::vector token_string {}; + + /// buffer for variable-length tokens (numbers, strings) + string_t token_buffer {}; + + /// a description of occurred lexer errors + const char* error_message = ""; + + // number values + number_integer_t value_integer = 0; + number_unsigned_t value_unsigned = 0; + number_float_t value_float = 0; + + /// the decimal point + const char_int_type decimal_point_char = '.'; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // size_t +#include // declval +#include // string + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +using null_function_t = decltype(std::declval().null()); + +template +using boolean_function_t = + decltype(std::declval().boolean(std::declval())); + +template +using number_integer_function_t = + decltype(std::declval().number_integer(std::declval())); + +template +using number_unsigned_function_t = + decltype(std::declval().number_unsigned(std::declval())); + +template +using number_float_function_t = decltype(std::declval().number_float( + std::declval(), std::declval())); + +template +using string_function_t = + decltype(std::declval().string(std::declval())); + +template +using binary_function_t = + decltype(std::declval().binary(std::declval())); + +template +using start_object_function_t = + decltype(std::declval().start_object(std::declval())); + +template +using key_function_t = + decltype(std::declval().key(std::declval())); + +template +using end_object_function_t = decltype(std::declval().end_object()); + +template +using start_array_function_t = + decltype(std::declval().start_array(std::declval())); + +template +using end_array_function_t = decltype(std::declval().end_array()); + +template +using parse_error_function_t = decltype(std::declval().parse_error( + std::declval(), std::declval(), + std::declval())); + +template +struct is_sax +{ + private: + static_assert(is_basic_json::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value; +}; + +template +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool binary(binary_t&)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +/// how to treat CBOR tags +enum class cbor_tag_handler_t +{ + error, ///< throw a parse_error exception in case of a tag + ignore, ///< ignore tags + store ///< store tags as binary type +}; + +/*! +@brief determine system byte order + +@return true if and only if system's byte order is little endian + +@note from https://stackoverflow.com/a/1001328/266378 +*/ +static inline bool little_endianness(int num = 1) noexcept +{ + return *reinterpret_cast(&num) == 1; +} + + +/////////////////// +// binary reader // +/////////////////// + +/*! +@brief deserialization of CBOR, MessagePack, and UBJSON values +*/ +template> +class binary_reader +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using json_sax_t = SAX; + using char_type = typename InputAdapterType::char_type; + using char_int_type = typename std::char_traits::int_type; + + public: + /*! + @brief create a binary reader + + @param[in] adapter input adapter to read from + */ + explicit binary_reader(InputAdapterType&& adapter) noexcept : ia(std::move(adapter)) + { + (void)detail::is_sax_static_asserts {}; + } + + // make class move-only + binary_reader(const binary_reader&) = delete; + binary_reader(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + binary_reader& operator=(const binary_reader&) = delete; + binary_reader& operator=(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~binary_reader() = default; + + /*! + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor + @param[in] strict whether to expect the input to be consumed completed + @param[in] tag_handler how to treat CBOR tags + + @return whether parsing was successful + */ + JSON_HEDLEY_NON_NULL(3) + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + sax = sax_; + bool result = false; + + switch (format) + { + case input_format_t::bson: + result = parse_bson_internal(); + break; + + case input_format_t::cbor: + result = parse_cbor_internal(true, tag_handler); + break; + + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; + + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; + + case input_format_t::json: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + + // strict mode: next byte must be EOF + if (result && strict) + { + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_HEDLEY_UNLIKELY(current != std::char_traits::eof())) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"), BasicJsonType())); + } + } + + return result; + } + + private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size{}; + get_number(input_format_t::bson, document_size); + + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in,out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast(current); + } + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in,out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"), BasicJsonType())); + } + + return get_string(input_format_t::bson, len - static_cast(1), result) && get() != std::char_traits::eof(); + } + + /*! + @brief Parses a byte array input of length @a len from the BSON input. + @param[in] len The length of the byte array to be read. + @param[in,out] result A reference to the binary variable where the read + array is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 0 + @return `true` if the byte array was successfully parsed + */ + template + bool get_bson_binary(const NumberType len, binary_t& result) + { + if (JSON_HEDLEY_UNLIKELY(len < 0)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "byte array length cannot be negative, is " + std::to_string(len), "binary"), BasicJsonType())); + } + + // All BSON binary values have a subtype + std::uint8_t subtype{}; + get_number(input_format_t::bson, subtype); + result.set_subtype(subtype); + + return get_binary(input_format_t::bson, len, result); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const char_int_type element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number{}; + return get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), ""); + } + + case 0x02: // string + { + std::int32_t len{}; + string_t value; + return get_number(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x05: // binary + { + std::int32_t len{}; + binary_t value; + return get_number(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value{}; + return get_number(input_format_t::bson, value) && sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value{}; + return get_number(input_format_t::bson, value) && sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + std::array cr{{}}; + static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + return sax->parse_error(element_type_parse_position, std::string(cr.data()), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr.data()), BasicJsonType())); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + + while (auto element_type = get()) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + if (!is_array && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size{}; + get_number(input_format_t::bson, document_size); + + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character should + be considered instead (false) + @param[in] tag_handler how CBOR tags should be treated + + @return whether a valid CBOR value was passed to the SAX parser + */ + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) + { + switch (get_char ? get() : current) + { + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::cbor, "value"); + + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + return sax->number_unsigned(static_cast(current)); + + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + std::uint8_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + std::uint16_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x1A: // Unsigned integer (four-byte uint32_t follows) + { + std::uint32_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) + { + std::uint64_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + return sax->number_integer(static_cast(0x20 - 1 - current)); + + case 0x38: // Negative integer (one-byte uint8_t follows) + { + std::uint8_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + { + std::uint16_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) + { + std::uint32_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) + { + std::uint64_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) + - static_cast(number)); + } + + // Binary data (0x00..0x17 bytes follow) + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: // Binary data (one-byte uint8_t for n follows) + case 0x59: // Binary data (two-byte uint16_t for n follow) + case 0x5A: // Binary data (four-byte uint32_t for n follow) + case 0x5B: // Binary data (eight-byte uint64_t for n follow) + case 0x5F: // Binary data (indefinite length) + { + binary_t b; + return get_cbor_binary(b) && sax->binary(b); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) + { + string_t s; + return get_cbor_string(s) && sax->string(s); + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + return get_cbor_array(static_cast(static_cast(current) & 0x1Fu), tag_handler); + + case 0x98: // array (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x99: // array (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x9A: // array (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x9B: // array (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(detail::conditional_static_cast(len), tag_handler); + } + + case 0x9F: // array (indefinite length) + return get_cbor_array(static_cast(-1), tag_handler); + + // map (0x00..0x17 pairs of data items follow) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return get_cbor_object(static_cast(static_cast(current) & 0x1Fu), tag_handler); + + case 0xB8: // map (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xB9: // map (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xBA: // map (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xBB: // map (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(detail::conditional_static_cast(len), tag_handler); + } + + case 0xBF: // map (indefinite length) + return get_cbor_object(static_cast(-1), tag_handler); + + case 0xC6: // tagged item + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD8: // tagged item (1 bytes follow) + case 0xD9: // tagged item (2 bytes follow) + case 0xDA: // tagged item (4 bytes follow) + case 0xDB: // tagged item (8 bytes follow) + { + switch (tag_handler) + { + case cbor_tag_handler_t::error: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + + case cbor_tag_handler_t::ignore: + { + // ignore binary subtype + switch (current) + { + case 0xD8: + { + std::uint8_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xD9: + { + std::uint16_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDA: + { + std::uint32_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDB: + { + std::uint64_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + default: + break; + } + return parse_cbor_internal(true, tag_handler); + } + + case cbor_tag_handler_t::store: + { + binary_t b; + // use binary subtype and store in binary container + switch (current) + { + case 0xD8: + { + std::uint8_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xD9: + { + std::uint16_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDA: + { + std::uint32_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDB: + { + std::uint64_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + default: + return parse_cbor_internal(true, tag_handler); + } + get(); + return get_cbor_binary(b) && sax->binary(b); + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } + } + + case 0xF4: // false + return sax->boolean(false); + + case 0xF5: // true + return sax->boolean(true); + + case 0xF6: // null + return sax->null(); + + case 0xF9: // Half-Precision Float (two-byte IEEE 754) + { + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte1 << 8u) + byte2); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), ""); + } + + case 0xFA: // Single-Precision Float (four-byte IEEE 754) + { + float number{}; + return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + } + + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) + { + double number{}; + return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + } + + default: // anything else (0xFF is handled inside the other types) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + return get_string(input_format_t::cbor, static_cast(current) & 0x1Fu, result); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7F: // UTF-8 string (indefinite length) + { + while (get() != 0xFF) + { + string_t chunk; + if (!get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + } + + /*! + @brief reads a CBOR byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into the byte array. + Additionally, CBOR's byte arrays with indefinite lengths are supported. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_cbor_binary(binary_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) + { + return false; + } + + switch (current) + { + // Binary data (0x00..0x17 bytes follow) + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + { + return get_binary(input_format_t::cbor, static_cast(current) & 0x1Fu, result); + } + + case 0x58: // Binary data (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x59: // Binary data (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5A: // Binary data (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5B: // Binary data (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5F: // Binary data (indefinite length) + { + while (get() != 0xFF) + { + binary_t chunk; + if (!get_cbor_binary(chunk)) + { + return false; + } + result.insert(result.end(), chunk.begin(), chunk.end()); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x" + last_token, "binary"), BasicJsonType())); + } + } + } + + /*! + @param[in] len the length of the array or static_cast(-1) for an + array of indefinite size + @param[in] tag_handler how CBOR tags should be treated + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len, + const cbor_tag_handler_t tag_handler) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + if (len != static_cast(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + } + } + else + { + while (get() != 0xFF) + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + { + return false; + } + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object or static_cast(-1) for an + object of indefinite size + @param[in] tag_handler how CBOR tags should be treated + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len, + const cbor_tag_handler_t tag_handler) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + if (len != 0) + { + string_t key; + if (len != static_cast(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + key.clear(); + } + } + } + + return sax->end_object(); + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() + { + switch (get()) + { + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); + + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + return sax->number_unsigned(static_cast(current)); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + return get_msgpack_object(static_cast(static_cast(current) & 0x0Fu)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + return get_msgpack_array(static_cast(static_cast(current) & 0x0Fu)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + return get_msgpack_string(s) && sax->string(s); + } + + case 0xC0: // nil + return sax->null(); + + case 0xC2: // false + return sax->boolean(false); + + case 0xC3: // true + return sax->boolean(true); + + case 0xC4: // bin 8 + case 0xC5: // bin 16 + case 0xC6: // bin 32 + case 0xC7: // ext 8 + case 0xC8: // ext 16 + case 0xC9: // ext 32 + case 0xD4: // fixext 1 + case 0xD5: // fixext 2 + case 0xD6: // fixext 4 + case 0xD7: // fixext 8 + case 0xD8: // fixext 16 + { + binary_t b; + return get_msgpack_binary(b) && sax->binary(b); + } + + case 0xCA: // float 32 + { + float number{}; + return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + } + + case 0xCB: // float 64 + { + double number{}; + return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + } + + case 0xCC: // uint 8 + { + std::uint8_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCD: // uint 16 + { + std::uint16_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCE: // uint 32 + { + std::uint32_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCF: // uint 64 + { + std::uint64_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xD0: // int 8 + { + std::int8_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD1: // int 16 + { + std::int16_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD2: // int 32 + { + std::int32_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD3: // int 64 + { + std::int64_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xDC: // array 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + } + + case 0xDD: // array 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + } + + case 0xDE: // map 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + } + + case 0xDF: // map 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + } + + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + return sax->number_integer(static_cast(current)); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @brief reads a MessagePack string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_msgpack_string(string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::msgpack, "string"))) + { + return false; + } + + switch (current) + { + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(input_format_t::msgpack, static_cast(current) & 0x1Fu, result); + } + + case 0xD9: // str 8 + { + std::uint8_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + case 0xDA: // str 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + case 0xDB: // str 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + } + + /*! + @brief reads a MessagePack byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into a byte array. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_msgpack_binary(binary_t& result) + { + // helper function to set the subtype + auto assign_and_return_true = [&result](std::int8_t subtype) + { + result.set_subtype(static_cast(subtype)); + return true; + }; + + switch (current) + { + case 0xC4: // bin 8 + { + std::uint8_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC5: // bin 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC6: // bin 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC7: // ext 8 + { + std::uint8_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xC8: // ext 16 + { + std::uint16_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xC9: // ext 32 + { + std::uint32_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xD4: // fixext 1 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 1, result) && + assign_and_return_true(subtype); + } + + case 0xD5: // fixext 2 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 2, result) && + assign_and_return_true(subtype); + } + + case 0xD6: // fixext 4 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 4, result) && + assign_and_return_true(subtype); + } + + case 0xD7: // fixext 8 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 8, result) && + assign_and_return_true(subtype); + } + + case 0xD8: // fixext 16 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 16, result) && + assign_and_return_true(subtype); + } + + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } + } + + /*! + @param[in] len the length of the array + @return whether array creation completed + */ + bool get_msgpack_array(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + { + return false; + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + string_t key; + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + { + return false; + } + key.clear(); + } + + return sax->end_object(); + } + + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser + */ + bool parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + + /*! + @brief reads a UBJSON string + + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. + + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether string creation completed + */ + bool get_ubjson_string(string_t& result, const bool get_char = true) + { + if (get_char) + { + get(); // TODO(niels): may we ignore N here? + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + + switch (current) + { + case 'U': + { + std::uint8_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'i': + { + std::int8_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'I': + { + std::int16_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'l': + { + std::int32_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'L': + { + std::int64_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': + { + std::uint8_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'i': + { + std::int8_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char + return true; + } + + case 'I': + { + std::int16_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'l': + { + std::int32_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'L': + { + std::int64_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"), BasicJsonType())); + } + } + } + + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair& result) + { + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') + { + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "type"))) + { + return false; + } + + get_ignore_noop(); + if (JSON_HEDLEY_UNLIKELY(current != '#')) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"), BasicJsonType())); + } + + return get_ubjson_size_value(result.first); + } + + if (current == '#') + { + return get_ubjson_size_value(result.first); + } + + return true; + } + + /*! + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const char_int_type prefix) + { + switch (prefix) + { + case std::char_traits::eof(): // EOF + return unexpect_eof(input_format_t::ubjson, "value"); + + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); + + case 'U': + { + std::uint8_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_unsigned(number); + } + + case 'i': + { + std::int8_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'I': + { + std::int16_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'l': + { + std::int32_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'L': + { + std::int64_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'd': + { + float number{}; + return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + } + + case 'D': + { + double number{}; + return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + } + + case 'H': + { + return get_ubjson_high_precision_number(); + } + + case 'C': // char + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "char"))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"), BasicJsonType())); + } + string_t s(1, static_cast(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) && sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @return whether array creation completed + */ + bool get_ubjson_array() + { + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + } + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + while (current != ']') + { + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); + } + } + + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); + } + } + + return sax->end_object(); + } + + // Note, no reader for UBJSON binary types is implemented because they do + // not exist + + bool get_ubjson_high_precision_number() + { + // get size of following number string + std::size_t size{}; + auto res = get_ubjson_size_value(size); + if (JSON_HEDLEY_UNLIKELY(!res)) + { + return res; + } + + // get number string + std::vector number_vector; + for (std::size_t i = 0; i < size; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "number"))) + { + return false; + } + number_vector.push_back(static_cast(current)); + } + + // parse number string + using ia_type = decltype(detail::input_adapter(number_vector)); + auto number_lexer = detail::lexer(detail::input_adapter(number_vector), false); + const auto result_number = number_lexer.scan(); + const auto number_string = number_lexer.get_token_string(); + const auto result_remainder = number_lexer.scan(); + + using token_type = typename detail::lexer_base::token_type; + + if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input)) + { + return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, exception_message(input_format_t::ubjson, "invalid number text: " + number_lexer.get_token_string(), "high-precision number"), BasicJsonType())); + } + + switch (result_number) + { + case token_type::value_integer: + return sax->number_integer(number_lexer.get_number_integer()); + case token_type::value_unsigned: + return sax->number_unsigned(number_lexer.get_number_unsigned()); + case token_type::value_float: + return sax->number_float(number_lexer.get_number_float(), std::move(number_string)); + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::begin_array: + case token_type::begin_object: + case token_type::end_array: + case token_type::end_object: + case token_type::name_separator: + case token_type::value_separator: + case token_type::parse_error: + case token_type::end_of_input: + case token_type::literal_or_value: + default: + return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, exception_message(input_format_t::ubjson, "invalid number text: " + number_lexer.get_token_string(), "high-precision number"), BasicJsonType())); + } + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits::eof()` in that case. + + @return character read from the input + */ + char_int_type get() + { + ++chars_read; + return current = ia.get_character(); + } + + /*! + @return character read from the input after ignoring all 'N' entries + */ + char_int_type get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianness, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. + */ + template + bool get_number(const input_format_t format, NumberType& result) + { + // step 1: read input into array with system's byte order + std::array vec{}; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "number"))) + { + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian != InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast(current); + } + else + { + vec[i] = static_cast(current); // LCOV_EXCL_LINE + } + } + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + for (NumberType i = 0; i < len; i++) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string"))) + { + success = false; + break; + } + result.push_back(static_cast(current)); + } + return success; + } + + /*! + @brief create a byte array by reading bytes from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of bytes to read + @param[out] result byte array created by reading @a len bytes + + @return whether byte array creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of memory. + */ + template + bool get_binary(const input_format_t format, + const NumberType len, + binary_t& result) + { + bool success = true; + for (NumberType i = 0; i < len; i++) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary"))) + { + success = false; + break; + } + result.push_back(static_cast(current)); + } + return success; + } + + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) + @return whether the last read character is not EOF + */ + JSON_HEDLEY_NON_NULL(3) + bool unexpect_eof(const input_format_t format, const char* context) const + { + if (JSON_HEDLEY_UNLIKELY(current == std::char_traits::eof())) + { + return sax->parse_error(chars_read, "", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), BasicJsonType())); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + std::array cr{{}}; + static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(current))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + return std::string{cr.data()}; + } + + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further context information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + case input_format_t::json: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + + return error_msg + " " + context + ": " + detail; + } + + private: + /// input adapter + InputAdapterType ia; + + /// the current character + char_int_type current = std::char_traits::eof(); + + /// the number of characters read + std::size_t chars_read = 0; + + /// whether we can assume little endianness + const bool is_little_endian = little_endianness(); + + /// the SAX parser + json_sax_t* sax = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + + +#include // isfinite +#include // uint8_t +#include // function +#include // string +#include // move +#include // vector + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +//////////// +// parser // +//////////// + +enum class parse_event_t : std::uint8_t +{ + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value +}; + +template +using parser_callback_t = + std::function; + +/*! +@brief syntax analysis + +This class implements a recursive descent parser. +*/ +template +class parser +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using lexer_t = lexer; + using token_type = typename lexer_t::token_type; + + public: + /// a parser reading from an input adapter + explicit parser(InputAdapterType&& adapter, + const parser_callback_t cb = nullptr, + const bool allow_exceptions_ = true, + const bool skip_comments = false) + : callback(cb) + , m_lexer(std::move(adapter), skip_comments) + , allow_exceptions(allow_exceptions_) + { + // read first token + get_token(); + } + + /*! + @brief public parser interface + + @param[in] strict whether to expect the last token to be EOF + @param[in,out] result parsed JSON value + + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ + void parse(const bool strict, BasicJsonType& result) + { + if (callback) + { + json_sax_dom_callback_parser sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + + // in strict mode, input must be completely read + if (strict && (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + + // set top-level value to null if it was discarded by the callback + // function + if (result.is_discarded()) + { + result = nullptr; + } + } + else + { + json_sax_dom_parser sdp(result, allow_exceptions); + sax_parse_internal(&sdp); + + // in strict mode, input must be completely read + if (strict && (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + } + + result.assert_invariant(); + } + + /*! + @brief public accept interface + + @param[in] strict whether to expect the last token to be EOF + @return whether the input is a proper JSON text + */ + bool accept(const bool strict = true) + { + json_sax_acceptor sax_acceptor; + return sax_parse(&sax_acceptor, strict); + } + + template + JSON_HEDLEY_NON_NULL(2) + bool sax_parse(SAX* sax, const bool strict = true) + { + (void)detail::is_sax_static_asserts {}; + const bool result = sax_parse_internal(sax); + + // strict mode: next byte must be EOF + if (result && strict && (get_token() != token_type::end_of_input)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + return result; + } + + private: + template + JSON_HEDLEY_NON_NULL(2) + bool sax_parse_internal(SAX* sax) + { + // stack to remember the hierarchy of structured values we are parsing + // true = array; false = object + std::vector states; + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; + + while (true) + { + if (!skip_to_state_evaluation) + { + // invariant: get_token() was called before each iteration + switch (last_token) + { + case token_type::begin_object: + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + // closing } -> we are done + if (get_token() == token_type::end_object) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + break; + } + + // parse key + if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), BasicJsonType())); + } + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + { + return false; + } + + // parse separator (:) + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), BasicJsonType())); + } + + // remember we are now inside an object + states.push_back(false); + + // parse values + get_token(); + continue; + } + + case token_type::begin_array: + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + // closing ] -> we are done + if (get_token() == token_type::end_array) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + break; + } + + // remember we are now inside an array + states.push_back(true); + + // parse values (no need to call get_token) + continue; + } + + case token_type::value_float: + { + const auto res = m_lexer.get_number_float(); + + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res))) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'", BasicJsonType())); + } + + if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) + { + return false; + } + + break; + } + + case token_type::literal_false: + { + if (JSON_HEDLEY_UNLIKELY(!sax->boolean(false))) + { + return false; + } + break; + } + + case token_type::literal_null: + { + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return false; + } + break; + } + + case token_type::literal_true: + { + if (JSON_HEDLEY_UNLIKELY(!sax->boolean(true))) + { + return false; + } + break; + } + + case token_type::value_integer: + { + if (JSON_HEDLEY_UNLIKELY(!sax->number_integer(m_lexer.get_number_integer()))) + { + return false; + } + break; + } + + case token_type::value_string: + { + if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string()))) + { + return false; + } + break; + } + + case token_type::value_unsigned: + { + if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(m_lexer.get_number_unsigned()))) + { + return false; + } + break; + } + + case token_type::parse_error: + { + // using "uninitialized" to avoid "expected" message + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), BasicJsonType())); + } + + case token_type::uninitialized: + case token_type::end_array: + case token_type::end_object: + case token_type::name_separator: + case token_type::value_separator: + case token_type::end_of_input: + case token_type::literal_or_value: + default: // the last token was unexpected + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), BasicJsonType())); + } + } + } + else + { + skip_to_state_evaluation = false; + } + + // we reached this line after we successfully parsed a value + if (states.empty()) + { + // empty stack: we reached the end of the hierarchy: done + return true; + } + + if (states.back()) // array + { + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse a new value + get_token(); + continue; + } + + // closing ] + if (JSON_HEDLEY_LIKELY(last_token == token_type::end_array)) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + JSON_ASSERT(!states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), BasicJsonType())); + } + + // states.back() is false -> object + + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse key + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), BasicJsonType())); + } + + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + { + return false; + } + + // parse separator (:) + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), BasicJsonType())); + } + + // parse values + get_token(); + continue; + } + + // closing } + if (JSON_HEDLEY_LIKELY(last_token == token_type::end_object)) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + JSON_ASSERT(!states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), BasicJsonType())); + } + } + + /// get next token from lexer + token_type get_token() + { + return last_token = m_lexer.scan(); + } + + std::string exception_message(const token_type expected, const std::string& context) + { + std::string error_msg = "syntax error "; + + if (!context.empty()) + { + error_msg += "while parsing " + context + " "; + } + + error_msg += "- "; + + if (last_token == token_type::parse_error) + { + error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + + m_lexer.get_token_string() + "'"; + } + else + { + error_msg += "unexpected " + std::string(lexer_t::token_type_name(last_token)); + } + + if (expected != token_type::uninitialized) + { + error_msg += "; expected " + std::string(lexer_t::token_type_name(expected)); + } + + return error_msg; + } + + private: + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + token_type last_token = token_type::uninitialized; + /// the lexer + lexer_t m_lexer; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +} // namespace detail +} // namespace nlohmann + +// #include + + +// #include + + +#include // ptrdiff_t +#include // numeric_limits + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/* +@brief an iterator for primitive JSON types + +This class models an iterator for primitive JSON types (boolean, number, +string). It's only purpose is to allow the iterator/const_iterator classes +to "iterate" over primitive values. Internally, the iterator is modeled by +a `difference_type` variable. Value begin_value (`0`) models the begin, +end_value (`1`) models past the end. +*/ +class primitive_iterator_t +{ + private: + using difference_type = std::ptrdiff_t; + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + JSON_PRIVATE_UNLESS_TESTED: + /// iterator as signed integer type + difference_type m_it = (std::numeric_limits::min)(); + + public: + constexpr difference_type get_value() const noexcept + { + return m_it; + } + + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return m_it == begin_value; + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return m_it == end_value; + } + + friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it == rhs.m_it; + } + + friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it < rhs.m_it; + } + + primitive_iterator_t operator+(difference_type n) noexcept + { + auto result = *this; + result += n; + return result; + } + + friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it - rhs.m_it; + } + + primitive_iterator_t& operator++() noexcept + { + ++m_it; + return *this; + } + + primitive_iterator_t const operator++(int) noexcept // NOLINT(readability-const-return-type) + { + auto result = *this; + ++m_it; + return result; + } + + primitive_iterator_t& operator--() noexcept + { + --m_it; + return *this; + } + + primitive_iterator_t const operator--(int) noexcept // NOLINT(readability-const-return-type) + { + auto result = *this; + --m_it; + return result; + } + + primitive_iterator_t& operator+=(difference_type n) noexcept + { + m_it += n; + return *this; + } + + primitive_iterator_t& operator-=(difference_type n) noexcept + { + m_it -= n; + return *this; + } +}; +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +/*! +@brief an iterator value + +@note This structure could easily be a union, but MSVC currently does not allow +unions members with complex constructors, see https://github.com/nlohmann/json/pull/105. +*/ +template struct internal_iterator +{ + /// iterator for JSON objects + typename BasicJsonType::object_t::iterator object_iterator {}; + /// iterator for JSON arrays + typename BasicJsonType::array_t::iterator array_iterator {}; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator {}; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next +#include // conditional, is_const, remove_const + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +// forward declare, to be able to friend it later on +template class iteration_proxy; +template class iteration_proxy_value; + +/*! +@brief a template for a bidirectional iterator for the @ref basic_json class +This class implements a both iterators (iterator and const_iterator) for the +@ref basic_json class. +@note An iterator is called *initialized* when a pointer to a JSON value has + been set (e.g., by a constructor or a copy assignment). If the iterator is + default-constructed, it is *uninitialized* and most methods are undefined. + **The library uses assertions to detect calls on uninitialized iterators.** +@requirement The class satisfies the following concept requirements: +- +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): + The iterator that can be moved can be moved in both directions (i.e. + incremented and decremented). +@since version 1.0.0, simplified in version 2.0.9, change to bidirectional + iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593) +*/ +template +class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +{ + /// the iterator with BasicJsonType of different const-ness + using other_iter_impl = iter_impl::value, typename std::remove_const::type, const BasicJsonType>::type>; + /// allow basic_json to access private members + friend other_iter_impl; + friend BasicJsonType; + friend iteration_proxy; + friend iteration_proxy_value; + + using object_t = typename BasicJsonType::object_t; + using array_t = typename BasicJsonType::array_t; + // make sure BasicJsonType is basic_json or const basic_json + static_assert(is_basic_json::type>::value, + "iter_impl only accepts (const) basic_json"); + + public: + + /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17. + /// The C++ Standard has never required user-defined iterators to derive from std::iterator. + /// A user-defined iterator should provide publicly accessible typedefs named + /// iterator_category, value_type, difference_type, pointer, and reference. + /// Note that value_type is required to be non-const, even for constant iterators. + using iterator_category = std::bidirectional_iterator_tag; + + /// the type of the values when the iterator is dereferenced + using value_type = typename BasicJsonType::value_type; + /// a type to represent differences between iterators + using difference_type = typename BasicJsonType::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename std::conditional::value, + typename BasicJsonType::const_pointer, + typename BasicJsonType::pointer>::type; + /// defines a reference to the type iterated over (value_type) + using reference = + typename std::conditional::value, + typename BasicJsonType::const_reference, + typename BasicJsonType::reference>::type; + + iter_impl() = default; + ~iter_impl() = default; + iter_impl(iter_impl&&) noexcept = default; + iter_impl& operator=(iter_impl&&) noexcept = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit iter_impl(pointer object) noexcept : m_object(object) + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /*! + @note The conventional copy constructor and copy assignment are implicitly + defined. Combined with the following converting constructor and + assignment, they support: (1) copy from iterator to iterator, (2) + copy from const iterator to const iterator, and (3) conversion from + iterator to const iterator. However conversion from const iterator + to iterator is not defined. + */ + + /*! + @brief const copy constructor + @param[in] other const iterator to copy from + @note This copy constructor had to be defined explicitly to circumvent a bug + occurring on msvc v19.0 compiler (VS 2015) debug build. For more + information refer to: https://github.com/nlohmann/json/issues/1608 + */ + iter_impl(const iter_impl& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief converting assignment + @param[in] other const iterator to copy from + @return const/non-const iterator + @note It is not checked whether @a other is initialized. + */ + iter_impl& operator=(const iter_impl& other) noexcept + { + if (&other != this) + { + m_object = other.m_object; + m_it = other.m_it; + } + return *this; + } + + /*! + @brief converting constructor + @param[in] other non-const iterator to copy from + @note It is not checked whether @a other is initialized. + */ + iter_impl(const iter_impl::type>& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief converting assignment + @param[in] other non-const iterator to copy from + @return const/non-const iterator + @note It is not checked whether @a other is initialized. + */ + iter_impl& operator=(const iter_impl::type>& other) noexcept // NOLINT(cert-oop54-cpp) + { + m_object = other.m_object; + m_it = other.m_it; + return *this; + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case value_t::array: + { + JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case value_t::null: + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return *m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case value_t::array: + { + JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl const operator++(int) // NOLINT(readability-const-return-type) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator++() + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl const operator--(int) // NOLINT(readability-const-return-type) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator--() + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + template < typename IterImpl, detail::enable_if_t < (std::is_same::value || std::is_same::value), std::nullptr_t > = nullptr > + bool operator==(const IterImpl& other) const + { + // if objects are not the same, the comparison is undefined + if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) + { + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers", *m_object)); + } + + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + return (m_it.object_iterator == other.m_it.object_iterator); + + case value_t::array: + return (m_it.array_iterator == other.m_it.array_iterator); + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + template < typename IterImpl, detail::enable_if_t < (std::is_same::value || std::is_same::value), std::nullptr_t > = nullptr > + bool operator!=(const IterImpl& other) const + { + return !operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const iter_impl& other) const + { + // if objects are not the same, the comparison is undefined + if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) + { + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers", *m_object)); + } + + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators", *m_object)); + + case value_t::array: + return (m_it.array_iterator < other.m_it.array_iterator); + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const iter_impl& other) const + { + return !other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const iter_impl& other) const + { + return !operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const iter_impl& other) const + { + return !operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator+=(difference_type i) + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", *m_object)); + + case value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief addition of distance and iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + friend iter_impl operator+(difference_type i, const iter_impl& it) + { + auto result = it; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const iter_impl& other) const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", *m_object)); + + case value_t::array: + return m_it.array_iterator - other.m_it.array_iterator; + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators", *m_object)); + + case value_t::array: + return *std::next(m_it.array_iterator, n); + + case value_t::null: + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.get_value() == -n)) + { + return *m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const typename object_t::key_type& key() const + { + JSON_ASSERT(m_object != nullptr); + + if (JSON_HEDLEY_LIKELY(m_object->is_object())) + { + return m_it.object_iterator->first; + } + + JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators", *m_object)); + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator::type> m_it {}; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // ptrdiff_t +#include // reverse_iterator +#include // declval + +namespace nlohmann +{ +namespace detail +{ +////////////////////// +// reverse_iterator // +////////////////////// + +/*! +@brief a template for a reverse iterator class + +@tparam Base the base iterator type to reverse. Valid types are @ref +iterator (to create @ref reverse_iterator) and @ref const_iterator (to +create @ref const_reverse_iterator). + +@requirement The class satisfies the following concept requirements: +- +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): + The iterator that can be moved can be moved in both directions (i.e. + incremented and decremented). +- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + +@since version 1.0.0 +*/ +template +class json_reverse_iterator : public std::reverse_iterator +{ + public: + using difference_type = std::ptrdiff_t; + /// shortcut to the reverse iterator adapter + using base_iterator = std::reverse_iterator; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) {} + + /// create reverse iterator from base class + explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} + + /// post-increment (it++) + json_reverse_iterator const operator++(int) // NOLINT(readability-const-return-type) + { + return static_cast(base_iterator::operator++(1)); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + return static_cast(base_iterator::operator++()); + } + + /// post-decrement (it--) + json_reverse_iterator const operator--(int) // NOLINT(readability-const-return-type) + { + return static_cast(base_iterator::operator--(1)); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + return static_cast(base_iterator::operator--()); + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + return static_cast(base_iterator::operator+=(i)); + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + return static_cast(base_iterator::operator+(i)); + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + return static_cast(base_iterator::operator-(i)); + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return base_iterator(*this) - base_iterator(other); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + auto key() const -> decltype(std::declval().key()) + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // all_of +#include // isdigit +#include // max +#include // accumulate +#include // string +#include // move +#include // vector + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ + +/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document +/// @sa https://json.nlohmann.me/api/json_pointer/ +template +class json_pointer +{ + // allow basic_json to access private members + NLOHMANN_BASIC_JSON_TPL_DECLARATION + friend class basic_json; + + public: + /// @brief create JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/json_pointer/ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /// @brief return a string representation of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ + std::string to_string() const + { + return std::accumulate(reference_tokens.begin(), reference_tokens.end(), + std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + detail::escape(b); + }); + } + + /// @brief return a string representation of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_string/ + operator std::string() const + { + return to_string(); + } + + /// @brief append another JSON pointer at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(const json_pointer& ptr) + { + reference_tokens.insert(reference_tokens.end(), + ptr.reference_tokens.begin(), + ptr.reference_tokens.end()); + return *this; + } + + /// @brief append an unescaped reference token at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(std::string token) + { + push_back(std::move(token)); + return *this; + } + + /// @brief append an array index at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(std::size_t array_idx) + { + return *this /= std::to_string(array_idx); + } + + /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, + const json_pointer& rhs) + { + return json_pointer(lhs) /= rhs; + } + + /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, std::string token) // NOLINT(performance-unnecessary-value-param) + { + return json_pointer(lhs) /= std::move(token); + } + + /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx) + { + return json_pointer(lhs) /= array_idx; + } + + /// @brief returns the parent of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/parent_pointer/ + json_pointer parent_pointer() const + { + if (empty()) + { + return *this; + } + + json_pointer res = *this; + res.pop_back(); + return res; + } + + /// @brief remove last reference token + /// @sa https://json.nlohmann.me/api/json_pointer/pop_back/ + void pop_back() + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + reference_tokens.pop_back(); + } + + /// @brief return last reference token + /// @sa https://json.nlohmann.me/api/json_pointer/back/ + const std::string& back() const + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + return reference_tokens.back(); + } + + /// @brief append an unescaped token at the end of the reference pointer + /// @sa https://json.nlohmann.me/api/json_pointer/push_back/ + void push_back(const std::string& token) + { + reference_tokens.push_back(token); + } + + /// @brief append an unescaped token at the end of the reference pointer + /// @sa https://json.nlohmann.me/api/json_pointer/push_back/ + void push_back(std::string&& token) + { + reference_tokens.push_back(std::move(token)); + } + + /// @brief return whether pointer points to the root document + /// @sa https://json.nlohmann.me/api/json_pointer/empty/ + bool empty() const noexcept + { + return reference_tokens.empty(); + } + + private: + /*! + @param[in] s reference token to be converted into an array index + + @return integer representation of @a s + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index begins not with a digit + @throw out_of_range.404 if string @a s could not be converted to an integer + @throw out_of_range.410 if an array index exceeds size_type + */ + static typename BasicJsonType::size_type array_index(const std::string& s) + { + using size_type = typename BasicJsonType::size_type; + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, "array index '" + s + "' must not begin with '0'", BasicJsonType())); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9'))) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + s + "' is not a number", BasicJsonType())); + } + + std::size_t processed_chars = 0; + unsigned long long res = 0; // NOLINT(runtime/int) + JSON_TRY + { + res = std::stoull(s, &processed_chars); + } + JSON_CATCH(std::out_of_range&) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'", BasicJsonType())); + } + + // check if the string was completely read + if (JSON_HEDLEY_UNLIKELY(processed_chars != s.size())) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'", BasicJsonType())); + } + + // only triggered on special platforms (like 32bit), see also + // https://github.com/nlohmann/json/pull/2203 + if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) + { + JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type", BasicJsonType())); // LCOV_EXCL_LINE + } + + return static_cast(res); + } + + JSON_PRIVATE_UNLESS_TESTED: + json_pointer top() const + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + private: + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + + @throw parse_error.109 if array index is not a number + @throw type_error.313 if value cannot be unflattened + */ + BasicJsonType& get_and_create(BasicJsonType& j) const + { + auto* result = &j; + + // in case no reference tokens exist, return a reference to the JSON value + // j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->type()) + { + case detail::value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case detail::value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // create an entry in the array + result = &result->operator[](array_index(reference_token)); + break; + } + + /* + The following code is only reached if there exists a reference + token _and_ the current value is primitive. In this case, we have + an error situation, because primitive values may only occur as + single value; that is, with an empty list of reference tokens. + */ + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", j)); + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @note This version does not throw if a value is not present, but tries to + create nested values instead. For instance, calling this function + with pointer `"/this/that"` on a null value is equivalent to calling + `operator[]("this").operator[]("that")` on that value, effectively + changing the null value to an object. + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + BasicJsonType& get_unchecked(BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->is_null()) + { + // check if reference token is a number + const bool nums = + std::all_of(reference_token.begin(), reference_token.end(), + [](const unsigned char x) + { + return std::isdigit(x); + }); + + // change value to array for numbers or "-" or to object otherwise + *ptr = (nums || reference_token == "-") + ? detail::value_t::array + : detail::value_t::object; + } + + switch (ptr->type()) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + ptr = &ptr->operator[](array_index(reference_token)); + } + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + BasicJsonType& get_checked(BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range", *ptr)); + } + + // note: at performs range check + ptr = &ptr->at(array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" cannot be used for const access + JSON_THROW(detail::out_of_range::create(402, "array index '-' (" + std::to_string(ptr->m_value.array->size()) + ") is out of range", *ptr)); + } + + // use unchecked array access + ptr = &ptr->operator[](array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + const BasicJsonType& get_checked(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range", *ptr)); + } + + // note: at performs range check + ptr = &ptr->at(array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + */ + bool contains(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + if (!ptr->contains(reference_token)) + { + // we did not find the key in the object + return false; + } + + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + return false; + } + if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !("0" <= reference_token && reference_token <= "9"))) + { + // invalid char + return false; + } + if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1)) + { + if (JSON_HEDLEY_UNLIKELY(!('1' <= reference_token[0] && reference_token[0] <= '9'))) + { + // first char should be between '1' and '9' + return false; + } + for (std::size_t i = 1; i < reference_token.size(); i++) + { + if (JSON_HEDLEY_UNLIKELY(!('0' <= reference_token[i] && reference_token[i] <= '9'))) + { + // other char should be between '0' and '9' + return false; + } + } + } + + const auto idx = array_index(reference_token); + if (idx >= ptr->size()) + { + // index out of range + return false; + } + + ptr = &ptr->operator[](idx); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + { + // we do not expect primitive values if there is still a + // reference token to process + return false; + } + } + } + + // no reference token left means we found a primitive value + return true; + } + + /*! + @brief split the string input to reference tokens + + @note This function is only called by the json_pointer constructor. + All exceptions below are documented there. + + @throw parse_error.107 if the pointer is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ + static std::vector split(const std::string& reference_string) + { + std::vector result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/')) + { + JSON_THROW(detail::parse_error::create(107, 1, "JSON pointer must be empty or begin with '/' - was: '" + reference_string + "'", BasicJsonType())); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + std::size_t slash = reference_string.find_first_of('/', 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == 0 (if slash == std::string::npos) + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = (slash == std::string::npos) ? 0 : slash + 1, + // find next slash + slash = reference_string.find_first_of('/', start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (std::size_t pos = reference_token.find_first_of('~'); + pos != std::string::npos; + pos = reference_token.find_first_of('~', pos + 1)) + { + JSON_ASSERT(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || + (reference_token[pos + 1] != '0' && + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'", BasicJsonType())); + } + } + + // finally, store the reference token + detail::unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + private: + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const BasicJsonType& value, + BasicJsonType& result) + { + switch (value.type()) + { + case detail::value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case detail::value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + detail::escape(element.first), element.second, result); + } + } + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + + @throw parse_error.109 if array index is not a number + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + @throw type_error.313 if value cannot be unflattened + */ + static BasicJsonType + unflatten(const BasicJsonType& value) + { + if (JSON_HEDLEY_UNLIKELY(!value.is_object())) + { + JSON_THROW(detail::type_error::create(314, "only objects can be unflattened", value)); + } + + BasicJsonType result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) + { + JSON_THROW(detail::type_error::create(315, "values in object must be primitive", element.second)); + } + + // assign value to reference pointed to by JSON pointer; Note that if + // the JSON pointer is "" (i.e., points to the whole value), function + // get_and_create returns a reference to result itself. An assignment + // will then create a primitive value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + /*! + @brief compares two JSON pointers for equality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is equal to @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return lhs.reference_tokens == rhs.reference_tokens; + } + + /*! + @brief compares two JSON pointers for inequality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is not equal @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return !(lhs == rhs); + } + + /// the reference tokens + std::vector reference_tokens; +}; +} // namespace nlohmann + +// #include + + +#include +#include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +class json_ref +{ + public: + using value_type = BasicJsonType; + + json_ref(value_type&& value) + : owned_value(std::move(value)) + {} + + json_ref(const value_type& value) + : value_ref(&value) + {} + + json_ref(std::initializer_list init) + : owned_value(init) + {} + + template < + class... Args, + enable_if_t::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward(args)...) + {} + + // class should be movable only + json_ref(json_ref&&) noexcept = default; + json_ref(const json_ref&) = delete; + json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; + + value_type moved_or_copied() const + { + if (value_ref == nullptr) + { + return std::move(owned_value); + } + return *value_ref; + } + + value_type const& operator*() const + { + return value_ref ? *value_ref : owned_value; + } + + value_type const* operator->() const + { + return &** this; + } + + private: + mutable value_type owned_value = nullptr; + value_type const* value_ref = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + +// #include + + +#include // reverse +#include // array +#include // isnan, isinf +#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // memcpy +#include // numeric_limits +#include // string +#include // move + +// #include + +// #include + +// #include + + +#include // copy +#include // size_t +#include // back_inserter +#include // shared_ptr, make_shared +#include // basic_string +#include // vector + +#ifndef JSON_NO_IO + #include // streamsize + #include // basic_ostream +#endif // JSON_NO_IO + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/// abstract output adapter interface +template struct output_adapter_protocol +{ + virtual void write_character(CharType c) = 0; + virtual void write_characters(const CharType* s, std::size_t length) = 0; + virtual ~output_adapter_protocol() = default; + + output_adapter_protocol() = default; + output_adapter_protocol(const output_adapter_protocol&) = default; + output_adapter_protocol(output_adapter_protocol&&) noexcept = default; + output_adapter_protocol& operator=(const output_adapter_protocol&) = default; + output_adapter_protocol& operator=(output_adapter_protocol&&) noexcept = default; +}; + +/// a type to simplify interfaces +template +using output_adapter_t = std::shared_ptr>; + +/// output adapter for byte vectors +template> +class output_vector_adapter : public output_adapter_protocol +{ + public: + explicit output_vector_adapter(std::vector& vec) noexcept + : v(vec) + {} + + void write_character(CharType c) override + { + v.push_back(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + std::copy(s, s + length, std::back_inserter(v)); + } + + private: + std::vector& v; +}; + +#ifndef JSON_NO_IO +/// output adapter for output streams +template +class output_stream_adapter : public output_adapter_protocol +{ + public: + explicit output_stream_adapter(std::basic_ostream& s) noexcept + : stream(s) + {} + + void write_character(CharType c) override + { + stream.put(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + stream.write(s, static_cast(length)); + } + + private: + std::basic_ostream& stream; +}; +#endif // JSON_NO_IO + +/// output adapter for basic_string +template> +class output_string_adapter : public output_adapter_protocol +{ + public: + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} + + void write_character(CharType c) override + { + str.push_back(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + str.append(s, length); + } + + private: + StringType& str; +}; + +template> +class output_adapter +{ + public: + template> + output_adapter(std::vector& vec) + : oa(std::make_shared>(vec)) {} + +#ifndef JSON_NO_IO + output_adapter(std::basic_ostream& s) + : oa(std::make_shared>(s)) {} +#endif // JSON_NO_IO + + output_adapter(StringType& s) + : oa(std::make_shared>(s)) {} + + operator output_adapter_t() + { + return oa; + } + + private: + output_adapter_t oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary writer // +/////////////////// + +/*! +@brief serialization to CBOR and MessagePack values +*/ +template +class binary_writer +{ + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using number_float_t = typename BasicJsonType::number_float_t; + + public: + /*! + @brief create a binary writer + + @param[in] adapter output adapter to write to + */ + explicit binary_writer(output_adapter_t adapter) : oa(std::move(adapter)) + { + JSON_ASSERT(oa); + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + { + write_bson_object(*j.m_value.object); + break; + } + + case value_t::null: + case value_t::array: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()), j)); + } + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_cbor(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xF6)); + break; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // CBOR does not differentiate between positive signed + // integers and unsigned integers. Therefore, we used the + // code from the value_t::number_unsigned case here. + if (j.m_value.number_integer <= 0x17) + { + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast(j.m_value.number_integer)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast(j.m_value.number_integer)); + } + } + else + { + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. + const auto positive_number = -1 - j.m_value.number_integer; + if (j.m_value.number_integer >= -24) + { + write_number(static_cast(0x20 + positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x38)); + write_number(static_cast(positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x39)); + write_number(static_cast(positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x3A)); + write_number(static_cast(positive_number)); + } + else + { + oa->write_character(to_char_type(0x3B)); + write_number(static_cast(positive_number)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= 0x17) + { + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast(j.m_value.number_unsigned)); + } + break; + } + + case value_t::number_float: + { + if (std::isnan(j.m_value.number_float)) + { + // NaN is 0xf97e00 in CBOR + oa->write_character(to_char_type(0xF9)); + oa->write_character(to_char_type(0x7E)); + oa->write_character(to_char_type(0x00)); + } + else if (std::isinf(j.m_value.number_float)) + { + // Infinity is 0xf97c00, -Infinity is 0xf9fc00 + oa->write_character(to_char_type(0xf9)); + oa->write_character(j.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa->write_character(to_char_type(0x00)); + } + else + { + write_compact_float(j.m_value.number_float, detail::input_format_t::cbor); + } + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 0x17) + { + write_number(static_cast(0x60 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x78)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x79)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x7A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x7B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write the string + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 0x17) + { + write_number(static_cast(0x80 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x98)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x99)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x9A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x9B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_cbor(el); + } + break; + } + + case value_t::binary: + { + if (j.m_value.binary->has_subtype()) + { + if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xd8)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xd9)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xda)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xdb)); + write_number(static_cast(j.m_value.binary->subtype())); + } + } + + // step 1: write control byte and the binary array size + const auto N = j.m_value.binary->size(); + if (N <= 0x17) + { + write_number(static_cast(0x40 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x58)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x59)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x5A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x5B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + N); + + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 0x17) + { + write_number(static_cast(0xA0 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xB8)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xB9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xBA)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xBB)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_cbor(el.first); + write_cbor(el.second); + } + break; + } + + case value_t::discarded: + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_msgpack(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: // nil + { + oa->write_character(to_char_type(0xC0)); + break; + } + + case value_t::boolean: // true and false + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // MessagePack does not differentiate between positive + // signed integers and unsigned integers. Therefore, we used + // the code from the value_t::number_unsigned case here. + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast(j.m_value.number_integer)); + } + } + else + { + if (j.m_value.number_integer >= -32) + { + // negative fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 8 + oa->write_character(to_char_type(0xD0)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 16 + oa->write_character(to_char_type(0xD1)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 32 + oa->write_character(to_char_type(0xD2)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 64 + oa->write_character(to_char_type(0xD3)); + write_number(static_cast(j.m_value.number_integer)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast(j.m_value.number_integer)); + } + break; + } + + case value_t::number_float: + { + write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack); + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 31) + { + // fixstr + write_number(static_cast(0xA0 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 8 + oa->write_character(to_char_type(0xD9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 16 + oa->write_character(to_char_type(0xDA)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 32 + oa->write_character(to_char_type(0xDB)); + write_number(static_cast(N)); + } + + // step 2: write the string + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 15) + { + // fixarray + write_number(static_cast(0x90 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 16 + oa->write_character(to_char_type(0xDC)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 32 + oa->write_character(to_char_type(0xDD)); + write_number(static_cast(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_msgpack(el); + } + break; + } + + case value_t::binary: + { + // step 0: determine if the binary type has a set subtype to + // determine whether or not to use the ext or fixext types + const bool use_ext = j.m_value.binary->has_subtype(); + + // step 1: write control byte and the byte string length + const auto N = j.m_value.binary->size(); + if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type{}; + bool fixed = true; + if (use_ext) + { + switch (N) + { + case 1: + output_type = 0xD4; // fixext 1 + break; + case 2: + output_type = 0xD5; // fixext 2 + break; + case 4: + output_type = 0xD6; // fixext 4 + break; + case 8: + output_type = 0xD7; // fixext 8 + break; + case 16: + output_type = 0xD8; // fixext 16 + break; + default: + output_type = 0xC7; // ext 8 + fixed = false; + break; + } + + } + else + { + output_type = 0xC4; // bin 8 + fixed = false; + } + + oa->write_character(to_char_type(output_type)); + if (!fixed) + { + write_number(static_cast(N)); + } + } + else if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type = use_ext + ? 0xC8 // ext 16 + : 0xC5; // bin 16 + + oa->write_character(to_char_type(output_type)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type = use_ext + ? 0xC9 // ext 32 + : 0xC6; // bin 32 + + oa->write_character(to_char_type(output_type)); + write_number(static_cast(N)); + } + + // step 1.5: if this is an ext type, write the subtype + if (use_ext) + { + write_number(static_cast(j.m_value.binary->subtype())); + } + + // step 2: write the byte string + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + N); + + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 15) + { + // fixmap + write_number(static_cast(0x80 | (N & 0xF))); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 16 + oa->write_character(to_char_type(0xDE)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 32 + oa->write_character(to_char_type(0xDF)); + write_number(static_cast(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_msgpack(el.first); + write_msgpack(el.second); + } + break; + } + + case value_t::discarded: + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(to_char_type('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + { + oa->write_character(j.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); + } + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + bool prefix_required = true; + if (use_type && !j.m_value.array->empty()) + { + JSON_ASSERT(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (!use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::binary: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + if (use_type && !j.m_value.binary->empty()) + { + JSON_ASSERT(use_count); + oa->write_character(to_char_type('$')); + oa->write_character('U'); + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.binary->size(), true); + } + + if (use_type) + { + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + j.m_value.binary->size()); + } + else + { + for (size_t i = 0; i < j.m_value.binary->size(); ++i) + { + oa->write_character(to_char_type('U')); + oa->write_character(j.m_value.binary->data()[i]); + } + } + + if (!use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(to_char_type('{')); + } + + bool prefix_required = true; + if (use_type && !j.m_value.object->empty()) + { + JSON_ASSERT(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (!use_count) + { + oa->write_character(to_char_type('}')); + } + + break; + } + + case value_t::discarded: + default: + break; + } + } + + private: + ////////// + // BSON // + ////////// + + /*! + @return The size of a BSON document entry header, including the id marker + and the entry name size (and its null-terminator). + */ + static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) + { + const auto it = name.find(static_cast(0)); + if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos)) + { + JSON_THROW(out_of_range::create(409, "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")", j)); + static_cast(j); + } + + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + /*! + @brief Writes the given @a element_type and @a name to the output adapter + */ + void write_bson_entry_header(const string_t& name, + const std::uint8_t element_type) + { + oa->write_character(to_char_type(element_type)); // boolean + oa->write_characters( + reinterpret_cast(name.c_str()), + name.size() + 1u); + } + + /*! + @brief Writes a BSON element with key @a name and boolean value @a value + */ + void write_bson_boolean(const string_t& name, + const bool value) + { + write_bson_entry_header(name, 0x08); + oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and double value @a value + */ + void write_bson_double(const string_t& name, + const double value) + { + write_bson_entry_header(name, 0x01); + write_number(value); + } + + /*! + @return The size of the BSON-encoded string in @a value + */ + static std::size_t calc_bson_string_size(const string_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and string value @a value + */ + void write_bson_string(const string_t& name, + const string_t& value) + { + write_bson_entry_header(name, 0x02); + + write_number(static_cast(value.size() + 1ul)); + oa->write_characters( + reinterpret_cast(value.c_str()), + value.size() + 1); + } + + /*! + @brief Writes a BSON element with key @a name and null value + */ + void write_bson_null(const string_t& name) + { + write_bson_entry_header(name, 0x0A); + } + + /*! + @return The size of the BSON-encoded integer @a value + */ + static std::size_t calc_bson_integer_size(const std::int64_t value) + { + return (std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)() + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and integer @a value + */ + void write_bson_integer(const string_t& name, + const std::int64_t value) + { + if ((std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)()) + { + write_bson_entry_header(name, 0x10); // int32 + write_number(static_cast(value)); + } + else + { + write_bson_entry_header(name, 0x12); // int64 + write_number(static_cast(value)); + } + } + + /*! + @return The size of the BSON-encoded unsigned integer in @a j + */ + static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept + { + return (value <= static_cast((std::numeric_limits::max)())) + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and unsigned @a value + */ + void write_bson_unsigned(const string_t& name, + const BasicJsonType& j) + { + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x10 /* int32 */); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x12 /* int64 */); + write_number(static_cast(j.m_value.number_unsigned)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BSON as it does not fit int64", j)); + } + } + + /*! + @brief Writes a BSON element with key @a name and object @a value + */ + void write_bson_object_entry(const string_t& name, + const typename BasicJsonType::object_t& value) + { + write_bson_entry_header(name, 0x03); // object + write_bson_object(value); + } + + /*! + @return The size of the BSON-encoded array @a value + */ + static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) + { + std::size_t array_index = 0ul; + + const std::size_t embedded_document_size = std::accumulate(std::begin(value), std::end(value), static_cast(0), [&array_index](std::size_t result, const typename BasicJsonType::array_t::value_type & el) + { + return result + calc_bson_element_size(std::to_string(array_index++), el); + }); + + return sizeof(std::int32_t) + embedded_document_size + 1ul; + } + + /*! + @return The size of the BSON-encoded binary array @a value + */ + static std::size_t calc_bson_binary_size(const typename BasicJsonType::binary_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and array @a value + */ + void write_bson_array(const string_t& name, + const typename BasicJsonType::array_t& value) + { + write_bson_entry_header(name, 0x04); // array + write_number(static_cast(calc_bson_array_size(value))); + + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + write_bson_element(std::to_string(array_index++), el); + } + + oa->write_character(to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and binary value @a value + */ + void write_bson_binary(const string_t& name, + const binary_t& value) + { + write_bson_entry_header(name, 0x05); + + write_number(static_cast(value.size())); + write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); + + oa->write_characters(reinterpret_cast(value.data()), value.size()); + } + + /*! + @brief Calculates the size necessary to serialize the JSON value @a j with its @a name + @return The calculated size for the BSON document entry for @a j with the given @a name. + */ + static std::size_t calc_bson_element_size(const string_t& name, + const BasicJsonType& j) + { + const auto header_size = calc_bson_entry_header_size(name, j); + switch (j.type()) + { + case value_t::object: + return header_size + calc_bson_object_size(*j.m_value.object); + + case value_t::array: + return header_size + calc_bson_array_size(*j.m_value.array); + + case value_t::binary: + return header_size + calc_bson_binary_size(*j.m_value.binary); + + case value_t::boolean: + return header_size + 1ul; + + case value_t::number_float: + return header_size + 8ul; + + case value_t::number_integer: + return header_size + calc_bson_integer_size(j.m_value.number_integer); + + case value_t::number_unsigned: + return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + + case value_t::string: + return header_size + calc_bson_string_size(*j.m_value.string); + + case value_t::null: + return header_size + 0ul; + + // LCOV_EXCL_START + case value_t::discarded: + default: + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) + return 0ul; + // LCOV_EXCL_STOP + } + } + + /*! + @brief Serializes the JSON value @a j to BSON and associates it with the + key @a name. + @param name The name to associate with the JSON entity @a j within the + current BSON document + */ + void write_bson_element(const string_t& name, + const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + return write_bson_object_entry(name, *j.m_value.object); + + case value_t::array: + return write_bson_array(name, *j.m_value.array); + + case value_t::binary: + return write_bson_binary(name, *j.m_value.binary); + + case value_t::boolean: + return write_bson_boolean(name, j.m_value.boolean); + + case value_t::number_float: + return write_bson_double(name, j.m_value.number_float); + + case value_t::number_integer: + return write_bson_integer(name, j.m_value.number_integer); + + case value_t::number_unsigned: + return write_bson_unsigned(name, j); + + case value_t::string: + return write_bson_string(name, *j.m_value.string); + + case value_t::null: + return write_bson_null(name); + + // LCOV_EXCL_START + case value_t::discarded: + default: + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) + return; + // LCOV_EXCL_STOP + } + } + + /*! + @brief Calculates the size of the BSON serialization of the given + JSON-object @a j. + @param[in] value JSON value to serialize + @pre value.type() == value_t::object + */ + static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + { + std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast(0), + [](size_t result, const typename BasicJsonType::object_t::value_type & el) + { + return result += calc_bson_element_size(el.first, el.second); + }); + + return sizeof(std::int32_t) + document_size + 1ul; + } + + /*! + @param[in] value JSON value to serialize + @pre value.type() == value_t::object + */ + void write_bson_object(const typename BasicJsonType::object_t& value) + { + write_number(static_cast(calc_bson_object_size(value))); + + for (const auto& el : value) + { + write_bson_element(el.first, el.second); + } + + oa->write_character(to_char_type(0x00)); + } + + ////////// + // CBOR // + ////////// + + static constexpr CharType get_cbor_float_prefix(float /*unused*/) + { + return to_char_type(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double /*unused*/) + { + return to_char_type(0xFB); // Double-Precision Float + } + + ///////////// + // MsgPack // + ///////////// + + static constexpr CharType get_msgpack_float_prefix(float /*unused*/) + { + return to_char_type(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double /*unused*/) + { + return to_char_type(0xCB); // float 64 + } + + //////////// + // UBJSON // + //////////// + + // UBJSON: write number (floating point) + template::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (add_prefix) + { + oa->write_character(get_ubjson_float_prefix(n)); + } + write_number(n); + } + + // UBJSON: write number (unsigned integer) + template::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast(n)); + } + else + { + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(static_cast(number[i]))); + } + } + } + + // UBJSON: write number (signed integer) + template < typename NumberType, typename std::enable_if < + std::is_signed::value&& + !std::is_floating_point::value, int >::type = 0 > + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast(n)); + } + else if (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast(n)); + } + // LCOV_EXCL_START + else + { + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(static_cast(number[i]))); + } + } + // LCOV_EXCL_STOP + } + + /*! + @brief determine the type prefix of container values + */ + CharType ubjson_prefix(const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'i'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'U'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'I'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'l'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; // LCOV_EXCL_LINE + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'i'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'U'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'I'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'l'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; // LCOV_EXCL_LINE + } + + case value_t::number_float: + return get_ubjson_float_prefix(j.m_value.number_float); + + case value_t::string: + return 'S'; + + case value_t::array: // fallthrough + case value_t::binary: + return '['; + + case value_t::object: + return '{'; + + case value_t::discarded: + default: // discarded values + return 'N'; + } + } + + static constexpr CharType get_ubjson_float_prefix(float /*unused*/) + { + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double /*unused*/) + { + return 'D'; // float 64 + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /* + @brief write a number to output input + @param[in] n number of type @a NumberType + @tparam NumberType the type of the number + @tparam OutputIsLittleEndian Set to true if output data is + required to be little endian + + @note This function needs to respect the system's endianness, because bytes + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. + */ + template + void write_number(const NumberType n) + { + // step 1: write number to array of length NumberType + std::array vec{}; + std::memcpy(vec.data(), &n, sizeof(NumberType)); + + // step 2: write array to output (with possible reordering) + if (is_little_endian != OutputIsLittleEndian) + { + // reverse byte order prior to conversion if necessary + std::reverse(vec.begin(), vec.end()); + } + + oa->write_characters(vec.data(), sizeof(NumberType)); + } + + void write_compact_float(const number_float_t n, detail::input_format_t format) + { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && + static_cast(n) <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(n)) == static_cast(n)) + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); + write_number(static_cast(n)); + } + else + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); + write_number(n); + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + } + + public: + // The following to_char_type functions are implement the conversion + // between uint8_t and CharType. In case CharType is not unsigned, + // such a conversion is required to allow values greater than 128. + // See for a discussion. + template < typename C = CharType, + enable_if_t < std::is_signed::value && std::is_signed::value > * = nullptr > + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return *reinterpret_cast(&x); + } + + template < typename C = CharType, + enable_if_t < std::is_signed::value && std::is_unsigned::value > * = nullptr > + static CharType to_char_type(std::uint8_t x) noexcept + { + static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); + static_assert(std::is_trivial::value, "CharType must be trivial"); + CharType result; + std::memcpy(&result, &x, sizeof(x)); + return result; + } + + template::value>* = nullptr> + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return x; + } + + template < typename InputCharType, typename C = CharType, + enable_if_t < + std::is_signed::value && + std::is_signed::value && + std::is_same::type>::value + > * = nullptr > + static constexpr CharType to_char_type(InputCharType x) noexcept + { + return x; + } + + private: + /// whether we can assume little endianness + const bool is_little_endian = little_endianness(); + + /// the output + output_adapter_t oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // reverse, remove, fill, find, none_of +#include // array +#include // localeconv, lconv +#include // labs, isfinite, isnan, signbit +#include // size_t, ptrdiff_t +#include // uint8_t +#include // snprintf +#include // numeric_limits +#include // string, char_traits +#include // setfill, setw +#include // is_same +#include // move + +// #include + + +#include // array +#include // signbit, isfinite +#include // intN_t, uintN_t +#include // memcpy, memmove +#include // numeric_limits +#include // conditional + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief implements the Grisu2 algorithm for binary to decimal floating-point +conversion. + +This implementation is a slightly modified version of the reference +implementation which may be obtained from +http://florian.loitsch.com/publications (bench.tar.gz). + +The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. + +For a detailed description of the algorithm see: + +[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with + Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming + Language Design and Implementation, PLDI 2010 +[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", + Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language + Design and Implementation, PLDI 1996 +*/ +namespace dtoa_impl +{ + +template +Target reinterpret_bits(const Source source) +{ + static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); + + Target target; + std::memcpy(&target, &source, sizeof(Source)); + return target; +} + +struct diyfp // f * 2^e +{ + static constexpr int kPrecision = 64; // = q + + std::uint64_t f = 0; + int e = 0; + + constexpr diyfp(std::uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + + /*! + @brief returns x - y + @pre x.e == y.e and x.f >= y.f + */ + static diyfp sub(const diyfp& x, const diyfp& y) noexcept + { + JSON_ASSERT(x.e == y.e); + JSON_ASSERT(x.f >= y.f); + + return {x.f - y.f, x.e}; + } + + /*! + @brief returns x * y + @note The result is rounded. (Only the upper q bits are returned.) + */ + static diyfp mul(const diyfp& x, const diyfp& y) noexcept + { + static_assert(kPrecision == 64, "internal error"); + + // Computes: + // f = round((x.f * y.f) / 2^q) + // e = x.e + y.e + q + + // Emulate the 64-bit * 64-bit multiplication: + // + // p = u * v + // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) + // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) + // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) + // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) + // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) + // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) + // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) + // + // (Since Q might be larger than 2^32 - 1) + // + // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) + // + // (Q_hi + H does not overflow a 64-bit int) + // + // = p_lo + 2^64 p_hi + + const std::uint64_t u_lo = x.f & 0xFFFFFFFFu; + const std::uint64_t u_hi = x.f >> 32u; + const std::uint64_t v_lo = y.f & 0xFFFFFFFFu; + const std::uint64_t v_hi = y.f >> 32u; + + const std::uint64_t p0 = u_lo * v_lo; + const std::uint64_t p1 = u_lo * v_hi; + const std::uint64_t p2 = u_hi * v_lo; + const std::uint64_t p3 = u_hi * v_hi; + + const std::uint64_t p0_hi = p0 >> 32u; + const std::uint64_t p1_lo = p1 & 0xFFFFFFFFu; + const std::uint64_t p1_hi = p1 >> 32u; + const std::uint64_t p2_lo = p2 & 0xFFFFFFFFu; + const std::uint64_t p2_hi = p2 >> 32u; + + std::uint64_t Q = p0_hi + p1_lo + p2_lo; + + // The full product might now be computed as + // + // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) + // p_lo = p0_lo + (Q << 32) + // + // But in this particular case here, the full p_lo is not required. + // Effectively we only need to add the highest bit in p_lo to p_hi (and + // Q_hi + 1 does not overflow). + + Q += std::uint64_t{1} << (64u - 32u - 1u); // round, ties up + + const std::uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32u); + + return {h, x.e + y.e + 64}; + } + + /*! + @brief normalize x such that the significand is >= 2^(q-1) + @pre x.f != 0 + */ + static diyfp normalize(diyfp x) noexcept + { + JSON_ASSERT(x.f != 0); + + while ((x.f >> 63u) == 0) + { + x.f <<= 1u; + x.e--; + } + + return x; + } + + /*! + @brief normalize x such that the result has the exponent E + @pre e >= x.e and the upper e - x.e bits of x.f must be zero. + */ + static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept + { + const int delta = x.e - target_exponent; + + JSON_ASSERT(delta >= 0); + JSON_ASSERT(((x.f << delta) >> delta) == x.f); + + return {x.f << delta, target_exponent}; + } +}; + +struct boundaries +{ + diyfp w; + diyfp minus; + diyfp plus; +}; + +/*! +Compute the (normalized) diyfp representing the input number 'value' and its +boundaries. + +@pre value must be finite and positive +*/ +template +boundaries compute_boundaries(FloatType value) +{ + JSON_ASSERT(std::isfinite(value)); + JSON_ASSERT(value > 0); + + // Convert the IEEE representation into a diyfp. + // + // If v is denormal: + // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) + // If v is normalized: + // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) + + static_assert(std::numeric_limits::is_iec559, + "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + + constexpr int kPrecision = std::numeric_limits::digits; // = p (includes the hidden bit) + constexpr int kBias = std::numeric_limits::max_exponent - 1 + (kPrecision - 1); + constexpr int kMinExp = 1 - kBias; + constexpr std::uint64_t kHiddenBit = std::uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + + using bits_type = typename std::conditional::type; + + const auto bits = static_cast(reinterpret_bits(value)); + const std::uint64_t E = bits >> (kPrecision - 1); + const std::uint64_t F = bits & (kHiddenBit - 1); + + const bool is_denormal = E == 0; + const diyfp v = is_denormal + ? diyfp(F, kMinExp) + : diyfp(F + kHiddenBit, static_cast(E) - kBias); + + // Compute the boundaries m- and m+ of the floating-point value + // v = f * 2^e. + // + // Determine v- and v+, the floating-point predecessor and successor if v, + // respectively. + // + // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) + // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) + // + // v+ = v + 2^e + // + // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ + // between m- and m+ round to v, regardless of how the input rounding + // algorithm breaks ties. + // + // ---+-------------+-------------+-------------+-------------+--- (A) + // v- m- v m+ v+ + // + // -----------------+------+------+-------------+-------------+--- (B) + // v- m- v m+ v+ + + const bool lower_boundary_is_closer = F == 0 && E > 1; + const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); + const diyfp m_minus = lower_boundary_is_closer + ? diyfp(4 * v.f - 1, v.e - 2) // (B) + : diyfp(2 * v.f - 1, v.e - 1); // (A) + + // Determine the normalized w+ = m+. + const diyfp w_plus = diyfp::normalize(m_plus); + + // Determine w- = m- such that e_(w-) = e_(w+). + const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + + return {diyfp::normalize(v), w_minus, w_plus}; +} + +// Given normalized diyfp w, Grisu needs to find a (normalized) cached +// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies +// within a certain range [alpha, gamma] (Definition 3.2 from [1]) +// +// alpha <= e = e_c + e_w + q <= gamma +// +// or +// +// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q +// <= f_c * f_w * 2^gamma +// +// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies +// +// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma +// +// or +// +// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) +// +// The choice of (alpha,gamma) determines the size of the table and the form of +// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well +// in practice: +// +// The idea is to cut the number c * w = f * 2^e into two parts, which can be +// processed independently: An integral part p1, and a fractional part p2: +// +// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e +// = (f div 2^-e) + (f mod 2^-e) * 2^e +// = p1 + p2 * 2^e +// +// The conversion of p1 into decimal form requires a series of divisions and +// modulos by (a power of) 10. These operations are faster for 32-bit than for +// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be +// achieved by choosing +// +// -e >= 32 or e <= -32 := gamma +// +// In order to convert the fractional part +// +// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... +// +// into decimal form, the fraction is repeatedly multiplied by 10 and the digits +// d[-i] are extracted in order: +// +// (10 * p2) div 2^-e = d[-1] +// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... +// +// The multiplication by 10 must not overflow. It is sufficient to choose +// +// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. +// +// Since p2 = f mod 2^-e < 2^-e, +// +// -e <= 60 or e >= -60 := alpha + +constexpr int kAlpha = -60; +constexpr int kGamma = -32; + +struct cached_power // c = f * 2^e ~= 10^k +{ + std::uint64_t f; + int e; + int k; +}; + +/*! +For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached +power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c +satisfies (Definition 3.2 from [1]) + + alpha <= e_c + e + q <= gamma. +*/ +inline cached_power get_cached_power_for_binary_exponent(int e) +{ + // Now + // + // alpha <= e_c + e + q <= gamma (1) + // ==> f_c * 2^alpha <= c * 2^e * 2^q + // + // and since the c's are normalized, 2^(q-1) <= f_c, + // + // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) + // ==> 2^(alpha - e - 1) <= c + // + // If c were an exact power of ten, i.e. c = 10^k, one may determine k as + // + // k = ceil( log_10( 2^(alpha - e - 1) ) ) + // = ceil( (alpha - e - 1) * log_10(2) ) + // + // From the paper: + // "In theory the result of the procedure could be wrong since c is rounded, + // and the computation itself is approximated [...]. In practice, however, + // this simple function is sufficient." + // + // For IEEE double precision floating-point numbers converted into + // normalized diyfp's w = f * 2^e, with q = 64, + // + // e >= -1022 (min IEEE exponent) + // -52 (p - 1) + // -52 (p - 1, possibly normalize denormal IEEE numbers) + // -11 (normalize the diyfp) + // = -1137 + // + // and + // + // e <= +1023 (max IEEE exponent) + // -52 (p - 1) + // -11 (normalize the diyfp) + // = 960 + // + // This binary exponent range [-1137,960] results in a decimal exponent + // range [-307,324]. One does not need to store a cached power for each + // k in this range. For each such k it suffices to find a cached power + // such that the exponent of the product lies in [alpha,gamma]. + // This implies that the difference of the decimal exponents of adjacent + // table entries must be less than or equal to + // + // floor( (gamma - alpha) * log_10(2) ) = 8. + // + // (A smaller distance gamma-alpha would require a larger table.) + + // NB: + // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. + + constexpr int kCachedPowersMinDecExp = -300; + constexpr int kCachedPowersDecStep = 8; + + static constexpr std::array kCachedPowers = + { + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + } + }; + + // This computation gives exactly the same results for k as + // k = ceil((kAlpha - e - 1) * 0.30102999566398114) + // for |e| <= 1500, but doesn't require floating-point operations. + // NB: log_10(2) ~= 78913 / 2^18 + JSON_ASSERT(e >= -1500); + JSON_ASSERT(e <= 1500); + const int f = kAlpha - e - 1; + const int k = (f * 78913) / (1 << 18) + static_cast(f > 0); + + const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; + JSON_ASSERT(index >= 0); + JSON_ASSERT(static_cast(index) < kCachedPowers.size()); + + const cached_power cached = kCachedPowers[static_cast(index)]; + JSON_ASSERT(kAlpha <= cached.e + e + 64); + JSON_ASSERT(kGamma >= cached.e + e + 64); + + return cached; +} + +/*! +For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. +For n == 0, returns 1 and sets pow10 := 1. +*/ +inline int find_largest_pow10(const std::uint32_t n, std::uint32_t& pow10) +{ + // LCOV_EXCL_START + if (n >= 1000000000) + { + pow10 = 1000000000; + return 10; + } + // LCOV_EXCL_STOP + if (n >= 100000000) + { + pow10 = 100000000; + return 9; + } + if (n >= 10000000) + { + pow10 = 10000000; + return 8; + } + if (n >= 1000000) + { + pow10 = 1000000; + return 7; + } + if (n >= 100000) + { + pow10 = 100000; + return 6; + } + if (n >= 10000) + { + pow10 = 10000; + return 5; + } + if (n >= 1000) + { + pow10 = 1000; + return 4; + } + if (n >= 100) + { + pow10 = 100; + return 3; + } + if (n >= 10) + { + pow10 = 10; + return 2; + } + + pow10 = 1; + return 1; +} + +inline void grisu2_round(char* buf, int len, std::uint64_t dist, std::uint64_t delta, + std::uint64_t rest, std::uint64_t ten_k) +{ + JSON_ASSERT(len >= 1); + JSON_ASSERT(dist <= delta); + JSON_ASSERT(rest <= delta); + JSON_ASSERT(ten_k > 0); + + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // ten_k + // <------> + // <---- rest ----> + // --------------[------------------+----+--------------]-------------- + // w V + // = buf * 10^k + // + // ten_k represents a unit-in-the-last-place in the decimal representation + // stored in buf. + // Decrement buf by ten_k while this takes buf closer to w. + + // The tests are written in this order to avoid overflow in unsigned + // integer arithmetic. + + while (rest < dist + && delta - rest >= ten_k + && (rest + ten_k < dist || dist - rest > rest + ten_k - dist)) + { + JSON_ASSERT(buf[len - 1] != '0'); + buf[len - 1]--; + rest += ten_k; + } +} + +/*! +Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. +M- and M+ must be normalized and share the same exponent -60 <= e <= -32. +*/ +inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, + diyfp M_minus, diyfp w, diyfp M_plus) +{ + static_assert(kAlpha >= -60, "internal error"); + static_assert(kGamma <= -32, "internal error"); + + // Generates the digits (and the exponent) of a decimal floating-point + // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's + // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. + // + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // Grisu2 generates the digits of M+ from left to right and stops as soon as + // V is in [M-,M+]. + + JSON_ASSERT(M_plus.e >= kAlpha); + JSON_ASSERT(M_plus.e <= kGamma); + + std::uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + std::uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + + // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): + // + // M+ = f * 2^e + // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e + // = ((p1 ) * 2^-e + (p2 )) * 2^e + // = p1 + p2 * 2^e + + const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e); + + auto p1 = static_cast(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + std::uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + + // 1) + // + // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + + JSON_ASSERT(p1 > 0); + + std::uint32_t pow10{}; + const int k = find_largest_pow10(p1, pow10); + + // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) + // + // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) + // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) + // + // M+ = p1 + p2 * 2^e + // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e + // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e + // = d[k-1] * 10^(k-1) + ( rest) * 2^e + // + // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) + // + // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] + // + // but stop as soon as + // + // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + + int n = k; + while (n > 0) + { + // Invariants: + // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) + // pow10 = 10^(n-1) <= p1 < 10^n + // + const std::uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const std::uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + // + // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e + // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) + // + JSON_ASSERT(d <= 9); + buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) + // + p1 = r; + n--; + // + // M+ = buffer * 10^n + (p1 + p2 * 2^e) + // pow10 = 10^n + // + + // Now check if enough digits have been generated. + // Compute + // + // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e + // + // Note: + // Since rest and delta share the same exponent e, it suffices to + // compare the significands. + const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2; + if (rest <= delta) + { + // V = buffer * 10^n, with M- <= V <= M+. + + decimal_exponent += n; + + // We may now just stop. But instead look if the buffer could be + // decremented to bring V closer to w. + // + // pow10 = 10^n is now 1 ulp in the decimal representation V. + // The rounding procedure works with diyfp's with an implicit + // exponent of e. + // + // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e + // + const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e; + grisu2_round(buffer, length, dist, delta, rest, ten_n); + + return; + } + + pow10 /= 10; + // + // pow10 = 10^(n-1) <= p1 < 10^n + // Invariants restored. + } + + // 2) + // + // The digits of the integral part have been generated: + // + // M+ = d[k-1]...d[1]d[0] + p2 * 2^e + // = buffer + p2 * 2^e + // + // Now generate the digits of the fractional part p2 * 2^e. + // + // Note: + // No decimal point is generated: the exponent is adjusted instead. + // + // p2 actually represents the fraction + // + // p2 * 2^e + // = p2 / 2^-e + // = d[-1] / 10^1 + d[-2] / 10^2 + ... + // + // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) + // + // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m + // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) + // + // using + // + // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) + // = ( d) * 2^-e + ( r) + // + // or + // 10^m * p2 * 2^e = d + r * 2^e + // + // i.e. + // + // M+ = buffer + p2 * 2^e + // = buffer + 10^-m * (d + r * 2^e) + // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e + // + // and stop as soon as 10^-m * r * 2^e <= delta * 2^e + + JSON_ASSERT(p2 > delta); + + int m = 0; + for (;;) + { + // Invariant: + // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e + // = buffer * 10^-m + 10^-m * (p2 ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e + // + JSON_ASSERT(p2 <= (std::numeric_limits::max)() / 10); + p2 *= 10; + const std::uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + // + // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) + // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + JSON_ASSERT(d <= 9); + buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + p2 = r; + m++; + // + // M+ = buffer * 10^-m + 10^-m * p2 * 2^e + // Invariant restored. + + // Check if enough digits have been generated. + // + // 10^-m * p2 * 2^e <= delta * 2^e + // p2 * 2^e <= 10^m * delta * 2^e + // p2 <= 10^m * delta + delta *= 10; + dist *= 10; + if (p2 <= delta) + { + break; + } + } + + // V = buffer * 10^-m, with M- <= V <= M+. + + decimal_exponent -= m; + + // 1 ulp in the decimal representation is now 10^-m. + // Since delta and dist are now scaled by 10^m, we need to do the + // same with ulp in order to keep the units in sync. + // + // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e + // + const std::uint64_t ten_m = one.f; + grisu2_round(buffer, length, dist, delta, p2, ten_m); + + // By construction this algorithm generates the shortest possible decimal + // number (Loitsch, Theorem 6.2) which rounds back to w. + // For an input number of precision p, at least + // + // N = 1 + ceil(p * log_10(2)) + // + // decimal digits are sufficient to identify all binary floating-point + // numbers (Matula, "In-and-Out conversions"). + // This implies that the algorithm does not produce more than N decimal + // digits. + // + // N = 17 for p = 53 (IEEE double precision) + // N = 9 for p = 24 (IEEE single precision) +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +JSON_HEDLEY_NON_NULL(1) +inline void grisu2(char* buf, int& len, int& decimal_exponent, + diyfp m_minus, diyfp v, diyfp m_plus) +{ + JSON_ASSERT(m_plus.e == m_minus.e); + JSON_ASSERT(m_plus.e == v.e); + + // --------(-----------------------+-----------------------)-------- (A) + // m- v m+ + // + // --------------------(-----------+-----------------------)-------- (B) + // m- v m+ + // + // First scale v (and m- and m+) such that the exponent is in the range + // [alpha, gamma]. + + const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + + const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + + // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] + const diyfp w = diyfp::mul(v, c_minus_k); + const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); + const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + + // ----(---+---)---------------(---+---)---------------(---+---)---- + // w- w w+ + // = c*m- = c*v = c*m+ + // + // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and + // w+ are now off by a small amount. + // In fact: + // + // w - v * 10^k < 1 ulp + // + // To account for this inaccuracy, add resp. subtract 1 ulp. + // + // --------+---[---------------(---+---)---------------]---+-------- + // w- M- w M+ w+ + // + // Now any number in [M-, M+] (bounds included) will round to w when input, + // regardless of how the input rounding algorithm breaks ties. + // + // And digit_gen generates the shortest possible such number in [M-, M+]. + // Note that this does not mean that Grisu2 always generates the shortest + // possible number in the interval (m-, m+). + const diyfp M_minus(w_minus.f + 1, w_minus.e); + const diyfp M_plus (w_plus.f - 1, w_plus.e ); + + decimal_exponent = -cached.k; // = -(-k) = k + + grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +template +JSON_HEDLEY_NON_NULL(1) +void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) +{ + static_assert(diyfp::kPrecision >= std::numeric_limits::digits + 3, + "internal error: not enough precision"); + + JSON_ASSERT(std::isfinite(value)); + JSON_ASSERT(value > 0); + + // If the neighbors (and boundaries) of 'value' are always computed for double-precision + // numbers, all float's can be recovered using strtod (and strtof). However, the resulting + // decimal representations are not exactly "short". + // + // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) + // says "value is converted to a string as if by std::sprintf in the default ("C") locale" + // and since sprintf promotes floats to doubles, I think this is exactly what 'std::to_chars' + // does. + // On the other hand, the documentation for 'std::to_chars' requires that "parsing the + // representation using the corresponding std::from_chars function recovers value exactly". That + // indicates that single precision floating-point numbers should be recovered using + // 'std::strtof'. + // + // NB: If the neighbors are computed for single-precision numbers, there is a single float + // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision + // value is off by 1 ulp. +#if 0 + const boundaries w = compute_boundaries(static_cast(value)); +#else + const boundaries w = compute_boundaries(value); +#endif + + grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +} + +/*! +@brief appends a decimal representation of e to buf +@return a pointer to the element following the exponent. +@pre -1000 < e < 1000 +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* append_exponent(char* buf, int e) +{ + JSON_ASSERT(e > -1000); + JSON_ASSERT(e < 1000); + + if (e < 0) + { + e = -e; + *buf++ = '-'; + } + else + { + *buf++ = '+'; + } + + auto k = static_cast(e); + if (k < 10) + { + // Always print at least two digits in the exponent. + // This is for compatibility with printf("%g"). + *buf++ = '0'; + *buf++ = static_cast('0' + k); + } + else if (k < 100) + { + *buf++ = static_cast('0' + k / 10); + k %= 10; + *buf++ = static_cast('0' + k); + } + else + { + *buf++ = static_cast('0' + k / 100); + k %= 100; + *buf++ = static_cast('0' + k / 10); + k %= 10; + *buf++ = static_cast('0' + k); + } + + return buf; +} + +/*! +@brief prettify v = buf * 10^decimal_exponent + +If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point +notation. Otherwise it will be printed in exponential notation. + +@pre min_exp < 0 +@pre max_exp > 0 +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* format_buffer(char* buf, int len, int decimal_exponent, + int min_exp, int max_exp) +{ + JSON_ASSERT(min_exp < 0); + JSON_ASSERT(max_exp > 0); + + const int k = len; + const int n = len + decimal_exponent; + + // v = buf * 10^(n-k) + // k is the length of the buffer (number of decimal digits) + // n is the position of the decimal point relative to the start of the buffer. + + if (k <= n && n <= max_exp) + { + // digits[000] + // len <= max_exp + 2 + + std::memset(buf + k, '0', static_cast(n) - static_cast(k)); + // Make it look like a floating-point number (#362, #378) + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (static_cast(n) + 2); + } + + if (0 < n && n <= max_exp) + { + // dig.its + // len <= max_digits10 + 1 + + JSON_ASSERT(k > n); + + std::memmove(buf + (static_cast(n) + 1), buf + n, static_cast(k) - static_cast(n)); + buf[n] = '.'; + return buf + (static_cast(k) + 1U); + } + + if (min_exp < n && n <= 0) + { + // 0.[000]digits + // len <= 2 + (-min_exp - 1) + max_digits10 + + std::memmove(buf + (2 + static_cast(-n)), buf, static_cast(k)); + buf[0] = '0'; + buf[1] = '.'; + std::memset(buf + 2, '0', static_cast(-n)); + return buf + (2U + static_cast(-n) + static_cast(k)); + } + + if (k == 1) + { + // dE+123 + // len <= 1 + 5 + + buf += 1; + } + else + { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + + std::memmove(buf + 2, buf + 1, static_cast(k) - 1); + buf[1] = '.'; + buf += 1 + static_cast(k); + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // namespace dtoa_impl + +/*! +@brief generates a decimal representation of the floating-point number value in [first, last). + +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. + +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must be large enough. +@note The result is NOT null-terminated. +*/ +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* to_chars(char* first, const char* last, FloatType value) +{ + static_cast(last); // maybe unused - fix warning + JSON_ASSERT(std::isfinite(value)); + + // Use signbit(value) instead of (value < 0) since signbit works for -0. + if (std::signbit(value)) + { + value = -value; + *first++ = '-'; + } + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::grisu2(first, len, decimal_exponent, value); + + JSON_ASSERT(len <= std::numeric_limits::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits::digits10; + + JSON_ASSERT(last - first >= kMaxExp + 2); + JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// serialization // +/////////////////// + +/// how to treat decoding errors +enum class error_handler_t +{ + strict, ///< throw a type_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore ///< ignore invalid UTF-8 sequences +}; + +template +class serializer +{ + using string_t = typename BasicJsonType::string_t; + using number_float_t = typename BasicJsonType::number_float_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using binary_char_t = typename BasicJsonType::binary_t::value_type; + static constexpr std::uint8_t UTF8_ACCEPT = 0; + static constexpr std::uint8_t UTF8_REJECT = 1; + + public: + /*! + @param[in] s output stream to serialize to + @param[in] ichar indentation character to use + @param[in] error_handler_ how to react on decoding errors + */ + serializer(output_adapter_t s, const char ichar, + error_handler_t error_handler_ = error_handler_t::strict) + : o(std::move(s)) + , loc(std::localeconv()) + , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) + , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + , indent_char(ichar) + , indent_string(512, indent_char) + , error_handler(error_handler_) + {} + + // delete because of pointer members + serializer(const serializer&) = delete; + serializer& operator=(const serializer&) = delete; + serializer(serializer&&) = delete; + serializer& operator=(serializer&&) = delete; + ~serializer() = default; + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + - binary values are serialized as objects containing the subtype and the + byte array + + @param[in] val value to serialize + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters + in the output are escaped with `\uXXXX` sequences, and the result consists + of ASCII characters only. + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(const BasicJsonType& val, + const bool pretty_print, + const bool ensure_ascii, + const unsigned int indent_step, + const unsigned int current_indent = 0) + { + switch (val.m_type) + { + case value_t::object: + { + if (val.m_value.object->empty()) + { + o->write_characters("{}", 2); + return; + } + + if (pretty_print) + { + o->write_characters("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + JSON_ASSERT(i != val.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_value.object->cend()); + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character('}'); + } + else + { + o->write_character('{'); + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + JSON_ASSERT(i != val.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_value.object->cend()); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + + o->write_character('}'); + } + + return; + } + + case value_t::array: + { + if (val.m_value.array->empty()) + { + o->write_characters("[]", 2); + return; + } + + if (pretty_print) + { + o->write_characters("[\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + dump(*i, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + JSON_ASSERT(!val.m_value.array->empty()); + o->write_characters(indent_string.c_str(), new_indent); + dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character(']'); + } + else + { + o->write_character('['); + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + dump(*i, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + JSON_ASSERT(!val.m_value.array->empty()); + dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + + o->write_character(']'); + } + + return; + } + + case value_t::string: + { + o->write_character('\"'); + dump_escaped(*val.m_value.string, ensure_ascii); + o->write_character('\"'); + return; + } + + case value_t::binary: + { + if (pretty_print) + { + o->write_characters("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + o->write_characters(indent_string.c_str(), new_indent); + + o->write_characters("\"bytes\": [", 10); + + if (!val.m_value.binary->empty()) + { + for (auto i = val.m_value.binary->cbegin(); + i != val.m_value.binary->cend() - 1; ++i) + { + dump_integer(*i); + o->write_characters(", ", 2); + } + dump_integer(val.m_value.binary->back()); + } + + o->write_characters("],\n", 3); + o->write_characters(indent_string.c_str(), new_indent); + + o->write_characters("\"subtype\": ", 11); + if (val.m_value.binary->has_subtype()) + { + dump_integer(val.m_value.binary->subtype()); + } + else + { + o->write_characters("null", 4); + } + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character('}'); + } + else + { + o->write_characters("{\"bytes\":[", 10); + + if (!val.m_value.binary->empty()) + { + for (auto i = val.m_value.binary->cbegin(); + i != val.m_value.binary->cend() - 1; ++i) + { + dump_integer(*i); + o->write_character(','); + } + dump_integer(val.m_value.binary->back()); + } + + o->write_characters("],\"subtype\":", 12); + if (val.m_value.binary->has_subtype()) + { + dump_integer(val.m_value.binary->subtype()); + o->write_character('}'); + } + else + { + o->write_characters("null}", 5); + } + } + return; + } + + case value_t::boolean: + { + if (val.m_value.boolean) + { + o->write_characters("true", 4); + } + else + { + o->write_characters("false", 5); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_value.number_float); + return; + } + + case value_t::discarded: + { + o->write_characters("", 11); + return; + } + + case value_t::null: + { + o->write_characters("null", 4); + return; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief dump escaped string + + Escape a string by replacing certain special characters by a sequence of an + escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. + + @param[in] s the string to escape + @param[in] ensure_ascii whether to escape non-ASCII characters with + \uXXXX sequences + + @complexity Linear in the length of string @a s. + */ + void dump_escaped(const string_t& s, const bool ensure_ascii) + { + std::uint32_t codepoint{}; + std::uint8_t state = UTF8_ACCEPT; + std::size_t bytes = 0; // number of bytes written to string_buffer + + // number of bytes written at the point of the last valid byte + std::size_t bytes_after_last_accept = 0; + std::size_t undumped_chars = 0; + + for (std::size_t i = 0; i < s.size(); ++i) + { + const auto byte = static_cast(s[i]); + + switch (decode(state, codepoint, byte)) + { + case UTF8_ACCEPT: // decode found a new code point + { + switch (codepoint) + { + case 0x08: // backspace + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'b'; + break; + } + + case 0x09: // horizontal tab + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 't'; + break; + } + + case 0x0A: // newline + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'n'; + break; + } + + case 0x0C: // formfeed + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'f'; + break; + } + + case 0x0D: // carriage return + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'r'; + break; + } + + case 0x22: // quotation mark + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\"'; + break; + } + + case 0x5C: // reverse solidus + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\\'; + break; + } + + default: + { + // escape control characters (0x00..0x1F) or, if + // ensure_ascii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F))) + { + if (codepoint <= 0xFFFF) + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + static_cast((std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast(codepoint))); + bytes += 6; + } + else + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + static_cast((std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast(0xD7C0u + (codepoint >> 10u)), + static_cast(0xDC00u + (codepoint & 0x3FFu)))); + bytes += 12; + } + } + else + { + // copy byte to buffer (all previous bytes + // been copied have in default case above) + string_buffer[bytes++] = s[i]; + } + break; + } + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + // remember the byte position of this accept + bytes_after_last_accept = bytes; + undumped_chars = 0; + break; + } + + case UTF8_REJECT: // decode found invalid UTF-8 byte + { + switch (error_handler) + { + case error_handler_t::strict: + { + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + hex_bytes(byte | 0), BasicJsonType())); + } + + case error_handler_t::ignore: + case error_handler_t::replace: + { + // in case we saw this character the first time, we + // would like to read it again, because the byte + // may be OK for itself, but just not OK for the + // previous sequence + if (undumped_chars > 0) + { + --i; + } + + // reset length buffer to the last accepted index; + // thus removing/ignoring the invalid characters + bytes = bytes_after_last_accept; + + if (error_handler == error_handler_t::replace) + { + // add a replacement character + if (ensure_ascii) + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'u'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'd'; + } + else + { + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xEF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBD'); + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + bytes_after_last_accept = bytes; + } + + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + break; + } + + default: // decode found yet incomplete multi-byte code point + { + if (!ensure_ascii) + { + // code point will not be escaped - copy byte to buffer + string_buffer[bytes++] = s[i]; + } + ++undumped_chars; + break; + } + } + } + + // we finished processing the string + if (JSON_HEDLEY_LIKELY(state == UTF8_ACCEPT)) + { + // write buffer + if (bytes > 0) + { + o->write_characters(string_buffer.data(), bytes); + } + } + else + { + // we finish reading, but do not accept: string was incomplete + switch (error_handler) + { + case error_handler_t::strict: + { + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + hex_bytes(static_cast(s.back() | 0)), BasicJsonType())); + } + + case error_handler_t::ignore: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + break; + } + + case error_handler_t::replace: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + // add a replacement character + if (ensure_ascii) + { + o->write_characters("\\ufffd", 6); + } + else + { + o->write_characters("\xEF\xBF\xBD", 3); + } + break; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + } + + private: + /*! + @brief count digits + + Count the number of decimal (base 10) digits for an input unsigned integer. + + @param[in] x unsigned integer number to count its digits + @return number of decimal digits + */ + inline unsigned int count_digits(number_unsigned_t x) noexcept + { + unsigned int n_digits = 1; + for (;;) + { + if (x < 10) + { + return n_digits; + } + if (x < 100) + { + return n_digits + 1; + } + if (x < 1000) + { + return n_digits + 2; + } + if (x < 10000) + { + return n_digits + 3; + } + x = x / 10000u; + n_digits += 4; + } + } + + /*! + * @brief convert a byte to a uppercase hex representation + * @param[in] byte byte to represent + * @return representation ("00".."FF") + */ + static std::string hex_bytes(std::uint8_t byte) + { + std::string result = "FF"; + constexpr const char* nibble_to_hex = "0123456789ABCDEF"; + result[0] = nibble_to_hex[byte / 16]; + result[1] = nibble_to_hex[byte % 16]; + return result; + } + + // templates to avoid warnings about useless casts + template ::value, int> = 0> + bool is_negative_number(NumberType x) + { + return x < 0; + } + + template < typename NumberType, enable_if_t ::value, int > = 0 > + bool is_negative_number(NumberType /*unused*/) + { + return false; + } + + /*! + @brief dump an integer + + Dump a given integer to output stream @a o. Works internally with + @a number_buffer. + + @param[in] x integer number (signed or unsigned) to dump + @tparam NumberType either @a number_integer_t or @a number_unsigned_t + */ + template < typename NumberType, detail::enable_if_t < + std::is_integral::value || + std::is_same::value || + std::is_same::value || + std::is_same::value, + int > = 0 > + void dump_integer(NumberType x) + { + static constexpr std::array, 100> digits_to_99 + { + { + {{'0', '0'}}, {{'0', '1'}}, {{'0', '2'}}, {{'0', '3'}}, {{'0', '4'}}, {{'0', '5'}}, {{'0', '6'}}, {{'0', '7'}}, {{'0', '8'}}, {{'0', '9'}}, + {{'1', '0'}}, {{'1', '1'}}, {{'1', '2'}}, {{'1', '3'}}, {{'1', '4'}}, {{'1', '5'}}, {{'1', '6'}}, {{'1', '7'}}, {{'1', '8'}}, {{'1', '9'}}, + {{'2', '0'}}, {{'2', '1'}}, {{'2', '2'}}, {{'2', '3'}}, {{'2', '4'}}, {{'2', '5'}}, {{'2', '6'}}, {{'2', '7'}}, {{'2', '8'}}, {{'2', '9'}}, + {{'3', '0'}}, {{'3', '1'}}, {{'3', '2'}}, {{'3', '3'}}, {{'3', '4'}}, {{'3', '5'}}, {{'3', '6'}}, {{'3', '7'}}, {{'3', '8'}}, {{'3', '9'}}, + {{'4', '0'}}, {{'4', '1'}}, {{'4', '2'}}, {{'4', '3'}}, {{'4', '4'}}, {{'4', '5'}}, {{'4', '6'}}, {{'4', '7'}}, {{'4', '8'}}, {{'4', '9'}}, + {{'5', '0'}}, {{'5', '1'}}, {{'5', '2'}}, {{'5', '3'}}, {{'5', '4'}}, {{'5', '5'}}, {{'5', '6'}}, {{'5', '7'}}, {{'5', '8'}}, {{'5', '9'}}, + {{'6', '0'}}, {{'6', '1'}}, {{'6', '2'}}, {{'6', '3'}}, {{'6', '4'}}, {{'6', '5'}}, {{'6', '6'}}, {{'6', '7'}}, {{'6', '8'}}, {{'6', '9'}}, + {{'7', '0'}}, {{'7', '1'}}, {{'7', '2'}}, {{'7', '3'}}, {{'7', '4'}}, {{'7', '5'}}, {{'7', '6'}}, {{'7', '7'}}, {{'7', '8'}}, {{'7', '9'}}, + {{'8', '0'}}, {{'8', '1'}}, {{'8', '2'}}, {{'8', '3'}}, {{'8', '4'}}, {{'8', '5'}}, {{'8', '6'}}, {{'8', '7'}}, {{'8', '8'}}, {{'8', '9'}}, + {{'9', '0'}}, {{'9', '1'}}, {{'9', '2'}}, {{'9', '3'}}, {{'9', '4'}}, {{'9', '5'}}, {{'9', '6'}}, {{'9', '7'}}, {{'9', '8'}}, {{'9', '9'}}, + } + }; + + // special case for "0" + if (x == 0) + { + o->write_character('0'); + return; + } + + // use a pointer to fill the buffer + auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-vararg,hicpp-vararg) + + number_unsigned_t abs_value; + + unsigned int n_chars{}; + + if (is_negative_number(x)) + { + *buffer_ptr = '-'; + abs_value = remove_sign(static_cast(x)); + + // account one more byte for the minus sign + n_chars = 1 + count_digits(abs_value); + } + else + { + abs_value = static_cast(x); + n_chars = count_digits(abs_value); + } + + // spare 1 byte for '\0' + JSON_ASSERT(n_chars < number_buffer.size() - 1); + + // jump to the end to generate the string from backward, + // so we later avoid reversing the result + buffer_ptr += n_chars; + + // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu + // See: https://www.youtube.com/watch?v=o4-CwDo2zpg + while (abs_value >= 100) + { + const auto digits_index = static_cast((abs_value % 100)); + abs_value /= 100; + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + + if (abs_value >= 10) + { + const auto digits_index = static_cast(abs_value); + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + else + { + *(--buffer_ptr) = static_cast('0' + abs_value); + } + + o->write_characters(number_buffer.data(), n_chars); + } + + /*! + @brief dump a floating-point number + + Dump a given floating-point number to output stream @a o. Works internally + with @a number_buffer. + + @param[in] x floating-point number to dump + */ + void dump_float(number_float_t x) + { + // NaN / inf + if (!std::isfinite(x)) + { + o->write_characters("null", 4); + return; + } + + // If number_float_t is an IEEE-754 single or double precision number, + // use the Grisu2 algorithm to produce short numbers which are + // guaranteed to round-trip, using strtof and strtod, resp. + // + // NB: The test below works if == . + static constexpr bool is_ieee_single_or_double + = (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 24 && std::numeric_limits::max_exponent == 128) || + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && std::numeric_limits::max_exponent == 1024); + + dump_float(x, std::integral_constant()); + } + + void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) + { + auto* begin = number_buffer.data(); + auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); + + o->write_characters(begin, static_cast(end - begin)); + } + + void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) + { + // get number of digits for a float -> text -> float round-trip + static constexpr auto d = std::numeric_limits::max_digits10; + + // the actual conversion + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); + + // negative value indicates an error + JSON_ASSERT(len > 0); + // check if buffer was large enough + JSON_ASSERT(static_cast(len) < number_buffer.size()); + + // erase thousands separator + if (thousands_sep != '\0') + { + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); + std::fill(end, number_buffer.end(), '\0'); + JSON_ASSERT((end - number_buffer.begin()) <= len); + len = (end - number_buffer.begin()); + } + + // convert decimal point to '.' + if (decimal_point != '\0' && decimal_point != '.') + { + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + if (dec_pos != number_buffer.end()) + { + *dec_pos = '.'; + } + } + + o->write_characters(number_buffer.data(), static_cast(len)); + + // determine if we need to append ".0" + const bool value_is_int_like = + std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1, + [](char c) + { + return c == '.' || c == 'e'; + }); + + if (value_is_int_like) + { + o->write_characters(".0", 2); + } + } + + /*! + @brief check whether a string is UTF-8 encoded + + The function checks each byte of a string whether it is UTF-8 encoded. The + result of the check is stored in the @a state parameter. The function must + be called initially with state 0 (accept). State 1 means the string must + be rejected, because the current byte is not allowed. If the string is + completely processed, but the state is non-zero, the string ended + prematurely; that is, the last byte indicated more bytes should have + followed. + + @param[in,out] state the state of the decoding + @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) + @param[in] byte next byte to decode + @return new state + + @note The function has been edited: a std::array is used. + + @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann + @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + */ + static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept + { + static const std::array utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + JSON_ASSERT(byte < utf8d.size()); + const std::uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); + + std::size_t index = 256u + static_cast(state) * 16u + static_cast(type); + JSON_ASSERT(index < 400); + state = utf8d[index]; + return state; + } + + /* + * Overload to make the compiler happy while it is instantiating + * dump_integer for number_unsigned_t. + * Must never be called. + */ + number_unsigned_t remove_sign(number_unsigned_t x) + { + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return x; // LCOV_EXCL_LINE + } + + /* + * Helper function for dump_integer + * + * This function takes a negative signed integer and returns its absolute + * value as unsigned integer. The plus/minus shuffling is necessary as we can + * not directly remove the sign of an arbitrary signed integer as the + * absolute values of INT_MIN and INT_MAX are usually not the same. See + * #1708 for details. + */ + inline number_unsigned_t remove_sign(number_integer_t x) noexcept + { + JSON_ASSERT(x < 0 && x < (std::numeric_limits::max)()); // NOLINT(misc-redundant-expression) + return static_cast(-(x + 1)) + 1; + } + + private: + /// the output of the serializer + output_adapter_t o = nullptr; + + /// a (hopefully) large enough character buffer + std::array number_buffer{{}}; + + /// the locale + const std::lconv* loc = nullptr; + /// the locale's thousand separator character + const char thousands_sep = '\0'; + /// the locale's decimal point character + const char decimal_point = '\0'; + + /// string buffer + std::array string_buffer{{}}; + + /// the indentation character + const char indent_char; + /// the indentation string + string_t indent_string; + + /// error_handler how to react on decoding errors + const error_handler_t error_handler; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + + +#include // less +#include // initializer_list +#include // input_iterator_tag, iterator_traits +#include // allocator +#include // for out_of_range +#include // enable_if, is_convertible +#include // pair +#include // vector + +// #include + + +namespace nlohmann +{ + +/// ordered_map: a minimal map-like container that preserves insertion order +/// for use within nlohmann::basic_json +template , + class Allocator = std::allocator>> + struct ordered_map : std::vector, Allocator> +{ + using key_type = Key; + using mapped_type = T; + using Container = std::vector, Allocator>; + using iterator = typename Container::iterator; + using const_iterator = typename Container::const_iterator; + using size_type = typename Container::size_type; + using value_type = typename Container::value_type; + + // Explicit constructors instead of `using Container::Container` + // otherwise older compilers choke on it (GCC <= 5.5, xcode <= 9.4) + ordered_map(const Allocator& alloc = Allocator()) : Container{alloc} {} + template + ordered_map(It first, It last, const Allocator& alloc = Allocator()) + : Container{first, last, alloc} {} + ordered_map(std::initializer_list init, const Allocator& alloc = Allocator() ) + : Container{init, alloc} {} + + std::pair emplace(const key_type& key, T&& t) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return {it, false}; + } + } + Container::emplace_back(key, t); + return {--this->end(), true}; + } + + T& operator[](const Key& key) + { + return emplace(key, T{}).first->second; + } + + const T& operator[](const Key& key) const + { + return at(key); + } + + T& at(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it->second; + } + } + + JSON_THROW(std::out_of_range("key not found")); + } + + const T& at(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it->second; + } + } + + JSON_THROW(std::out_of_range("key not found")); + } + + size_type erase(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + // Since we cannot move const Keys, re-construct them in place + for (auto next = it; ++next != this->end(); ++it) + { + it->~value_type(); // Destroy but keep allocation + new (&*it) value_type{std::move(*next)}; + } + Container::pop_back(); + return 1; + } + } + return 0; + } + + iterator erase(iterator pos) + { + return erase(pos, std::next(pos)); + } + + iterator erase(iterator first, iterator last) + { + const auto elements_affected = std::distance(first, last); + const auto offset = std::distance(Container::begin(), first); + + // This is the start situation. We need to delete elements_affected + // elements (3 in this example: e, f, g), and need to return an + // iterator past the last deleted element (h in this example). + // Note that offset is the distance from the start of the vector + // to first. We will need this later. + + // [ a, b, c, d, e, f, g, h, i, j ] + // ^ ^ + // first last + + // Since we cannot move const Keys, we re-construct them in place. + // We start at first and re-construct (viz. copy) the elements from + // the back of the vector. Example for first iteration: + + // ,--------. + // v | destroy e and re-construct with h + // [ a, b, c, d, e, f, g, h, i, j ] + // ^ ^ + // it it + elements_affected + + for (auto it = first; std::next(it, elements_affected) != Container::end(); ++it) + { + it->~value_type(); // destroy but keep allocation + new (&*it) value_type{std::move(*std::next(it, elements_affected))}; // "move" next element to it + } + + // [ a, b, c, d, h, i, j, h, i, j ] + // ^ ^ + // first last + + // remove the unneeded elements at the end of the vector + Container::resize(this->size() - static_cast(elements_affected)); + + // [ a, b, c, d, h, i, j ] + // ^ ^ + // first last + + // first is now pointing past the last deleted element, but we cannot + // use this iterator, because it may have been invalidated by the + // resize call. Instead, we can return begin() + offset. + return Container::begin() + offset; + } + + size_type count(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return 1; + } + } + return 0; + } + + iterator find(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it; + } + } + return Container::end(); + } + + const_iterator find(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it; + } + } + return Container::end(); + } + + std::pair insert( value_type&& value ) + { + return emplace(value.first, std::move(value.second)); + } + + std::pair insert( const value_type& value ) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == value.first) + { + return {it, false}; + } + } + Container::push_back(value); + return {--this->end(), true}; + } + + template + using require_input_iter = typename std::enable_if::iterator_category, + std::input_iterator_tag>::value>::type; + + template> + void insert(InputIt first, InputIt last) + { + for (auto it = first; it != last; ++it) + { + insert(*it); + } + } +}; + +} // namespace nlohmann + + +#if defined(JSON_HAS_CPP_17) + #include +#endif + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + +/*! +@brief a class to store JSON values + +@internal +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@note ObjectType trick from https://stackoverflow.com/a/9860911 +@endinternal + +@since version 1.0.0 + +@nosubgrouping +*/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +{ + private: + template friend struct detail::external_constructor; + friend ::nlohmann::json_pointer; + + template + friend class ::nlohmann::detail::parser; + friend ::nlohmann::detail::serializer; + template + friend class ::nlohmann::detail::iter_impl; + template + friend class ::nlohmann::detail::binary_writer; + template + friend class ::nlohmann::detail::binary_reader; + template + friend class ::nlohmann::detail::json_sax_dom_parser; + template + friend class ::nlohmann::detail::json_sax_dom_callback_parser; + friend class ::nlohmann::detail::exception; + + /// workaround type for MSVC + using basic_json_t = NLOHMANN_BASIC_JSON_TPL; + + JSON_PRIVATE_UNLESS_TESTED: + // convenience aliases for types residing in namespace detail; + using lexer = ::nlohmann::detail::lexer_base; + + template + static ::nlohmann::detail::parser parser( + InputAdapterType adapter, + detail::parser_callback_tcb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false + ) + { + return ::nlohmann::detail::parser(std::move(adapter), + std::move(cb), allow_exceptions, ignore_comments); + } + + private: + using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t; + template + using internal_iterator = ::nlohmann::detail::internal_iterator; + template + using iter_impl = ::nlohmann::detail::iter_impl; + template + using iteration_proxy = ::nlohmann::detail::iteration_proxy; + template using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator; + + template + using output_adapter_t = ::nlohmann::detail::output_adapter_t; + + template + using binary_reader = ::nlohmann::detail::binary_reader; + template using binary_writer = ::nlohmann::detail::binary_writer; + + JSON_PRIVATE_UNLESS_TESTED: + using serializer = ::nlohmann::detail::serializer; + + public: + using value_t = detail::value_t; + /// JSON Pointer, see @ref nlohmann::json_pointer + using json_pointer = ::nlohmann::json_pointer; + template + using json_serializer = JSONSerializer; + /// how to treat decoding errors + using error_handler_t = detail::error_handler_t; + /// how to treat CBOR tags + using cbor_tag_handler_t = detail::cbor_tag_handler_t; + /// helper type for initializer lists of basic_json values + using initializer_list_t = std::initializer_list>; + + using input_format_t = detail::input_format_t; + /// SAX interface type, see @ref nlohmann::json_sax + using json_sax_t = json_sax; + + //////////////// + // exceptions // + //////////////// + + /// @name exceptions + /// Classes to implement user-defined exceptions. + /// @{ + + using exception = detail::exception; + using parse_error = detail::parse_error; + using invalid_iterator = detail::invalid_iterator; + using type_error = detail::type_error; + using out_of_range = detail::out_of_range; + using other_error = detail::other_error; + + /// @} + + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + using iterator = iter_impl; + /// a const iterator for a basic_json container + using const_iterator = iter_impl; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /// @brief returns the allocator associated with the container + /// @sa https://json.nlohmann.me/api/basic_json/get_allocator/ + static allocator_type get_allocator() + { + return allocator_type(); + } + + /// @brief returns version information on the library + /// @sa https://json.nlohmann.me/api/basic_json/meta/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json meta() + { + basic_json result; + + result["copyright"] = "(C) 2013-2022 Niels Lohmann"; + result["name"] = "JSON for Modern C++"; + result["url"] = "https://github.com/nlohmann/json"; + result["version"]["string"] = + std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_PATCH); + result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR; + result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR; + result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH; + +#ifdef _WIN32 + result["platform"] = "win32"; +#elif defined __linux__ + result["platform"] = "linux"; +#elif defined __APPLE__ + result["platform"] = "apple"; +#elif defined __unix__ + result["platform"] = "unix"; +#else + result["platform"] = "unknown"; +#endif + +#if defined(__ICC) || defined(__INTEL_COMPILER) + result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}}; +#elif defined(__clang__) + result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; +#elif defined(__GNUC__) || defined(__GNUG__) + result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}}; +#elif defined(__HP_cc) || defined(__HP_aCC) + result["compiler"] = "hp" +#elif defined(__IBMCPP__) + result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; +#elif defined(_MSC_VER) + result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}}; +#elif defined(__PGI) + result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}}; +#elif defined(__SUNPRO_CC) + result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}}; +#else + result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; +#endif + +#ifdef __cplusplus + result["compiler"]["c++"] = std::to_string(__cplusplus); +#else + result["compiler"]["c++"] = "unknown"; +#endif + return result; + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + + /// @brief object key comparator type + /// @sa https://json.nlohmann.me/api/basic_json/object_comparator_t/ +#if defined(JSON_HAS_CPP_14) + // Use transparent comparator if possible, combined with perfect forwarding + // on find() and count() calls prevents unnecessary string construction. + using object_comparator_t = std::less<>; +#else + using object_comparator_t = std::less; +#endif + + /// @brief a type for an object + /// @sa https://json.nlohmann.me/api/basic_json/object_t/ + using object_t = ObjectType>>; + + /// @brief a type for an array + /// @sa https://json.nlohmann.me/api/basic_json/array_t/ + using array_t = ArrayType>; + + /// @brief a type for a string + /// @sa https://json.nlohmann.me/api/basic_json/string_t/ + using string_t = StringType; + + /// @brief a type for a boolean + /// @sa https://json.nlohmann.me/api/basic_json/boolean_t/ + using boolean_t = BooleanType; + + /// @brief a type for a number (integer) + /// @sa https://json.nlohmann.me/api/basic_json/number_integer_t/ + using number_integer_t = NumberIntegerType; + + /// @brief a type for a number (unsigned) + /// @sa https://json.nlohmann.me/api/basic_json/number_unsigned_t/ + using number_unsigned_t = NumberUnsignedType; + + /// @brief a type for a number (floating-point) + /// @sa https://json.nlohmann.me/api/basic_json/number_float_t/ + using number_float_t = NumberFloatType; + + /// @brief a type for a packed binary type + /// @sa https://json.nlohmann.me/api/basic_json/binary_t/ + using binary_t = nlohmann::byte_container_with_subtype; + + /// @} + + private: + + /// helper for exception-safe object creation + template + JSON_HEDLEY_RETURNS_NON_NULL + static T* create(Args&& ... args) + { + AllocatorType alloc; + using AllocatorTraits = std::allocator_traits>; + + auto deleter = [&](T * obj) + { + AllocatorTraits::deallocate(alloc, obj, 1); + }; + std::unique_ptr obj(AllocatorTraits::allocate(alloc, 1), deleter); + AllocatorTraits::construct(alloc, obj.get(), std::forward(args)...); + JSON_ASSERT(obj != nullptr); + return obj.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + binary | binary | pointer to @ref binary_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// binary (stored with pointer to save storage) + binary_t* binary; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } + + case value_t::array: + { + array = create(); + break; + } + + case value_t::string: + { + string = create(""); + break; + } + + case value_t::binary: + { + binary = create(); + break; + } + + case value_t::boolean: + { + boolean = static_cast(false); + break; + } + + case value_t::number_integer: + { + number_integer = static_cast(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = static_cast(0); + break; + } + + case value_t::number_float: + { + number_float = static_cast(0.0); + break; + } + + case value_t::null: + { + object = nullptr; // silence warning, see #821 + break; + } + + case value_t::discarded: + default: + { + object = nullptr; // silence warning, see #821 + if (JSON_HEDLEY_UNLIKELY(t == value_t::null)) + { + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.10.5", basic_json())); // LCOV_EXCL_LINE + } + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) : string(create(value)) {} + + /// constructor for rvalue strings + json_value(string_t&& value) : string(create(std::move(value))) {} + + /// constructor for objects + json_value(const object_t& value) : object(create(value)) {} + + /// constructor for rvalue objects + json_value(object_t&& value) : object(create(std::move(value))) {} + + /// constructor for arrays + json_value(const array_t& value) : array(create(value)) {} + + /// constructor for rvalue arrays + json_value(array_t&& value) : array(create(std::move(value))) {} + + /// constructor for binary arrays + json_value(const typename binary_t::container_type& value) : binary(create(value)) {} + + /// constructor for rvalue binary arrays + json_value(typename binary_t::container_type&& value) : binary(create(std::move(value))) {} + + /// constructor for binary arrays (internal type) + json_value(const binary_t& value) : binary(create(value)) {} + + /// constructor for rvalue binary arrays (internal type) + json_value(binary_t&& value) : binary(create(std::move(value))) {} + + void destroy(value_t t) + { + if (t == value_t::array || t == value_t::object) + { + // flatten the current json_value to a heap-allocated stack + std::vector stack; + + // move the top-level items to stack + if (t == value_t::array) + { + stack.reserve(array->size()); + std::move(array->begin(), array->end(), std::back_inserter(stack)); + } + else + { + stack.reserve(object->size()); + for (auto&& it : *object) + { + stack.push_back(std::move(it.second)); + } + } + + while (!stack.empty()) + { + // move the last item to local variable to be processed + basic_json current_item(std::move(stack.back())); + stack.pop_back(); + + // if current_item is array/object, move + // its children to the stack to be processed later + if (current_item.is_array()) + { + std::move(current_item.m_value.array->begin(), current_item.m_value.array->end(), std::back_inserter(stack)); + + current_item.m_value.array->clear(); + } + else if (current_item.is_object()) + { + for (auto&& it : *current_item.m_value.object) + { + stack.push_back(std::move(it.second)); + } + + current_item.m_value.object->clear(); + } + + // it's now safe that current_item get destructed + // since it doesn't have any children + } + } + + switch (t) + { + case value_t::object: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, object); + std::allocator_traits::deallocate(alloc, object, 1); + break; + } + + case value_t::array: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, array); + std::allocator_traits::deallocate(alloc, array, 1); + break; + } + + case value_t::string: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, string); + std::allocator_traits::deallocate(alloc, string, 1); + break; + } + + case value_t::binary: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, binary); + std::allocator_traits::deallocate(alloc, binary, 1); + break; + } + + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + { + break; + } + } + } + }; + + private: + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + + Furthermore, the parent relation is checked for arrays and objects: If + @a check_parents true and the value is an array or object, then the + container's elements must have the current value as parent. + + @param[in] check_parents whether the parent relation should be checked. + The value is true by default and should only be set to false + during destruction of objects when the invariant does not + need to hold. + */ + void assert_invariant(bool check_parents = true) const noexcept + { + JSON_ASSERT(m_type != value_t::object || m_value.object != nullptr); + JSON_ASSERT(m_type != value_t::array || m_value.array != nullptr); + JSON_ASSERT(m_type != value_t::string || m_value.string != nullptr); + JSON_ASSERT(m_type != value_t::binary || m_value.binary != nullptr); + +#if JSON_DIAGNOSTICS + JSON_TRY + { + // cppcheck-suppress assertWithSideEffect + JSON_ASSERT(!check_parents || !is_structured() || std::all_of(begin(), end(), [this](const basic_json & j) + { + return j.m_parent == this; + })); + } + JSON_CATCH(...) {} // LCOV_EXCL_LINE +#endif + static_cast(check_parents); + } + + void set_parents() + { +#if JSON_DIAGNOSTICS + switch (m_type) + { + case value_t::array: + { + for (auto& element : *m_value.array) + { + element.m_parent = this; + } + break; + } + + case value_t::object: + { + for (auto& element : *m_value.object) + { + element.second.m_parent = this; + } + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + break; + } +#endif + } + + iterator set_parents(iterator it, typename iterator::difference_type count_set_parents) + { +#if JSON_DIAGNOSTICS + for (typename iterator::difference_type i = 0; i < count_set_parents; ++i) + { + (it + i)->m_parent = this; + } +#else + static_cast(count_set_parents); +#endif + return it; + } + + reference set_parent(reference j, std::size_t old_capacity = static_cast(-1)) + { +#if JSON_DIAGNOSTICS + if (old_capacity != static_cast(-1)) + { + // see https://github.com/nlohmann/json/issues/2838 + JSON_ASSERT(type() == value_t::array); + if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity)) + { + // capacity has changed: update all parents + set_parents(); + return j; + } + } + + // ordered_json uses a vector internally, so pointers could have + // been invalidated; see https://github.com/nlohmann/json/issues/2962 +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(push ) +#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr +#endif + if (detail::is_ordered_map::value) + { + set_parents(); + return j; + } +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning( pop ) +#endif + + j.m_parent = this; +#else + static_cast(j); + static_cast(old_capacity); +#endif + return j; + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /// @brief parser event types + /// @sa https://json.nlohmann.me/api/basic_json/parse_event_t/ + using parse_event_t = detail::parse_event_t; + + /// @brief per-element parser callback type + /// @sa https://json.nlohmann.me/api/basic_json/parser_callback_t/ + using parser_callback_t = detail::parser_callback_t; + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /// @brief create an empty value with a given type + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(const value_t v) + : m_type(v), m_value(v) + { + assert_invariant(); + } + + /// @brief create a null object + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(std::nullptr_t = nullptr) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /// @brief create a JSON value from compatible types + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < typename CompatibleType, + typename U = detail::uncvref_t, + detail::enable_if_t < + !detail::is_basic_json::value && detail::is_compatible_type::value, int > = 0 > + basic_json(CompatibleType && val) noexcept(noexcept( // NOLINT(bugprone-forwarding-reference-overload,bugprone-exception-escape) + JSONSerializer::to_json(std::declval(), + std::forward(val)))) + { + JSONSerializer::to_json(*this, std::forward(val)); + set_parents(); + assert_invariant(); + } + + /// @brief create a JSON value from an existing one + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < typename BasicJsonType, + detail::enable_if_t < + detail::is_basic_json::value&& !std::is_same::value, int > = 0 > + basic_json(const BasicJsonType& val) + { + using other_boolean_t = typename BasicJsonType::boolean_t; + using other_number_float_t = typename BasicJsonType::number_float_t; + using other_number_integer_t = typename BasicJsonType::number_integer_t; + using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using other_string_t = typename BasicJsonType::string_t; + using other_object_t = typename BasicJsonType::object_t; + using other_array_t = typename BasicJsonType::array_t; + using other_binary_t = typename BasicJsonType::binary_t; + + switch (val.type()) + { + case value_t::boolean: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_float: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_integer: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_unsigned: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::string: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::object: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::array: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::binary: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::null: + *this = nullptr; + break; + case value_t::discarded: + m_type = value_t::discarded; + break; + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + set_parents(); + assert_invariant(); + } + + /// @brief create a container (array or object) from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(initializer_list_t init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const detail::json_ref& element_ref) + { + return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[0].is_string(); + }); + + // adjust type if type deduction is not wanted + if (!type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object)) + { + JSON_THROW(type_error::create(301, "cannot create object from initializer list", basic_json())); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + for (auto& element_ref : init) + { + auto element = element_ref.moved_or_copied(); + m_value.object->emplace( + std::move(*((*element.m_value.array)[0].m_value.string)), + std::move((*element.m_value.array)[1])); + } + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create(init.begin(), init.end()); + } + + set_parents(); + assert_invariant(); + } + + /// @brief explicitly create a binary array (without subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(const typename binary_t::container_type& init) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = init; + return res; + } + + /// @brief explicitly create a binary array (with subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = binary_t(init, subtype); + return res; + } + + /// @brief explicitly create a binary array + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(typename binary_t::container_type&& init) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = std::move(init); + return res; + } + + /// @brief explicitly create a binary array (with subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = binary_t(std::move(init), subtype); + return res; + } + + /// @brief explicitly create an array from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/array/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json array(initializer_list_t init = {}) + { + return basic_json(init, false, value_t::array); + } + + /// @brief explicitly create an object from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/object/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json object(initializer_list_t init = {}) + { + return basic_json(init, false, value_t::object); + } + + /// @brief construct an array with count copies of given value + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create(cnt, val); + set_parents(); + assert_invariant(); + } + + /// @brief construct a JSON container given an iterator range + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < class InputIT, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type = 0 > + basic_json(InputIT first, InputIT last) + { + JSON_ASSERT(first.m_object != nullptr); + JSON_ASSERT(last.m_object != nullptr); + + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(201, "iterators are not compatible", basic_json())); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() + || !last.m_it.primitive_iterator.is_end())) + { + JSON_THROW(invalid_iterator::create(204, "iterators out of range", *first.m_object)); + } + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + case value_t::binary: + { + m_value = *first.m_object->m_value.binary; + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " + std::string(first.m_object->type_name()), *first.m_object)); + } + + set_parents(); + assert_invariant(); + } + + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + template, + std::is_same>::value, int> = 0 > + basic_json(const JsonRef& ref) : basic_json(ref.moved_or_copied()) {} + + /// @brief copy constructor + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + case value_t::binary: + { + m_value = *other.m_value.binary; + break; + } + + case value_t::null: + case value_t::discarded: + default: + break; + } + + set_parents(); + assert_invariant(); + } + + /// @brief move constructor + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(false); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + set_parents(); + assert_invariant(); + } + + /// @brief copy assignment + /// @sa https://json.nlohmann.me/api/basic_json/operator=/ + basic_json& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + set_parents(); + assert_invariant(); + return *this; + } + + /// @brief destructor + /// @sa https://json.nlohmann.me/api/basic_json/~basic_json/ + ~basic_json() noexcept + { + assert_invariant(false); + m_value.destroy(m_type); + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /// @brief serialization + /// @sa https://json.nlohmann.me/api/basic_json/dump/ + string_t dump(const int indent = -1, + const char indent_char = ' ', + const bool ensure_ascii = false, + const error_handler_t error_handler = error_handler_t::strict) const + { + string_t result; + serializer s(detail::output_adapter(result), indent_char, error_handler); + + if (indent >= 0) + { + s.dump(*this, true, ensure_ascii, static_cast(indent)); + } + else + { + s.dump(*this, false, ensure_ascii, 0); + } + + return result; + } + + /// @brief return the type of the JSON value (explicit) + /// @sa https://json.nlohmann.me/api/basic_json/type/ + constexpr value_t type() const noexcept + { + return m_type; + } + + /// @brief return whether type is primitive + /// @sa https://json.nlohmann.me/api/basic_json/is_primitive/ + constexpr bool is_primitive() const noexcept + { + return is_null() || is_string() || is_boolean() || is_number() || is_binary(); + } + + /// @brief return whether type is structured + /// @sa https://json.nlohmann.me/api/basic_json/is_structured/ + constexpr bool is_structured() const noexcept + { + return is_array() || is_object(); + } + + /// @brief return whether value is null + /// @sa https://json.nlohmann.me/api/basic_json/is_null/ + constexpr bool is_null() const noexcept + { + return m_type == value_t::null; + } + + /// @brief return whether value is a boolean + /// @sa https://json.nlohmann.me/api/basic_json/is_boolean/ + constexpr bool is_boolean() const noexcept + { + return m_type == value_t::boolean; + } + + /// @brief return whether value is a number + /// @sa https://json.nlohmann.me/api/basic_json/is_number/ + constexpr bool is_number() const noexcept + { + return is_number_integer() || is_number_float(); + } + + /// @brief return whether value is an integer number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/ + constexpr bool is_number_integer() const noexcept + { + return m_type == value_t::number_integer || m_type == value_t::number_unsigned; + } + + /// @brief return whether value is an unsigned integer number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/ + constexpr bool is_number_unsigned() const noexcept + { + return m_type == value_t::number_unsigned; + } + + /// @brief return whether value is a floating-point number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_float/ + constexpr bool is_number_float() const noexcept + { + return m_type == value_t::number_float; + } + + /// @brief return whether value is an object + /// @sa https://json.nlohmann.me/api/basic_json/is_object/ + constexpr bool is_object() const noexcept + { + return m_type == value_t::object; + } + + /// @brief return whether value is an array + /// @sa https://json.nlohmann.me/api/basic_json/is_array/ + constexpr bool is_array() const noexcept + { + return m_type == value_t::array; + } + + /// @brief return whether value is a string + /// @sa https://json.nlohmann.me/api/basic_json/is_string/ + constexpr bool is_string() const noexcept + { + return m_type == value_t::string; + } + + /// @brief return whether value is a binary array + /// @sa https://json.nlohmann.me/api/basic_json/is_binary/ + constexpr bool is_binary() const noexcept + { + return m_type == value_t::binary; + } + + /// @brief return whether value is discarded + /// @sa https://json.nlohmann.me/api/basic_json/is_discarded/ + constexpr bool is_discarded() const noexcept + { + return m_type == value_t::discarded; + } + + /// @brief return the type of the JSON value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/operator_value_t/ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get a boolean (explicit) + boolean_t get_impl(boolean_t* /*unused*/) const + { + if (JSON_HEDLEY_LIKELY(is_boolean())) + { + return m_value.boolean; + } + + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(type_name()), *this)); + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t* /*unused*/) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t* /*unused*/) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t* /*unused*/) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (binary) + binary_t* get_impl_ptr(binary_t* /*unused*/) noexcept + { + return is_binary() ? m_value.binary : nullptr; + } + + /// get a pointer to the value (binary) + constexpr const binary_t* get_impl_ptr(const binary_t* /*unused*/) const noexcept + { + return is_binary() ? m_value.binary : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This function helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw type_error.303 if ReferenceType does not match underlying value + type of the current JSON + */ + template + static ReferenceType get_ref_impl(ThisType& obj) + { + // delegate the call to get_ptr<>() + auto* ptr = obj.template get_ptr::type>(); + + if (JSON_HEDLEY_LIKELY(ptr != nullptr)) + { + return *ptr; + } + + JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + std::string(obj.type_name()), obj)); + } + + public: + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /// @brief get a pointer value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/ + template::value, int>::type = 0> + auto get_ptr() noexcept -> decltype(std::declval().get_impl_ptr(std::declval())) + { + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast(nullptr)); + } + + /// @brief get a pointer value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/ + template < typename PointerType, typename std::enable_if < + std::is_pointer::value&& + std::is_const::type>::value, int >::type = 0 > + constexpr auto get_ptr() const noexcept -> decltype(std::declval().get_impl_ptr(std::declval())) + { + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast(nullptr)); + } + + private: + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value + which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer has a `from_json()` method of the form + `void from_json(const basic_json&, ValueType&)`, and + - @ref json_serializer does not have a `from_json()` method of + the form `ValueType from_json(const basic_json&)` + + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,get__ValueType_const} + + @since version 2.1.0 + */ + template < typename ValueType, + detail::enable_if_t < + detail::is_default_constructible::value&& + detail::has_from_json::value, + int > = 0 > + ValueType get_impl(detail::priority_tag<0> /*unused*/) const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), std::declval()))) + { + auto ret = ValueType(); + JSONSerializer::from_json(*this, ret); + return ret; + } + + /*! + @brief get a value (explicit); special case + + Explicit type conversion between the JSON value and a compatible value + which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + return JSONSerializer::from_json(*this); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json and + - @ref json_serializer has a `from_json()` method of the form + `ValueType from_json(const basic_json&)` + + @note If @ref json_serializer has both overloads of + `from_json()`, this one is chosen. + + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @since version 2.1.0 + */ + template < typename ValueType, + detail::enable_if_t < + detail::has_non_default_from_json::value, + int > = 0 > + ValueType get_impl(detail::priority_tag<1> /*unused*/) const noexcept(noexcept( + JSONSerializer::from_json(std::declval()))) + { + return JSONSerializer::from_json(*this); + } + + /*! + @brief get special-case overload + + This overloads converts the current @ref basic_json in a different + @ref basic_json type + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this, converted into @a BasicJsonType + + @complexity Depending on the implementation of the called `from_json()` + method. + + @since version 3.2.0 + */ + template < typename BasicJsonType, + detail::enable_if_t < + detail::is_basic_json::value, + int > = 0 > + BasicJsonType get_impl(detail::priority_tag<2> /*unused*/) const + { + return *this; + } + + /*! + @brief get special-case overload + + This overloads avoids a lot of template boilerplate, it can be seen as the + identity method + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this + + @complexity Constant. + + @since version 2.1.0 + */ + template::value, + int> = 0> + basic_json get_impl(detail::priority_tag<3> /*unused*/) const + { + return *this; + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template::value, + int> = 0> + constexpr auto get_impl(detail::priority_tag<4> /*unused*/) const noexcept + -> decltype(std::declval().template get_ptr()) + { + // delegate the call to get_ptr + return get_ptr(); + } + + public: + /*! + @brief get a (pointer) value (explicit) + + Performs explicit type conversion between the JSON value and a compatible value if required. + + - If the requested type is a pointer to the internally stored JSON value that pointer is returned. + No copies are made. + + - If the requested type is the current @ref basic_json, or a different @ref basic_json convertible + from the current @ref basic_json. + + - Otherwise the value is converted by calling the @ref json_serializer `from_json()` + method. + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @tparam ValueType if necessary + + @throw what @ref json_serializer `from_json()` method throws if conversion is required + + @since version 2.1.0 + */ + template < typename ValueTypeCV, typename ValueType = detail::uncvref_t> +#if defined(JSON_HAS_CPP_14) + constexpr +#endif + auto get() const noexcept( + noexcept(std::declval().template get_impl(detail::priority_tag<4> {}))) + -> decltype(std::declval().template get_impl(detail::priority_tag<4> {})) + { + // we cannot static_assert on ValueTypeCV being non-const, because + // there is support for get(), which is why we + // still need the uncvref + static_assert(!std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + return get_impl(detail::priority_tag<4> {}); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa see @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template::value, int>::type = 0> + auto get() noexcept -> decltype(std::declval().template get_ptr()) + { + // delegate the call to get_ptr + return get_ptr(); + } + + /// @brief get a value (explicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_to/ + template < typename ValueType, + detail::enable_if_t < + !detail::is_basic_json::value&& + detail::has_from_json::value, + int > = 0 > + ValueType & get_to(ValueType& v) const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), v))) + { + JSONSerializer::from_json(*this, v); + return v; + } + + // specialization to allow calling get_to with a basic_json value + // see https://github.com/nlohmann/json/issues/2175 + template::value, + int> = 0> + ValueType & get_to(ValueType& v) const + { + v = *this; + return v; + } + + template < + typename T, std::size_t N, + typename Array = T (&)[N], // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + detail::enable_if_t < + detail::has_from_json::value, int > = 0 > + Array get_to(T (&v)[N]) const // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + noexcept(noexcept(JSONSerializer::from_json( + std::declval(), v))) + { + JSONSerializer::from_json(*this, v); + return v; + } + + /// @brief get a reference value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ref/ + template::value, int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /// @brief get a reference value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ref/ + template < typename ReferenceType, typename std::enable_if < + std::is_reference::value&& + std::is_const::type>::value, int >::type = 0 > + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw type_error.302 in case passed type @a ValueType is incompatible + to the JSON value type (e.g., the JSON value is of type boolean, but a + string is requested); see example below + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename std::enable_if < + detail::conjunction < + detail::negation>, + detail::negation>>, + detail::negation>, + detail::negation>, + detail::negation>>, + +#if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) && _MSC_VER >= 1910 && _MSC_VER <= 1914)) + detail::negation>, +#endif + detail::is_detected_lazy + >::value, int >::type = 0 > + JSON_EXPLICIT operator ValueType() const + { + // delegate the call to get<>() const + return get(); + } + + /// @brief get a binary value + /// @sa https://json.nlohmann.me/api/basic_json/get_binary/ + binary_t& get_binary() + { + if (!is_binary()) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(type_name()), *this)); + } + + return *get_ptr(); + } + + /// @brief get a binary value + /// @sa https://json.nlohmann.me/api/basic_json/get_binary/ + const binary_t& get_binary() const + { + if (!is_binary()) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(type_name()), *this)); + } + + return *get_ptr(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /// @brief access specified array element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(size_type idx) + { + // at only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + JSON_TRY + { + return set_parent(m_value.array->at(idx)); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified array element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(size_type idx) const + { + // at only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + JSON_TRY + { + return m_value.array->at(idx); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified object element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_TRY + { + return set_parent(m_value.object->at(key)); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified object element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_TRY + { + return m_value.object->at(key); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified array element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create(); + assert_invariant(); + } + + // operator[] only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { +#if JSON_DIAGNOSTICS + // remember array size & capacity before resizing + const auto old_size = m_value.array->size(); + const auto old_capacity = m_value.array->capacity(); +#endif + m_value.array->resize(idx + 1); + +#if JSON_DIAGNOSTICS + if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity)) + { + // capacity has changed: update all parents + set_parents(); + } + else + { + // set parent for values added above + set_parents(begin() + static_cast(old_size), static_cast(idx + 1 - old_size)); + } +#endif + assert_invariant(); + } + + return m_value.array->operator[](idx); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified array element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + return m_value.array->operator[](idx); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + // operator[] only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return set_parent(m_value.object->operator[](key)); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_ASSERT(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + template + JSON_HEDLEY_NON_NULL(2) + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return set_parent(m_value.object->operator[](key)); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + template + JSON_HEDLEY_NON_NULL(2) + const_reference operator[](T* key) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_ASSERT(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// using std::is_convertible in a std::enable_if will fail when using explicit conversions + template < class ValueType, typename std::enable_if < + detail::is_getable::value + && !std::is_same::value, int >::type = 0 > + ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return it->template get(); + } + + return default_value; + } + + JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// overload for a default value of type const char* + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /// @brief access specified object element via JSON Pointer with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + template::value, int>::type = 0> + ValueType value(const json_pointer& ptr, const ValueType& default_value) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + // if pointer resolves a value, return it or use default value + JSON_TRY + { + return ptr.get_checked(this).template get(); + } + JSON_INTERNAL_CATCH (out_of_range&) + { + return default_value; + } + } + + JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element via JSON Pointer with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// overload for a default value of type const char* + JSON_HEDLEY_NON_NULL(3) + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /// @brief access the first element + /// @sa https://json.nlohmann.me/api/basic_json/front/ + reference front() + { + return *begin(); + } + + /// @brief access the first element + /// @sa https://json.nlohmann.me/api/basic_json/front/ + const_reference front() const + { + return *cbegin(); + } + + /// @brief access the last element + /// @sa https://json.nlohmann.me/api/basic_json/back/ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /// @brief access the last element + /// @sa https://json.nlohmann.me/api/basic_json/back/ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /// @brief remove element given an iterator + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + template < class IteratorType, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type + = 0 > + IteratorType erase(IteratorType pos) + { + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(this != pos.m_object)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + case value_t::binary: + { + if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin())) + { + JSON_THROW(invalid_iterator::create(205, "iterator out of range", *this)); + } + + if (is_string()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.string); + std::allocator_traits::deallocate(alloc, m_value.string, 1); + m_value.string = nullptr; + } + else if (is_binary()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.binary); + std::allocator_traits::deallocate(alloc, m_value.binary, 1); + m_value.binary = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + return result; + } + + /// @brief remove elements given an iterator range + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + template < class IteratorType, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type + = 0 > + IteratorType erase(IteratorType first, IteratorType last) + { + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object)) + { + JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value", *this)); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + case value_t::binary: + { + if (JSON_HEDLEY_LIKELY(!first.m_it.primitive_iterator.is_begin() + || !last.m_it.primitive_iterator.is_end())) + { + JSON_THROW(invalid_iterator::create(204, "iterators out of range", *this)); + } + + if (is_string()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.string); + std::allocator_traits::deallocate(alloc, m_value.string, 1); + m_value.string = nullptr; + } + else if (is_binary()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.binary); + std::allocator_traits::deallocate(alloc, m_value.binary, 1); + m_value.binary = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + return result; + } + + /// @brief remove element from a JSON object given a key + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return m_value.object->erase(key); + } + + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + /// @brief remove element from a JSON array given an index + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + void erase(const size_type idx) + { + // this erase only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + if (JSON_HEDLEY_UNLIKELY(idx >= size())) + { + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + + m_value.array->erase(m_value.array->begin() + static_cast(idx)); + } + else + { + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /// @brief find an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/find/ + template + iterator find(KeyT&& key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(std::forward(key)); + } + + return result; + } + + /// @brief find an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/find/ + template + const_iterator find(KeyT&& key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(std::forward(key)); + } + + return result; + } + + /// @brief returns the number of occurrences of a key in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/count/ + template + size_type count(KeyT&& key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(std::forward(key)) : 0; + } + + /// @brief check the existence of an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/contains/ + template < typename KeyT, typename std::enable_if < + !std::is_same::type, json_pointer>::value, int >::type = 0 > + bool contains(KeyT && key) const + { + return is_object() && m_value.object->find(std::forward(key)) != m_value.object->end(); + } + + /// @brief check the existence of an element in a JSON object given a JSON pointer + /// @sa https://json.nlohmann.me/api/basic_json/contains/ + bool contains(const json_pointer& ptr) const + { + return ptr.contains(this); + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /// @brief returns an iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/begin/ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /// @brief returns an iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/begin/ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /// @brief returns a const iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/cbegin/ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/end/ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/end/ + const_iterator end() const noexcept + { + return cend(); + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/cend/ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /// @brief returns an iterator to the reverse-beginning + /// @sa https://json.nlohmann.me/api/basic_json/rbegin/ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /// @brief returns an iterator to the reverse-beginning + /// @sa https://json.nlohmann.me/api/basic_json/rbegin/ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /// @brief returns an iterator to the reverse-end + /// @sa https://json.nlohmann.me/api/basic_json/rend/ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /// @brief returns an iterator to the reverse-end + /// @sa https://json.nlohmann.me/api/basic_json/rend/ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /// @brief returns a const reverse iterator to the last element + /// @sa https://json.nlohmann.me/api/basic_json/crbegin/ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /// @brief returns a const reverse iterator to one before the first + /// @sa https://json.nlohmann.me/api/basic_json/crend/ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + public: + /// @brief wrapper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + /// @deprecated This function is deprecated since 3.1.0 and will be removed in + /// version 4.0.0 of the library. Please use @ref items() instead; + /// that is, replace `json::iterator_wrapper(j)` with `j.items()`. + JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items()) + static iteration_proxy iterator_wrapper(reference ref) noexcept + { + return ref.items(); + } + + /// @brief wrapper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + /// @deprecated This function is deprecated since 3.1.0 and will be removed in + /// version 4.0.0 of the library. Please use @ref items() instead; + /// that is, replace `json::iterator_wrapper(j)` with `j.items()`. + JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items()) + static iteration_proxy iterator_wrapper(const_reference ref) noexcept + { + return ref.items(); + } + + /// @brief helper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + iteration_proxy items() noexcept + { + return iteration_proxy(*this); + } + + /// @brief helper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + iteration_proxy items() const noexcept + { + return iteration_proxy(*this); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /// @brief checks whether the container is empty. + /// @sa https://json.nlohmann.me/api/basic_json/empty/ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types are nonempty + return false; + } + } + } + + /// @brief returns the number of elements + /// @sa https://json.nlohmann.me/api/basic_json/size/ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types have size 1 + return 1; + } + } + } + + /// @brief returns the maximum possible number of elements + /// @sa https://json.nlohmann.me/api/basic_json/max_size/ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /// @brief clears the contents + /// @sa https://json.nlohmann.me/api/basic_json/clear/ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::binary: + { + m_value.binary->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + case value_t::null: + case value_t::discarded: + default: + break; + } + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + const auto old_capacity = m_value.array->capacity(); + m_value.array->push_back(std::move(val)); + set_parent(m_value.array->back(), old_capacity); + // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + const auto old_capacity = m_value.array->capacity(); + m_value.array->push_back(val); + set_parent(m_value.array->back(), old_capacity); + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to object + auto res = m_value.object->insert(val); + set_parent(res.first->second); + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(initializer_list_t init) + { + if (is_object() && init.size() == 2 && (*init.begin())->is_string()) + { + basic_json&& key = init.begin()->moved_or_copied(); + push_back(typename object_t::value_type( + std::move(key.get_ref()), (init.begin() + 1)->moved_or_copied())); + } + else + { + push_back(basic_json(init)); + } + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(initializer_list_t init) + { + push_back(init); + return *this; + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/emplace_back/ + template + reference emplace_back(Args&& ... args) + { + // emplace_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (perfect forwarding) + const auto old_capacity = m_value.array->capacity(); + m_value.array->emplace_back(std::forward(args)...); + return set_parent(m_value.array->back(), old_capacity); + } + + /// @brief add an object to an object if key does not exist + /// @sa https://json.nlohmann.me/api/basic_json/emplace/ + template + std::pair emplace(Args&& ... args) + { + // emplace only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object()))) + { + JSON_THROW(type_error::create(311, "cannot use emplace() with " + std::string(type_name()), *this)); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array (perfect forwarding) + auto res = m_value.object->emplace(std::forward(args)...); + set_parent(res.first->second); + + // create result iterator and set iterator to the result of emplace + auto it = begin(); + it.m_it.object_iterator = res.first; + + // return pair of iterator and boolean + return {it, res.second}; + } + + /// Helper for insertion of an iterator + /// @note: This uses std::distance to support GCC 4.8, + /// see https://github.com/nlohmann/json/pull/1257 + template + iterator insert_iterator(const_iterator pos, Args&& ... args) + { + iterator result(this); + JSON_ASSERT(m_value.array != nullptr); + + auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator); + m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); + result.m_it.array_iterator = m_value.array->begin() + insert_pos; + + // This could have been written as: + // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + // but the return value of insert is missing in GCC 4.8, so it is written this way instead. + + set_parents(); + return result; + } + + /// @brief inserts element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, val); + } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + /// @brief inserts element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /// @brief inserts copies of element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, cnt, val); + } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + /// @brief inserts range of elements into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (JSON_HEDLEY_UNLIKELY(!is_array())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + if (JSON_HEDLEY_UNLIKELY(first.m_object == this)) + { + JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator); + } + + /// @brief inserts elements from initializer list into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, initializer_list_t ilist) + { + // insert only works for arrays + if (JSON_HEDLEY_UNLIKELY(!is_array())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, ilist.begin(), ilist.end()); + } + + /// @brief inserts range of elements into object + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + void insert(const_iterator first, const_iterator last) + { + // insert only works for objects + if (JSON_HEDLEY_UNLIKELY(!is_object())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + // passed iterators must belong to objects + if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects", *this)); + } + + m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); + } + + /// @brief updates a JSON object from another object, overwriting existing keys + /// @sa https://json.nlohmann.me/api/basic_json/update/ + void update(const_reference j, bool merge_objects = false) + { + update(j.begin(), j.end(), merge_objects); + } + + /// @brief updates a JSON object from another object, overwriting existing keys + /// @sa https://json.nlohmann.me/api/basic_json/update/ + void update(const_iterator first, const_iterator last, bool merge_objects = false) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + if (JSON_HEDLEY_UNLIKELY(!is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name()), *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + // passed iterators must belong to objects + if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(first.m_object->type_name()), *first.m_object)); + } + + for (auto it = first; it != last; ++it) + { + if (merge_objects && it.value().is_object()) + { + auto it2 = m_value.object->find(it.key()); + if (it2 != m_value.object->end()) + { + it2->second.update(it.value(), true); + continue; + } + } + m_value.object->operator[](it.key()) = it.value(); +#if JSON_DIAGNOSTICS + m_value.object->operator[](it.key()).m_parent = this; +#endif + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + + set_parents(); + other.set_parents(); + assert_invariant(); + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + friend void swap(reference left, reference right) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + left.swap(right); + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(array_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + std::swap(*(m_value.array), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(object_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + std::swap(*(m_value.object), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(string_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_string())) + { + std::swap(*(m_value.string), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(binary_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_binary())) + { + std::swap(*(m_value.binary), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(typename binary_t::container_type& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_binary())) + { + std::swap(*(m_value.binary), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @} + + public: + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + return *lhs.m_value.array == *rhs.m_value.array; + + case value_t::object: + return *lhs.m_value.object == *rhs.m_value.object; + + case value_t::null: + return true; + + case value_t::string: + return *lhs.m_value.string == *rhs.m_value.string; + + case value_t::boolean: + return lhs.m_value.boolean == rhs.m_value.boolean; + + case value_t::number_integer: + return lhs.m_value.number_integer == rhs.m_value.number_integer; + + case value_t::number_unsigned: + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + + case value_t::number_float: + return lhs.m_value.number_float == rhs.m_value.number_float; + + case value_t::binary: + return *lhs.m_value.binary == *rhs.m_value.binary; + + case value_t::discarded: + default: + return false; + } + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + } + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + template::value, int>::type = 0> + friend bool operator==(const_reference lhs, ScalarType rhs) noexcept + { + return lhs == basic_json(rhs); + } + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + template::value, int>::type = 0> + friend bool operator==(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) == rhs; + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs == rhs); + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + template::value, int>::type = 0> + friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs != basic_json(rhs); + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + template::value, int>::type = 0> + friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) != rhs; + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + // note parentheses are necessary, see + // https://github.com/nlohmann/json/issues/1530 + return (*lhs.m_value.array) < (*rhs.m_value.array); + + case value_t::object: + return (*lhs.m_value.object) < (*rhs.m_value.object); + + case value_t::null: + return false; + + case value_t::string: + return (*lhs.m_value.string) < (*rhs.m_value.string); + + case value_t::boolean: + return (lhs.m_value.boolean) < (rhs.m_value.boolean); + + case value_t::number_integer: + return (lhs.m_value.number_integer) < (rhs.m_value.number_integer); + + case value_t::number_unsigned: + return (lhs.m_value.number_unsigned) < (rhs.m_value.number_unsigned); + + case value_t::number_float: + return (lhs.m_value.number_float) < (rhs.m_value.number_float); + + case value_t::binary: + return (*lhs.m_value.binary) < (*rhs.m_value.binary); + + case value_t::discarded: + default: + return false; + } + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + template::value, int>::type = 0> + friend bool operator<(const_reference lhs, ScalarType rhs) noexcept + { + return lhs < basic_json(rhs); + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + template::value, int>::type = 0> + friend bool operator<(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) < rhs; + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return !(rhs < lhs); + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + template::value, int>::type = 0> + friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs <= basic_json(rhs); + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + template::value, int>::type = 0> + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) <= rhs; + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs <= rhs); + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + template::value, int>::type = 0> + friend bool operator>(const_reference lhs, ScalarType rhs) noexcept + { + return lhs > basic_json(rhs); + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + template::value, int>::type = 0> + friend bool operator>(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) > rhs; + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs < rhs); + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + template::value, int>::type = 0> + friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs >= basic_json(rhs); + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + template::value, int>::type = 0> + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) >= rhs; + } + + /// @} + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ +#ifndef JSON_NO_IO + /// @brief serialize to stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = o.width() > 0; + const auto indentation = pretty_print ? o.width() : 0; + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // do the actual serialization + serializer s(detail::output_adapter(o), o.fill()); + s.dump(j, pretty_print, false, static_cast(indentation)); + return o; + } + + /// @brief serialize to stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/ + /// @deprecated This function is deprecated since 3.0.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// operator<<(std::ostream&, const basic_json&) instead; that is, + /// replace calls like `j >> o;` with `o << j;`. + JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator<<(std::ostream&, const basic_json&)) + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } +#endif // JSON_NO_IO + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /// @brief deserialize from a compatible input + /// @sa https://json.nlohmann.me/api/basic_json/parse/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json parse(InputType&& i, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(detail::input_adapter(std::forward(i)), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + /// @brief deserialize from a pair of character iterators + /// @sa https://json.nlohmann.me/api/basic_json/parse/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json parse(IteratorType first, + IteratorType last, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, parse(ptr, ptr + len)) + static basic_json parse(detail::span_input_adapter&& i, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(i.get(), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + /// @brief check if the input is valid JSON + /// @sa https://json.nlohmann.me/api/basic_json/accept/ + template + static bool accept(InputType&& i, + const bool ignore_comments = false) + { + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments).accept(true); + } + + /// @brief check if the input is valid JSON + /// @sa https://json.nlohmann.me/api/basic_json/accept/ + template + static bool accept(IteratorType first, IteratorType last, + const bool ignore_comments = false) + { + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments).accept(true); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, accept(ptr, ptr + len)) + static bool accept(detail::span_input_adapter&& i, + const bool ignore_comments = false) + { + return parser(i.get(), nullptr, false, ignore_comments).accept(true); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + template + JSON_HEDLEY_NON_NULL(2) + static bool sax_parse(InputType&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = detail::input_adapter(std::forward(i)); + return format == input_format_t::json + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + template + JSON_HEDLEY_NON_NULL(3) + static bool sax_parse(IteratorType first, IteratorType last, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = detail::input_adapter(std::move(first), std::move(last)); + return format == input_format_t::json + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + /// @deprecated This function is deprecated since 3.8.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// sax_parse(ptr, ptr + len) instead. + template + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, sax_parse(ptr, ptr + len, ...)) + JSON_HEDLEY_NON_NULL(2) + static bool sax_parse(detail::span_input_adapter&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = i.get(); + return format == input_format_t::json + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } +#ifndef JSON_NO_IO + /// @brief deserialize from stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/ + /// @deprecated This stream operator is deprecated since 3.0.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// operator>>(std::istream&, basic_json&) instead; that is, + /// replace calls like `j << i;` with `i >> j;`. + JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator>>(std::istream&, basic_json&)) + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + return operator>>(i, j); + } + + /// @brief deserialize from stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + parser(detail::input_adapter(i)).parse(false, j); + return i; + } +#endif // JSON_NO_IO + /// @} + + /////////////////////////// + // convenience functions // + /////////////////////////// + + /// @brief return the type as string + /// @sa https://json.nlohmann.me/api/basic_json/type_name/ + JSON_HEDLEY_RETURNS_NON_NULL + const char* type_name() const noexcept + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::binary: + return "binary"; + case value_t::discarded: + return "discarded"; + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + default: + return "number"; + } + } + + + JSON_PRIVATE_UNLESS_TESTED: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + +#if JSON_DIAGNOSTICS + /// a pointer to a parent value (for debugging purposes) + basic_json* m_parent = nullptr; +#endif + + ////////////////////////////////////////// + // binary serialization/deserialization // + ////////////////////////////////////////// + + /// @name binary serialization/deserialization support + /// @{ + + public: + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static std::vector to_cbor(const basic_json& j) + { + std::vector result; + to_cbor(j, result); + return result; + } + + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static void to_cbor(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_cbor(j); + } + + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static void to_cbor(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_cbor(j); + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static std::vector to_msgpack(const basic_json& j) + { + std::vector result; + to_msgpack(j, result); + return result; + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static void to_msgpack(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_msgpack(j); + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static void to_msgpack(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_msgpack(j); + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static std::vector to_ubjson(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector result; + to_ubjson(j, result, use_size, use_type); + return result; + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type); + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static std::vector to_bson(const basic_json& j) + { + std::vector result; + to_bson(j, result); + return result; + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static void to_bson(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bson(j); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static void to_bson(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bson(j); + } + + /// @brief create a JSON value from an input in CBOR format + /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_cbor(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in CBOR format + /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_cbor(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len)) + static basic_json from_cbor(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + return from_cbor(ptr, ptr + len, strict, allow_exceptions, tag_handler); + } + + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len)) + static basic_json from_cbor(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in MessagePack format + /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_msgpack(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in MessagePack format + /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_msgpack(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len)) + static basic_json from_msgpack(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_msgpack(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len)) + static basic_json from_msgpack(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in UBJSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_ubjson(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in UBJSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_ubjson(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len)) + static basic_json from_ubjson(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_ubjson(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len)) + static basic_json from_ubjson(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_bson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bson(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_bson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bson(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len)) + static basic_json from_bson(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_bson(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len)) + static basic_json from_bson(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + /// @} + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /// @brief return flattened JSON value + /// @sa https://json.nlohmann.me/api/basic_json/flatten/ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /// @brief unflatten a previously flattened JSON value + /// @sa https://json.nlohmann.me/api/basic_json/unflatten/ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /// @brief applies a JSON patch + /// @sa https://json.nlohmann.me/api/basic_json/patch/ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string & op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.empty()) + { + result = val; + return; + } + + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.back(); + ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = json_pointer::array_index(last_path); + if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) + { + // avoid undefined behavior + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", parent)); + } + + // default case: insert add offset + parent.insert(parent.begin() + static_cast(idx), val); + } + break; + } + + // if there exists a parent it cannot be primitive + case value_t::string: // LCOV_EXCL_LINE + case value_t::boolean: // LCOV_EXCL_LINE + case value_t::number_integer: // LCOV_EXCL_LINE + case value_t::number_unsigned: // LCOV_EXCL_LINE + case value_t::number_float: // LCOV_EXCL_LINE + case value_t::binary: // LCOV_EXCL_LINE + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [this, &result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.back(); + ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (JSON_HEDLEY_LIKELY(it != parent.end())) + { + parent.erase(it); + } + else + { + JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found", *this)); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(json_pointer::array_index(last_path)); + } + }; + + // type check: top level value must be an array + if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array())) + { + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects", json_patch)); + } + + // iterate and apply the operations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json & + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (JSON_HEDLEY_UNLIKELY(it == val.m_value.object->end())) + { + // NOLINTNEXTLINE(performance-inefficient-string-concatenation) + JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'", val)); + } + + // check if result is of type string + if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) + { + // NOLINTNEXTLINE(performance-inefficient-string-concatenation) + JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'", val)); + } + + // no error: return value + return it->second; + }; + + // type check: every element of the array must be an object + if (JSON_HEDLEY_UNLIKELY(!val.is_object())) + { + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects", val)); + } + + // collect mandatory members + const auto op = get_value("op", "op", true).template get(); + const auto path = get_value(op, "path", true).template get(); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const auto from_path = get_value("move", "from", true).template get(); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const auto from_path = get_value("copy", "from", true).template get(); + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The copy is functionally identical to an "add" + // operation at the target location using the value + // specified in the "from" member. + operation_add(ptr, v); + break; + } + + case patch_operations::test: + { + bool success = false; + JSON_TRY + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + success = (result.at(ptr) == get_value("test", "value", false)); + } + JSON_INTERNAL_CATCH (out_of_range&) + { + // ignore out of range errors: success remains false + } + + // throw an exception if test fails + if (JSON_HEDLEY_UNLIKELY(!success)) + { + JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump(), val)); + } + + break; + } + + case patch_operations::invalid: + default: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid", val)); + } + } + } + + return result; + } + + /// @brief creates a diff as a JSON patch + /// @sa https://json.nlohmann.me/api/basic_json/diff/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json diff(const basic_json& source, const basic_json& target, + const std::string& path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + return result; + } + + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + std::size_t i = 0; + while (i < source.size() && i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // We now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/-"}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + // escape the key name to be used in a JSON patch + const auto path_key = path + "/" + detail::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path_key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, {"path", path_key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto path_key = path + "/" + detail::escape(it.key()); + result.push_back( + { + {"op", "add"}, {"path", path_key}, + {"value", it.value()} + }); + } + } + + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + break; + } + } + + return result; + } + + /// @} + + //////////////////////////////// + // JSON Merge Patch functions // + //////////////////////////////// + + /// @name JSON Merge Patch functions + /// @{ + + /// @brief applies a JSON Merge Patch + /// @sa https://json.nlohmann.me/api/basic_json/merge_patch/ + void merge_patch(const basic_json& apply_patch) + { + if (apply_patch.is_object()) + { + if (!is_object()) + { + *this = object(); + } + for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it) + { + if (it.value().is_null()) + { + erase(it.key()); + } + else + { + operator[](it.key()).merge_patch(it.value()); + } + } + } + else + { + *this = apply_patch; + } + } + + /// @} +}; + +/// @brief user-defined to_string function for JSON values +/// @sa https://json.nlohmann.me/api/basic_json/to_string/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +std::string to_string(const NLOHMANN_BASIC_JSON_TPL& j) +{ + return j.dump(); +} + +} // namespace nlohmann + +/////////////////////// +// nonmember support // +/////////////////////// + +namespace std // NOLINT(cert-dcl58-cpp) +{ + +/// @brief hash value for JSON objects +/// @sa https://json.nlohmann.me/api/basic_json/std_hash/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct hash +{ + std::size_t operator()(const nlohmann::NLOHMANN_BASIC_JSON_TPL& j) const + { + return nlohmann::detail::hash(j); + } +}; + +// specialization for std::less +template<> +struct less< ::nlohmann::detail::value_t> // do not remove the space after '<', see https://github.com/nlohmann/json/pull/679 +{ + /*! + @brief compare two value_t enum values + @since version 3.0.0 + */ + bool operator()(nlohmann::detail::value_t lhs, + nlohmann::detail::value_t rhs) const noexcept + { + return nlohmann::detail::operator<(lhs, rhs); + } +}; + +// C++20 prohibit function specialization in the std namespace. +#ifndef JSON_HAS_CPP_20 + +/// @brief exchanges the values of two JSON objects +/// @sa https://json.nlohmann.me/api/basic_json/std_swap/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept( // NOLINT(readability-inconsistent-declaration-parameter-name) + is_nothrow_move_constructible::value&& // NOLINT(misc-redundant-expression) + is_nothrow_move_assignable::value) +{ + j1.swap(j2); +} + +#endif + +} // namespace std + +/// @brief user-defined string literal for JSON values +/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json/ +JSON_HEDLEY_NON_NULL(1) +inline nlohmann::json operator "" _json(const char* s, std::size_t n) +{ + return nlohmann::json::parse(s, s + n); +} + +/// @brief user-defined string literal for JSON pointer +/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json_pointer/ +JSON_HEDLEY_NON_NULL(1) +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n) +{ + return nlohmann::json::json_pointer(std::string(s, n)); +} + +// #include + + +// restore clang diagnostic settings +#if defined(__clang__) + #pragma clang diagnostic pop +#endif + +// clean up +#undef JSON_ASSERT +#undef JSON_INTERNAL_CATCH +#undef JSON_CATCH +#undef JSON_THROW +#undef JSON_TRY +#undef JSON_PRIVATE_UNLESS_TESTED +#undef JSON_HAS_CPP_11 +#undef JSON_HAS_CPP_14 +#undef JSON_HAS_CPP_17 +#undef JSON_HAS_CPP_20 +#undef JSON_HAS_FILESYSTEM +#undef JSON_HAS_EXPERIMENTAL_FILESYSTEM +#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION +#undef NLOHMANN_BASIC_JSON_TPL +#undef JSON_EXPLICIT +#undef NLOHMANN_CAN_CALL_STD_FUNC_IMPL + +// #include + + +#undef JSON_HEDLEY_ALWAYS_INLINE +#undef JSON_HEDLEY_ARM_VERSION +#undef JSON_HEDLEY_ARM_VERSION_CHECK +#undef JSON_HEDLEY_ARRAY_PARAM +#undef JSON_HEDLEY_ASSUME +#undef JSON_HEDLEY_BEGIN_C_DECLS +#undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#undef JSON_HEDLEY_CLANG_HAS_FEATURE +#undef JSON_HEDLEY_CLANG_HAS_WARNING +#undef JSON_HEDLEY_COMPCERT_VERSION +#undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#undef JSON_HEDLEY_CONCAT +#undef JSON_HEDLEY_CONCAT3 +#undef JSON_HEDLEY_CONCAT3_EX +#undef JSON_HEDLEY_CONCAT_EX +#undef JSON_HEDLEY_CONST +#undef JSON_HEDLEY_CONSTEXPR +#undef JSON_HEDLEY_CONST_CAST +#undef JSON_HEDLEY_CPP_CAST +#undef JSON_HEDLEY_CRAY_VERSION +#undef JSON_HEDLEY_CRAY_VERSION_CHECK +#undef JSON_HEDLEY_C_DECL +#undef JSON_HEDLEY_DEPRECATED +#undef JSON_HEDLEY_DEPRECATED_FOR +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#undef JSON_HEDLEY_DIAGNOSTIC_POP +#undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#undef JSON_HEDLEY_DMC_VERSION +#undef JSON_HEDLEY_DMC_VERSION_CHECK +#undef JSON_HEDLEY_EMPTY_BASES +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#undef JSON_HEDLEY_END_C_DECLS +#undef JSON_HEDLEY_FLAGS +#undef JSON_HEDLEY_FLAGS_CAST +#undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_BUILTIN +#undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_EXTENSION +#undef JSON_HEDLEY_GCC_HAS_FEATURE +#undef JSON_HEDLEY_GCC_HAS_WARNING +#undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#undef JSON_HEDLEY_GCC_VERSION +#undef JSON_HEDLEY_GCC_VERSION_CHECK +#undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#undef JSON_HEDLEY_GNUC_HAS_FEATURE +#undef JSON_HEDLEY_GNUC_HAS_WARNING +#undef JSON_HEDLEY_GNUC_VERSION +#undef JSON_HEDLEY_GNUC_VERSION_CHECK +#undef JSON_HEDLEY_HAS_ATTRIBUTE +#undef JSON_HEDLEY_HAS_BUILTIN +#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_HAS_EXTENSION +#undef JSON_HEDLEY_HAS_FEATURE +#undef JSON_HEDLEY_HAS_WARNING +#undef JSON_HEDLEY_IAR_VERSION +#undef JSON_HEDLEY_IAR_VERSION_CHECK +#undef JSON_HEDLEY_IBM_VERSION +#undef JSON_HEDLEY_IBM_VERSION_CHECK +#undef JSON_HEDLEY_IMPORT +#undef JSON_HEDLEY_INLINE +#undef JSON_HEDLEY_INTEL_CL_VERSION +#undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#undef JSON_HEDLEY_INTEL_VERSION +#undef JSON_HEDLEY_INTEL_VERSION_CHECK +#undef JSON_HEDLEY_IS_CONSTANT +#undef JSON_HEDLEY_IS_CONSTEXPR_ +#undef JSON_HEDLEY_LIKELY +#undef JSON_HEDLEY_MALLOC +#undef JSON_HEDLEY_MCST_LCC_VERSION +#undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#undef JSON_HEDLEY_MESSAGE +#undef JSON_HEDLEY_MSVC_VERSION +#undef JSON_HEDLEY_MSVC_VERSION_CHECK +#undef JSON_HEDLEY_NEVER_INLINE +#undef JSON_HEDLEY_NON_NULL +#undef JSON_HEDLEY_NO_ESCAPE +#undef JSON_HEDLEY_NO_RETURN +#undef JSON_HEDLEY_NO_THROW +#undef JSON_HEDLEY_NULL +#undef JSON_HEDLEY_PELLES_VERSION +#undef JSON_HEDLEY_PELLES_VERSION_CHECK +#undef JSON_HEDLEY_PGI_VERSION +#undef JSON_HEDLEY_PGI_VERSION_CHECK +#undef JSON_HEDLEY_PREDICT +#undef JSON_HEDLEY_PRINTF_FORMAT +#undef JSON_HEDLEY_PRIVATE +#undef JSON_HEDLEY_PUBLIC +#undef JSON_HEDLEY_PURE +#undef JSON_HEDLEY_REINTERPRET_CAST +#undef JSON_HEDLEY_REQUIRE +#undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#undef JSON_HEDLEY_REQUIRE_MSG +#undef JSON_HEDLEY_RESTRICT +#undef JSON_HEDLEY_RETURNS_NON_NULL +#undef JSON_HEDLEY_SENTINEL +#undef JSON_HEDLEY_STATIC_ASSERT +#undef JSON_HEDLEY_STATIC_CAST +#undef JSON_HEDLEY_STRINGIFY +#undef JSON_HEDLEY_STRINGIFY_EX +#undef JSON_HEDLEY_SUNPRO_VERSION +#undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#undef JSON_HEDLEY_TINYC_VERSION +#undef JSON_HEDLEY_TINYC_VERSION_CHECK +#undef JSON_HEDLEY_TI_ARMCL_VERSION +#undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL2000_VERSION +#undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL430_VERSION +#undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL6X_VERSION +#undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL7X_VERSION +#undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#undef JSON_HEDLEY_TI_CLPRU_VERSION +#undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#undef JSON_HEDLEY_TI_VERSION +#undef JSON_HEDLEY_TI_VERSION_CHECK +#undef JSON_HEDLEY_UNAVAILABLE +#undef JSON_HEDLEY_UNLIKELY +#undef JSON_HEDLEY_UNPREDICTABLE +#undef JSON_HEDLEY_UNREACHABLE +#undef JSON_HEDLEY_UNREACHABLE_RETURN +#undef JSON_HEDLEY_VERSION +#undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#undef JSON_HEDLEY_VERSION_DECODE_MINOR +#undef JSON_HEDLEY_VERSION_DECODE_REVISION +#undef JSON_HEDLEY_VERSION_ENCODE +#undef JSON_HEDLEY_WARNING +#undef JSON_HEDLEY_WARN_UNUSED_RESULT +#undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#undef JSON_HEDLEY_FALL_THROUGH + + + +#endif // INCLUDE_NLOHMANN_JSON_HPP_ diff --git a/utils/loggingcpp/format.h b/utils/loggingcpp/format.h new file mode 100644 index 000000000..f12d019ce --- /dev/null +++ b/utils/loggingcpp/format.h @@ -0,0 +1,92 @@ +/* + Copyright (c) 2021 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace logging +{ +template +void formatOne(std::string& errMsg, Iter iter, uint32_t position) +{ + T arg = boost::any_cast(*iter); + std::string token = std::string("%") + std::to_string(position) + std::string("%"); + size_t index = 0; + + while (true) + { + index = errMsg.find(token, index); + if (index == std::string::npos) + break; + + if constexpr (std::is_same_v) + { + errMsg.replace(index, token.length(), arg); + } + else + { + errMsg.replace(index, token.length(), std::to_string(arg)); + } + + index += token.length(); + } +} + +template +void formatMany(std::string& errMsg, const T& args) +{ + auto iter = args.begin(); + auto end = args.end(); + uint32_t position = 1; + + while (iter != end) + { + if (iter->type() == typeid(long)) + { + formatOne(errMsg, iter, position); + } + else if (iter->type() == typeid(uint64_t)) + { + formatOne(errMsg, iter, position); + } + else if (iter->type() == typeid(double)) + { + formatOne(errMsg, iter, position); + } + else if (iter->type() == typeid(std::string)) + { + formatOne(errMsg, iter, position); + } + else + { + throw std::logic_error("logggin::format: unexpected type in argslist"); + } + ++iter; + ++position; + } + static std::regex restToken("%[0-9]%"); + errMsg = std::regex_replace(errMsg, restToken, ""); +} + +} // namespace logging \ No newline at end of file diff --git a/utils/loggingcpp/idberrorinfo.cpp b/utils/loggingcpp/idberrorinfo.cpp index 2b0054881..dbcd5c920 100644 --- a/utils/loggingcpp/idberrorinfo.cpp +++ b/utils/loggingcpp/idberrorinfo.cpp @@ -29,7 +29,6 @@ #include using namespace std; -#include #include #include using namespace boost; @@ -43,6 +42,8 @@ using namespace config; #include "installdir.h" +#include "format.h" + namespace logging { IDBErrorInfo* IDBErrorInfo::fInstance = 0; @@ -156,43 +157,7 @@ string IDBErrorInfo::logError(const logging::LOG_TYPE logLevel, const logging::L void IDBErrorInfo::format(string& errMsg, const Message::Args& args) { - Message::Args::AnyVec::const_iterator iter = args.args().begin(); - Message::Args::AnyVec::const_iterator end = args.args().end(); - - boost::format fmt(errMsg); - fmt.exceptions(boost::io::no_error_bits); - - while (iter != end) - { - if (iter->type() == typeid(long)) - { - long l = any_cast(*iter); - fmt % l; - } - else if (iter->type() == typeid(uint64_t)) - { - uint64_t u64 = any_cast(*iter); - fmt % u64; - } - else if (iter->type() == typeid(double)) - { - double d = any_cast(*iter); - fmt % d; - } - else if (iter->type() == typeid(string)) - { - string s = any_cast(*iter); - fmt % s; - } - else - { - throw logic_error("IDBErrorInfo::format: unexpected type in argslist"); - } - - ++iter; - } - - errMsg = fmt.str(); + formatMany(errMsg, args.args()); } /* static */ diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index 316923f6f..f8edf9703 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -29,7 +29,7 @@ #include using namespace std; -#include + #include #include using namespace boost; @@ -41,6 +41,7 @@ using namespace config; #include "installdir.h" +#include "format.h" namespace { boost::mutex mx; @@ -143,43 +144,7 @@ void Message::Args::reset() void Message::format(const Args& args) { - Args::AnyVec::const_iterator iter = args.args().begin(); - Args::AnyVec::const_iterator end = args.args().end(); - - boost::format fmt(fMsg); - fmt.exceptions(boost::io::no_error_bits); - - while (iter != end) - { - if (iter->type() == typeid(long)) - { - long l = any_cast(*iter); - fmt % l; - } - else if (iter->type() == typeid(uint64_t)) - { - uint64_t u64 = any_cast(*iter); - fmt % u64; - } - else if (iter->type() == typeid(double)) - { - double d = any_cast(*iter); - fmt % d; - } - else if (iter->type() == typeid(string)) - { - string s = any_cast(*iter); - fmt % s; - } - else - { - throw logic_error("Message::format: unexpected type in argslist"); - } - - ++iter; - } - - fMsg = fmt.str(); + formatMany(fMsg, args.args()); } /* static */ diff --git a/utils/rwlock/rwlock.h b/utils/rwlock/rwlock.h index 08711bddc..cd961ad81 100644 --- a/utils/rwlock/rwlock.h +++ b/utils/rwlock/rwlock.h @@ -45,17 +45,10 @@ namespace rwlock /// the layout of the shmseg struct State { -#ifdef _MSC_VER - volatile LONG writerswaiting; - volatile LONG writing; - volatile LONG readerswaiting; - volatile LONG reading; -#else - volatile int writerswaiting; - volatile int writing; - volatile int readerswaiting; - volatile int reading; -#endif + int writerswaiting; + int writing; + int readerswaiting; + int reading; boost::interprocess::interprocess_semaphore sems[3]; }; diff --git a/utils/winport/grepit.cpp b/utils/winport/grepit.cpp index f20182efa..d6948fd0e 100644 --- a/utils/winport/grepit.cpp +++ b/utils/winport/grepit.cpp @@ -19,7 +19,7 @@ #include using namespace std; -#include +#include #include "grepit.h" @@ -32,13 +32,13 @@ namespace winport { bool grepit(istream& is, const string& pattern) { - boost::regex pat(pattern); + std::regex pat(pattern); string cInput; getline(is, cInput); while (is.good()) { - if (boost::regex_match(cInput, pat)) + if (std::regex_match(cInput, pat)) return true; getline(is, cInput); diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index bba15d97d..6fbca9fc7 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -78,7 +78,7 @@ class TableInfo : public WeUIDGID int fCurrentReadBuffer; // Id of current buffer being popu- // lated by the read thread RID fTotalReadRows; // Total number of rows read - volatile unsigned fTotalErrRows; // Total error rows among all input + unsigned fTotalErrRows; // Total error rows among all input // for this table. Is volatile to // insure parser & reader threads // see the latest value. diff --git a/writeengine/server/we_dataloader.cpp b/writeengine/server/we_dataloader.cpp index 18a616799..1e9984c8d 100644 --- a/writeengine/server/we_dataloader.cpp +++ b/writeengine/server/we_dataloader.cpp @@ -891,7 +891,7 @@ void WEDataLoader::onReceiveData(ByteStream& Ibs) if (aQsz < MAX_QSIZE) sendDataRequest(); - if (aQsz > 1.5 * MAX_QSIZE) // > 2*250 + if (aQsz > 1.5 * static_cast(MAX_QSIZE)) // > 2*250 { cout << "WARNING : Data Queuing up : QSize = " << aQsz << endl; From 749b8f16ee7def18c50d0b08069bac90e2c5dee3 Mon Sep 17 00:00:00 2001 From: Commander thrashdin Date: Fri, 18 Mar 2022 18:13:06 +0200 Subject: [PATCH 35/55] Added mcs-named UDFs to cpp --- dbcon/mysql/ha_mcs_client_udfs.cpp | 453 ++++++++++++++++++++++++++--- 1 file changed, 415 insertions(+), 38 deletions(-) diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 2ea1a68a2..2981c41cd 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -69,7 +69,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calsetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcssetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { char parameter[MAXSTRINGLENGTH]; @@ -135,6 +135,68 @@ extern "C" return result; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 2 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) + { + strcpy(message, "MCSSETPARMS() requires two string arguments"); + return 1; + } + + initid->max_length = MAXSTRINGLENGTH; + + char valuestr[MAXSTRINGLENGTH]; + size_t vlen = args->lengths[1]; + + memcpy(valuestr, args->args[1], vlen--); + + for (size_t i = 0; i < vlen; ++i) + if (!isdigit(valuestr[i])) + { + strcpy(message, "MCSSETPARMS() second argument must be numeric or end in G, M or K"); + return 1; + } + + if (!isdigit(valuestr[vlen])) + { + switch (valuestr[vlen]) + { + case 'G': + case 'g': + case 'M': + case 'm': + case 'K': + case 'k': + case '\0': break; + + default: + strcpy(message, "MCSSETPARMS() second argument must be numeric or end in G, M or K"); + return 1; + } + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssetparms_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calsetparms(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcssetparms(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -191,7 +253,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcsgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) @@ -215,6 +277,39 @@ extern "C" return result; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, "MCSGETSTATS() takes no arguments"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsgetstats_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calgetstats(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsgetstats(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -242,7 +337,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - long long calsettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + long long mcssettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -257,6 +352,35 @@ extern "C" return oldTrace; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 1 || args->arg_type[0] != INT_RESULT) + { + strcpy(message, "MCSSETTRACE() requires one INTEGER argument"); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssettrace_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long calsettrace(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return mcssettrace(initid, args, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -417,26 +541,26 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool mcsviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) { - strcpy(message, "CALVIEWTABLELOCK() requires two string arguments"); + strcpy(message, "MCSVIEWTABLELOCK() requires two string arguments"); return 1; } else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) { - strcpy(message, "CALVIEWTABLELOCK() requires one string argument"); + strcpy(message, "MCSVIEWTABLELOCK() requires one string argument"); return 1; } else if (args->arg_count > 2) { - strcpy(message, "CALVIEWTABLELOCK() takes one or two arguments only"); + strcpy(message, "MCSVIEWTABLELOCK() takes one or two arguments only"); return 1; } else if (args->arg_count == 0) { - strcpy(message, "CALVIEWTABLELOCK() requires at least one argument"); + strcpy(message, "MCSVIEWTABLELOCK() requires at least one argument"); return 1; } @@ -449,7 +573,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcsviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { THD* thd = current_thd; @@ -504,18 +628,33 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - void calviewtablelock_deinit(UDF_INIT* initid) + void mcsviewtablelock_deinit(UDF_INIT* initid) { } #ifdef _MSC_VER __declspec(dllexport) #endif - my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT)) + if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) { - strcpy(message, "CALCLEARTABLELOCK() requires one integer argument (the lockID)"); + strcpy(message, "CALVIEWTABLELOCK() requires two string arguments"); + return 1; + } + else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) + { + strcpy(message, "CALVIEWTABLELOCK() requires one string argument"); + return 1; + } + else if (args->arg_count > 2) + { + strcpy(message, "CALVIEWTABLELOCK() takes one or two arguments only"); + return 1; + } + else if (args->arg_count == 0) + { + strcpy(message, "CALVIEWTABLELOCK() requires at least one argument"); return 1; } @@ -528,7 +667,40 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calcleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsviewtablelock(initid, args, result, length, is_null, error); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calviewtablelock_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcscleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT)) + { + strcpy(message, "MCSCLEARTABLELOCK() requires one integer argument (the lockID)"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* mcscleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) @@ -555,33 +727,18 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - void calcleartablelock_deinit(UDF_INIT* initid) + void mcscleartablelock_deinit(UDF_INIT* initid) { } #ifdef _MSC_VER __declspec(dllexport) #endif - my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) + if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT)) { - strcpy(message, "CALLASTINSRTID() requires two string arguments"); - return 1; - } - else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) - { - strcpy(message, "CALLASTINSERTID() requires one string argument"); - return 1; - } - else if (args->arg_count > 2) - { - strcpy(message, "CALLASTINSERTID() takes one or two arguments only"); - return 1; - } - else if (args->arg_count == 0) - { - strcpy(message, "CALLASTINSERTID() requires at least one argument"); + strcpy(message, "CALCLEARTABLELOCK() requires one integer argument (the lockID)"); return 1; } @@ -594,7 +751,55 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - long long callastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + const char* calcleartablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcscleartablelock(initid, args, result, length, is_null, error); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void calcleartablelock_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcslastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) + { + strcpy(message, "MCSLASTINSRTID() requires two string arguments"); + return 1; + } + else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) + { + strcpy(message, "MCSLASTINSERTID() requires one string argument"); + return 1; + } + else if (args->arg_count > 2) + { + strcpy(message, "MCSLASTINSERTID() takes one or two arguments only"); + return 1; + } + else if (args->arg_count == 0) + { + strcpy(message, "MCSLASTINSERTID() requires at least one argument"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long mcslastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { THD* thd = current_thd; @@ -656,6 +861,53 @@ extern "C" return (nextVal - 1); } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcslastinsertid_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) + { + strcpy(message, "CALLASTINSRTID() requires two string arguments"); + return 1; + } + else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) + { + strcpy(message, "CALLASTINSERTID() requires one string argument"); + return 1; + } + else if (args->arg_count > 2) + { + strcpy(message, "CALLASTINSERTID() takes one or two arguments only"); + return 1; + } + else if (args->arg_count == 0) + { + strcpy(message, "CALLASTINSERTID() requires at least one argument"); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long callastinsertid(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return mcslastinsertid(initid, args, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -663,6 +915,35 @@ extern "C" { } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsflushcache_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long mcsflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + return static_cast(cacheutils::flushPrimProcCache()); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, "MCSFLUSHCACHE() takes no arguments"); + return 1; + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -675,7 +956,7 @@ extern "C" #endif long long calflushcache(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { - return static_cast(cacheutils::flushPrimProcCache()); + return mcsflushcache(initid, args, is_null, error); } #ifdef _MSC_VER @@ -700,7 +981,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcsgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { const std::string* msgp; @@ -741,6 +1022,42 @@ extern "C" return msgp->c_str(); } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { +#if 0 + + if (args->arg_count != 0) + { + strcpy(message, "MCSGETTRACE() takes no arguments"); + return 1; + } + +#endif + initid->maybe_null = 1; + initid->max_length = TraceSize; + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsgettrace_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calgettrace(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsgettrace(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -771,7 +1088,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcsgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { std::string version(columnstore_version); @@ -780,6 +1097,36 @@ extern "C" return result; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, "MCSGETVERSION() takes no arguments"); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsgetversion_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsgetversion(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -804,7 +1151,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif - const char* calgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + const char* mcsgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { if (get_fe_conn_info_ptr() == NULL) @@ -843,6 +1190,36 @@ extern "C" return result; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, "MCSGETSQLCOUNT() takes no arguments"); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcsgetsqlcount_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + const char* calgetsqlcount(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) + { + return mcsgetsqlcount(initid, args, result, length, is_null, error); + } + #ifdef _MSC_VER __declspec(dllexport) #endif From 8d31478b7234b6127bd22ad917e6b6e51b44b188 Mon Sep 17 00:00:00 2001 From: Commander thrashdin Date: Fri, 18 Mar 2022 18:15:19 +0200 Subject: [PATCH 36/55] Added mcsUDFs to install.sh+removed nonexistent fn --- dbcon/mysql/install_mcs_mysql.sh.in | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/install_mcs_mysql.sh.in b/dbcon/mysql/install_mcs_mysql.sh.in index 1bd6e371a..9b2dddf4e 100755 --- a/dbcon/mysql/install_mcs_mysql.sh.in +++ b/dbcon/mysql/install_mcs_mysql.sh.in @@ -19,16 +19,25 @@ fi # DELETE libcalmysql.so entries first as they are in ha_columnstore.so in 1.4.2 onwards $MDB 2> ${tmpdir}/mysql_install.log < Date: Wed, 23 Mar 2022 17:04:51 +0200 Subject: [PATCH 37/55] Added crude tests for cal/mcs client UDFs --- .../columnstore/basic/r/cal_named_udfs.result | 33 +++++++++++++++++++ .../columnstore/basic/r/mcs_named_udfs.result | 33 +++++++++++++++++++ .../columnstore/basic/t/cal_named_udfs.test | 25 ++++++++++++++ .../columnstore/basic/t/mcs_named_udfs.test | 25 ++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 mysql-test/columnstore/basic/r/cal_named_udfs.result create mode 100644 mysql-test/columnstore/basic/r/mcs_named_udfs.result create mode 100644 mysql-test/columnstore/basic/t/cal_named_udfs.test create mode 100644 mysql-test/columnstore/basic/t/mcs_named_udfs.test diff --git a/mysql-test/columnstore/basic/r/cal_named_udfs.result b/mysql-test/columnstore/basic/r/cal_named_udfs.result new file mode 100644 index 000000000..8a96f7c62 --- /dev/null +++ b/mysql-test/columnstore/basic/r/cal_named_udfs.result @@ -0,0 +1,33 @@ +DROP DATABASE IF EXISTS cal_test_db; +CREATE DATABASE cal_test_db; +USE cal_test_db; +create table t1(a int, b int comment 'autoincrement=1') engine=columnstore; +select calflushcache(); +calflushcache() +0 +select calsettrace(0); +calsettrace(0) +0 +select calsetparms("pmmaxmemorysmallside","2048000000"); +calsetparms("pmmaxmemorysmallside","2048000000") +Updated pmmaxmemorysmallside 2048000000 +select calgettrace(); +calgettrace() +NULL +select calgetversion(); +calgetversion() +8.1.1 +select calviewtablelock("t1"); +calviewtablelock("t1") + Table cal_test_db.t1 is not locked by any process. +select calcleartablelock(0); +calcleartablelock(0) +No table lock found for specified table lock ID +select callastinsertid("t1"); +callastinsertid("t1") +0 +select calgetsqlcount(); +calgetsqlcount() +Running SQL statements 0, Waiting SQL statments 0 +DROP TABLE t1; +DROP DATABASE cal_test_db; diff --git a/mysql-test/columnstore/basic/r/mcs_named_udfs.result b/mysql-test/columnstore/basic/r/mcs_named_udfs.result new file mode 100644 index 000000000..6f79cbe75 --- /dev/null +++ b/mysql-test/columnstore/basic/r/mcs_named_udfs.result @@ -0,0 +1,33 @@ +DROP DATABASE IF EXISTS cal_test_db; +CREATE DATABASE cal_test_db; +USE cal_test_db; +create table t1(a int, b int comment 'autoincrement=1') engine=columnstore; +select mcsflushcache(); +mcsflushcache() +0 +select mcssettrace(0); +mcssettrace(0) +0 +select mcssetparms("pmmaxmemorysmallside","2048000000"); +mcssetparms("pmmaxmemorysmallside","2048000000") +Updated pmmaxmemorysmallside 2048000000 +select mcsgettrace(); +mcsgettrace() +NULL +select mcsgetversion(); +mcsgetversion() +8.1.1 +select mcsviewtablelock("t1"); +mcsviewtablelock("t1") + Table cal_test_db.t1 is not locked by any process. +select mcscleartablelock(0); +mcscleartablelock(0) +No table lock found for specified table lock ID +select mcslastinsertid("t1"); +mcslastinsertid("t1") +0 +select mcsgetsqlcount(); +mcsgetsqlcount() +Running SQL statements 0, Waiting SQL statments 0 +DROP TABLE t1; +DROP DATABASE cal_test_db; diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test new file mode 100644 index 000000000..c60cd946a --- /dev/null +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -0,0 +1,25 @@ +-- source ../include/have_columnstore.inc + +--disable_warnings +DROP DATABASE IF EXISTS cal_test_db; +--enable_warnings + +CREATE DATABASE cal_test_db; +USE cal_test_db; + +create table t1(a int, b int comment 'autoincrement=1') engine=columnstore; + +select calflushcache(); +select calsettrace(0); +select calsetparms("pmmaxmemorysmallside","2048000000"); +select calgettrace(); +select calgetversion(); + + +select calviewtablelock("t1"); +select calcleartablelock(0); +select callastinsertid("t1"); +select calgetsqlcount(); + +DROP TABLE t1; +DROP DATABASE cal_test_db; diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test new file mode 100644 index 000000000..eda419676 --- /dev/null +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -0,0 +1,25 @@ +-- source ../include/have_columnstore.inc + +--disable_warnings +DROP DATABASE IF EXISTS cal_test_db; +--enable_warnings + +CREATE DATABASE cal_test_db; +USE cal_test_db; + +create table t1(a int, b int comment 'autoincrement=1') engine=columnstore; + +select mcsflushcache(); +select mcssettrace(0); +select mcssetparms("pmmaxmemorysmallside","2048000000"); +select mcsgettrace(); +select mcsgetversion(); + + +select mcsviewtablelock("t1"); +select mcscleartablelock(0); +select mcslastinsertid("t1"); +select mcsgetsqlcount(); + +DROP TABLE t1; +DROP DATABASE cal_test_db; From f28e00c206bf5c869d1df03db71c67c7b773fbeb Mon Sep 17 00:00:00 2001 From: Commander thrashdin Date: Mon, 28 Mar 2022 21:21:29 +0300 Subject: [PATCH 38/55] No repeating code in client_udfs + better test --- dbcon/mysql/ha_mcs_client_udfs.cpp | 423 +++++++----------- .../columnstore/basic/r/cal_named_udfs.result | 6 +- .../columnstore/basic/r/mcs_named_udfs.result | 6 +- .../columnstore/basic/t/cal_named_udfs.test | 2 +- .../columnstore/basic/t/mcs_named_udfs.test | 3 +- 5 files changed, 177 insertions(+), 263 deletions(-) diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 2981c41cd..7581b67df 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -66,6 +66,49 @@ extern "C" return str; } + my_bool setparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 2 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) + { + sprintf(message, "%s() requires two string arguments", funcname); + return 1; + } + + initid->max_length = MAXSTRINGLENGTH; + + char valuestr[MAXSTRINGLENGTH]; + size_t vlen = args->lengths[1]; + + memcpy(valuestr, args->args[1], vlen--); + + for (size_t i = 0; i < vlen; ++i) + if (!isdigit(valuestr[i])) + { + sprintf(message, "%s() second argument must be numeric or end in G, M or K", funcname); + return 1; + } + + if (!isdigit(valuestr[vlen])) + { + switch (valuestr[vlen]) + { + case 'G': + case 'g': + case 'M': + case 'm': + case 'K': + case 'k': + case '\0': break; + + default: + sprintf(message, "%s() second argument must be numeric or end in G, M or K", funcname); + return 1; + } + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -135,50 +178,13 @@ extern "C" return result; } + #ifdef _MSC_VER __declspec(dllexport) #endif my_bool mcssetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 2 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) - { - strcpy(message, "MCSSETPARMS() requires two string arguments"); - return 1; - } - - initid->max_length = MAXSTRINGLENGTH; - - char valuestr[MAXSTRINGLENGTH]; - size_t vlen = args->lengths[1]; - - memcpy(valuestr, args->args[1], vlen--); - - for (size_t i = 0; i < vlen; ++i) - if (!isdigit(valuestr[i])) - { - strcpy(message, "MCSSETPARMS() second argument must be numeric or end in G, M or K"); - return 1; - } - - if (!isdigit(valuestr[vlen])) - { - switch (valuestr[vlen]) - { - case 'G': - case 'g': - case 'M': - case 'm': - case 'K': - case 'k': - case '\0': break; - - default: - strcpy(message, "MCSSETPARMS() second argument must be numeric or end in G, M or K"); - return 1; - } - } - - return 0; + return setparms_init(initid, args, message, "MCSSETPARMS"); } #ifdef _MSC_VER @@ -188,6 +194,7 @@ extern "C" { } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -202,45 +209,7 @@ extern "C" #endif my_bool calsetparms_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 2 || args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) - { - strcpy(message, "CALSETPARMS() requires two string arguments"); - return 1; - } - - initid->max_length = MAXSTRINGLENGTH; - - char valuestr[MAXSTRINGLENGTH]; - size_t vlen = args->lengths[1]; - - memcpy(valuestr, args->args[1], vlen--); - - for (size_t i = 0; i < vlen; ++i) - if (!isdigit(valuestr[i])) - { - strcpy(message, "CALSETPARMS() second argument must be numeric or end in G, M or K"); - return 1; - } - - if (!isdigit(valuestr[vlen])) - { - switch (valuestr[vlen]) - { - case 'G': - case 'g': - case 'M': - case 'm': - case 'K': - case 'k': - case '\0': break; - - default: - strcpy(message, "CALSETPARMS() second argument must be numeric or end in G, M or K"); - return 1; - } - } - - return 0; + return setparms_init(initid, args, message, "CALSETPARMS"); } #ifdef _MSC_VER @@ -250,6 +219,20 @@ extern "C" { } + my_bool getstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 0) + { + sprintf(message, "%s() takes no arguments", funcname); + return 1; + } + + initid->maybe_null = 1; + initid->max_length = 255; + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -277,21 +260,13 @@ extern "C" return result; } + #ifdef _MSC_VER __declspec(dllexport) #endif my_bool mcsgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "MCSGETSTATS() takes no arguments"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; + return getstats_init(initid, args, message, "MCSGETSTATS"); } #ifdef _MSC_VER @@ -315,16 +290,7 @@ extern "C" #endif my_bool calgetstats_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "CALGETSTATS() takes no arguments"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; + return getstats_init(initid, args, message, "CALGETSTATS"); } #ifdef _MSC_VER @@ -334,6 +300,17 @@ extern "C" { } + my_bool settrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 1 || args->arg_type[0] != INT_RESULT) + { + sprintf(message, "%s() requires one INTEGER argument", funcname); + return 1; + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -357,13 +334,7 @@ extern "C" #endif my_bool mcssettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 1 || args->arg_type[0] != INT_RESULT) - { - strcpy(message, "MCSSETTRACE() requires one INTEGER argument"); - return 1; - } - - return 0; + return settrace_init(initid, args, message, "MCSSETTRACE"); } #ifdef _MSC_VER @@ -386,13 +357,7 @@ extern "C" #endif my_bool calsettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 1 || args->arg_type[0] != INT_RESULT) - { - strcpy(message, "CALSETTRACE() requires one INTEGER argument"); - return 1; - } - - return 0; + return settrace_init(initid, args, message, "CALSETTRACE"); } #ifdef _MSC_VER @@ -538,29 +503,26 @@ extern "C" { } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool mcsviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool viewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) { - strcpy(message, "MCSVIEWTABLELOCK() requires two string arguments"); + sprintf(message, "%s() requires two string arguments", funcname); return 1; } else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) { - strcpy(message, "MCSVIEWTABLELOCK() requires one string argument"); + sprintf(message, "%s() requires one string argument", funcname); return 1; } else if (args->arg_count > 2) { - strcpy(message, "MCSVIEWTABLELOCK() takes one or two arguments only"); + sprintf(message, "%s() takes one or two arguments only", funcname); return 1; } else if (args->arg_count == 0) { - strcpy(message, "MCSVIEWTABLELOCK() requires at least one argument"); + sprintf(message, "%s() requires at least one argument", funcname); return 1; } @@ -570,6 +532,14 @@ extern "C" return 0; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return viewtablelock_init(initid, args, message, "MCSVIEWTABLELOCK"); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -637,31 +607,7 @@ extern "C" #endif my_bool calviewtablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) - { - strcpy(message, "CALVIEWTABLELOCK() requires two string arguments"); - return 1; - } - else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) - { - strcpy(message, "CALVIEWTABLELOCK() requires one string argument"); - return 1; - } - else if (args->arg_count > 2) - { - strcpy(message, "CALVIEWTABLELOCK() takes one or two arguments only"); - return 1; - } - else if (args->arg_count == 0) - { - strcpy(message, "CALVIEWTABLELOCK() requires at least one argument"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; + return viewtablelock_init(initid, args, message, "CALVIEWTABLELOCK"); } #ifdef _MSC_VER @@ -670,7 +616,7 @@ extern "C" const char* calviewtablelock(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { - return mcsviewtablelock(initid, args, result, length, is_null, error); + return mcsviewtablelock(initid, args, result, length, is_null, error); } #ifdef _MSC_VER @@ -680,14 +626,11 @@ extern "C" { } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool mcscleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool cleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT)) { - strcpy(message, "MCSCLEARTABLELOCK() requires one integer argument (the lockID)"); + sprintf(message, "%s() requires one integer argument (the lockID)", funcname); return 1; } @@ -697,6 +640,15 @@ extern "C" return 0; } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcscleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return cleartablelock_init(initid, args, message, "MCSCLEARTABLELOCK"); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -736,16 +688,7 @@ extern "C" #endif my_bool calcleartablelock_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if ((args->arg_count != 1) || (args->arg_type[0] != INT_RESULT)) - { - strcpy(message, "CALCLEARTABLELOCK() requires one integer argument (the lockID)"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; + return cleartablelock_init(initid, args, message, "CALCLEARTABLELOCK"); } #ifdef _MSC_VER @@ -764,29 +707,26 @@ extern "C" { } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool mcslastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool lastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) { - strcpy(message, "MCSLASTINSRTID() requires two string arguments"); + sprintf(message, "%s() requires two string arguments", funcname); return 1; } else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) { - strcpy(message, "MCSLASTINSERTID() requires one string argument"); + sprintf(message, "%s() requires one string argument", funcname); return 1; } else if (args->arg_count > 2) { - strcpy(message, "MCSLASTINSERTID() takes one or two arguments only"); + sprintf(message, "%s() takes one or two arguments only", funcname); return 1; } else if (args->arg_count == 0) { - strcpy(message, "MCSLASTINSERTID() requires at least one argument"); + sprintf(message, "%s() requires at least one argument", funcname); return 1; } @@ -796,6 +736,15 @@ extern "C" return 0; } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcslastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return lastinsertid_init(initid, args, message, "MCSLASTINSERTID"); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -873,31 +822,7 @@ extern "C" #endif my_bool callastinsertid_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count == 2 && (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT)) - { - strcpy(message, "CALLASTINSRTID() requires two string arguments"); - return 1; - } - else if ((args->arg_count == 1) && (args->arg_type[0] != STRING_RESULT)) - { - strcpy(message, "CALLASTINSERTID() requires one string argument"); - return 1; - } - else if (args->arg_count > 2) - { - strcpy(message, "CALLASTINSERTID() takes one or two arguments only"); - return 1; - } - else if (args->arg_count == 0) - { - strcpy(message, "CALLASTINSERTID() requires at least one argument"); - return 1; - } - - initid->maybe_null = 1; - initid->max_length = 255; - - return 0; + return lastinsertid_init(initid, args, message, "CALLASTINSERTID"); } #ifdef _MSC_VER @@ -915,6 +840,17 @@ extern "C" { } + my_bool flushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 0) + { + sprintf(message, "%s() takes no arguments", funcname); + return 1; + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -935,13 +871,7 @@ extern "C" #endif my_bool mcsflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "MCSFLUSHCACHE() takes no arguments"); - return 1; - } - - return 0; + return flushcache_init(initid, args, message, "MCSFLUSHCACHE"); } #ifdef _MSC_VER @@ -964,13 +894,7 @@ extern "C" #endif my_bool calflushcache_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "CALFLUSHCACHE() takes no arguments"); - return 1; - } - - return 0; + return flushcache_init(initid, args, message, "CALFLUSHCACHE"); } static const unsigned long TraceSize = 16 * 1024; @@ -1022,16 +946,13 @@ extern "C" return msgp->c_str(); } -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool mcsgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + my_bool gettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) { #if 0 if (args->arg_count != 0) { - strcpy(message, "MCSGETTRACE() takes no arguments"); + sprintf(message, "%s() takes no arguments", funcname); return 1; } @@ -1042,6 +963,14 @@ extern "C" return 0; } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcsgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return gettrace_init(initid, args, message, "MCSGETTRACE"); + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -1063,19 +992,7 @@ extern "C" #endif my_bool calgettrace_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { -#if 0 - - if (args->arg_count != 0) - { - strcpy(message, "CALGETTRACE() takes no arguments"); - return 1; - } - -#endif - initid->maybe_null = 1; - initid->max_length = TraceSize; - - return 0; + return gettrace_init(initid, args, message, "CALGETTRACE"); } #ifdef _MSC_VER @@ -1085,6 +1002,17 @@ extern "C" { } + my_bool getversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 0) + { + sprintf(message, "%s() takes no arguments", funcname); + return 1; + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -1102,13 +1030,7 @@ extern "C" #endif my_bool mcsgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "MCSGETVERSION() takes no arguments"); - return 1; - } - - return 0; + return getversion_init(initid, args, message, "MCSGETVERSION"); } #ifdef _MSC_VER @@ -1132,13 +1054,7 @@ extern "C" #endif my_bool calgetversion_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "CALGETVERSION() takes no arguments"); - return 1; - } - - return 0; + return getversion_init(initid, args, message, "CALGETVERSION"); } #ifdef _MSC_VER @@ -1148,6 +1064,17 @@ extern "C" { } + my_bool getsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message, const char* funcname) + { + if (args->arg_count != 0) + { + sprintf(message, "%s() takes no arguments", funcname); + return 1; + } + + return 0; + } + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -1195,13 +1122,7 @@ extern "C" #endif my_bool mcsgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "MCSGETSQLCOUNT() takes no arguments"); - return 1; - } - - return 0; + return getstats_init(initid, args, message, "MCSGETSQLCOUNT"); } #ifdef _MSC_VER @@ -1225,13 +1146,7 @@ extern "C" #endif my_bool calgetsqlcount_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - if (args->arg_count != 0) - { - strcpy(message, "CALGETSQLCOUNT() takes no arguments"); - return 1; - } - - return 0; + return getstats_init(initid, args, message, "CALGETSQLCOUNT"); } #ifdef _MSC_VER diff --git a/mysql-test/columnstore/basic/r/cal_named_udfs.result b/mysql-test/columnstore/basic/r/cal_named_udfs.result index 8a96f7c62..26b20fb8b 100644 --- a/mysql-test/columnstore/basic/r/cal_named_udfs.result +++ b/mysql-test/columnstore/basic/r/cal_named_udfs.result @@ -14,9 +14,9 @@ Updated pmmaxmemorysmallside 2048000000 select calgettrace(); calgettrace() NULL -select calgetversion(); -calgetversion() -8.1.1 +select calgetversion()=mcsgetversion(); +calgetversion()=mcsgetversion() +1 select calviewtablelock("t1"); calviewtablelock("t1") Table cal_test_db.t1 is not locked by any process. diff --git a/mysql-test/columnstore/basic/r/mcs_named_udfs.result b/mysql-test/columnstore/basic/r/mcs_named_udfs.result index 6f79cbe75..4a09eb41d 100644 --- a/mysql-test/columnstore/basic/r/mcs_named_udfs.result +++ b/mysql-test/columnstore/basic/r/mcs_named_udfs.result @@ -14,9 +14,9 @@ Updated pmmaxmemorysmallside 2048000000 select mcsgettrace(); mcsgettrace() NULL -select mcsgetversion(); -mcsgetversion() -8.1.1 +select mcsgetversion()=calgetversion(); +mcsgetversion()=calgetversion() +1 select mcsviewtablelock("t1"); mcsviewtablelock("t1") Table cal_test_db.t1 is not locked by any process. diff --git a/mysql-test/columnstore/basic/t/cal_named_udfs.test b/mysql-test/columnstore/basic/t/cal_named_udfs.test index c60cd946a..f127716d2 100644 --- a/mysql-test/columnstore/basic/t/cal_named_udfs.test +++ b/mysql-test/columnstore/basic/t/cal_named_udfs.test @@ -13,7 +13,7 @@ select calflushcache(); select calsettrace(0); select calsetparms("pmmaxmemorysmallside","2048000000"); select calgettrace(); -select calgetversion(); +select calgetversion()=mcsgetversion(); select calviewtablelock("t1"); diff --git a/mysql-test/columnstore/basic/t/mcs_named_udfs.test b/mysql-test/columnstore/basic/t/mcs_named_udfs.test index eda419676..980c3fc69 100644 --- a/mysql-test/columnstore/basic/t/mcs_named_udfs.test +++ b/mysql-test/columnstore/basic/t/mcs_named_udfs.test @@ -13,8 +13,7 @@ select mcsflushcache(); select mcssettrace(0); select mcssetparms("pmmaxmemorysmallside","2048000000"); select mcsgettrace(); -select mcsgetversion(); - +select mcsgetversion()=calgetversion(); select mcsviewtablelock("t1"); select mcscleartablelock(0); From 7cdc914b4e850f2fd15d4b067f657069e16eb790 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 17 Feb 2022 22:29:14 +0000 Subject: [PATCH 39/55] MCOL-4809 This patch introduces vectorized scanning/filtering for short CHAR/VARCHAR columns Short CHAR/VARCHAR column values contain integer-encoded strings. After certain manipulations(orderSwap(strnxfrm(str))) the values become integers that preserve original strings order relation according to a certain translation rules(collation). Prepared values are ready to be SIMD-processed. --- primitives/linux-port/column.cpp | 140 ++++++++-- tests/simd_processors.cpp | 1 + utils/common/collation.h | 28 +- utils/common/simd_sse.h | 459 ++++++++++++++++++++++++++++++- 4 files changed, 602 insertions(+), 26 deletions(-) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 974a277da..01be9e39d 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016-2021 MariaDB Corporation + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -22,6 +22,7 @@ #include #include #include +#include #ifndef _MSC_VER #include #else @@ -62,6 +63,55 @@ inline uint64_t order_swap(uint64_t x) return ret; } +// Dummy template +template= sizeof(uint128_t), T>::type* = nullptr> +inline T orderSwap(T x) +{ + return x; +} + +template::type* = nullptr> +inline T orderSwap(T x) +{ + T ret = (x >> 56) | + ((x << 40) & 0x00FF000000000000ULL) | + ((x << 24) & 0x0000FF0000000000ULL) | + ((x << 8) & 0x000000FF00000000ULL) | + ((x >> 8) & 0x00000000FF000000ULL) | + ((x >> 24) & 0x0000000000FF0000ULL) | + ((x >> 40) & 0x000000000000FF00ULL) | + (x << 56); + return ret; +} + +template::type* = nullptr> +inline T orderSwap(T x) +{ + T ret = (x >> 24) | + ((x << 8) & 0x00FF0000U) | + ((x >> 8) & 0x0000FF00U) | + (x << 24); + return ret; +} + +template::type* = nullptr> +inline T orderSwap(T x) +{ + T ret = (x >> 8) | (x <<8); + return ret; +} + +template::type* = nullptr> +inline T orderSwap(T x) +{ + return x; +} + template inline int compareBlock(const void* a, const void* b) { @@ -107,8 +157,11 @@ inline bool colCompare_(const T& val1, const T& val2, uint8_t COP) } } -inline bool colCompareStr(const ColRequestHeaderDataType& type, uint8_t COP, const utils::ConstString& val1, - const utils::ConstString& val2) +inline bool colCompareStr(const ColRequestHeaderDataType &type, + uint8_t COP, + const utils::ConstString &val1, + const utils::ConstString &val2, + const bool printOut = false) { int error = 0; bool rc = primitives::StringComparator(type).op(&error, COP, val1, val2); @@ -1179,7 +1232,7 @@ void scalarFiltering( #if defined(__x86_64__) template ::type* = nullptr> -inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, const T* origSrcArray, +inline SIMD_WRAPPER_TYPE simdDataLoad(VT& processor, const T* srcArray, const T* origSrcArray, const primitives::RIDType* ridArray, const uint16_t iter) { return {processor.loadFrom(reinterpret_cast(srcArray))}; @@ -1189,7 +1242,7 @@ inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, // TODO Move the logic into simd namespace class methods and use intrinsics template ::type* = nullptr> -inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, const T* origSrcArray, +inline SIMD_WRAPPER_TYPE simdDataLoad(VT& processor, const T* srcArray, const T* origSrcArray, const primitives::RIDType* ridArray, const uint16_t iter) { constexpr const uint16_t WIDTH = sizeof(T); @@ -1205,6 +1258,32 @@ inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, return {result}; } +template ::type* = nullptr> +inline SIMD_WRAPPER_TYPE simdSwapedOrderDataLoad(const ColRequestHeaderDataType &type, VT& processor, typename VT::SimdType& dataVector) +{ + return {dataVector}; +} + +template ::type* = nullptr> +inline SIMD_WRAPPER_TYPE simdSwapedOrderDataLoad(const ColRequestHeaderDataType &type, + VT& processor, typename VT::SimdType& dataVector) +{ + constexpr const uint16_t WIDTH = sizeof(T); + constexpr const uint16_t VECTOR_SIZE = VT::vecByteSize / WIDTH; + using SimdType = typename VT::SimdType; + SimdType result; + T* resultTypedPtr = reinterpret_cast(&result); + T* srcTypedPtr = reinterpret_cast(&dataVector); + for (uint32_t i = 0; i < VECTOR_SIZE; ++i) + { + utils::ConstString s{reinterpret_cast(&srcTypedPtr[i]), WIDTH}; + resultTypedPtr[i] = orderSwap(type.strnxfrm(s.rtrimZero())); + } + return {result}; +} + // This routine filters input block in a vectorized manner. // It supports all output types, all input types. // It doesn't support KIND==TEXT so upper layers filters this KIND out beforehand. @@ -1214,8 +1293,8 @@ inline SIMD_WRAPPER_TYPE simdDataLoadTemplate(VT& processor, const T* srcArray, // to glue the masks produced by actual filters. // Then it takes a vector of data, run filters and logical function using pointers. // See the corresponding dispatcher to get more details on vector processing class. -template +template void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* srcArray, const uint32_t srcSize, primitives::RIDType* ridArray, const uint16_t ridSize, ParsedColumnFilter* parsedColumnFilter, const bool validMinMax, const T emptyValue, @@ -1225,8 +1304,12 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* using SimdType = typename VT::SimdType; using SimdWrapperType = typename VT::SimdWrapperType; using FilterType = typename VT::FilterType; + using UT = typename std::conditional::value || datatypes::is_uint128_t::value || std::is_same::value, + FilterType, typename datatypes::make_unsigned::type>::type; VT simdProcessor; SimdType dataVec; + [[maybe_unused]] SimdType swapedOrderDataVec; + [[maybe_unused]] auto typeHolder = in->colType; SimdType emptyFilterArgVec = simdProcessor.emptyNullLoadValue(emptyValue); SimdType nullFilterArgVec = simdProcessor.emptyNullLoadValue(nullValue); MT writeMask, nonEmptyMask, nonNullMask, nonNullOrEmptyMask; @@ -1292,11 +1375,27 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* for (uint32_t j = 0; j < filterCount; ++j) { // Preload filter argument values only once. - filterArgsVectors.push_back(simdProcessor.loadValue(*((FilterType*)&filterValues[j]))); + if constexpr (KIND == KIND_TEXT) + { + // Preload filter argument values only once. + // First cast filter value as the corresponding unsigned int value + UT filterValue = *((UT*)&filterValues[j]); + // Cast to ConstString to preprocess the string + utils::ConstString s{reinterpret_cast(&filterValue), sizeof(UT)}; + // Strip all 0 bytes on the right, convert byte into collation weights array + // and swap bytes order. + UT bigEndianFilterWeights = orderSwap(typeHolder.strnxfrm(s.rtrimZero())); + filterArgsVectors.push_back(simdProcessor.loadValue(bigEndianFilterWeights)); + } + else + { + FilterType filterValue = *((FilterType*)&filterValues[j]); + filterArgsVectors.push_back(simdProcessor.loadValue(filterValue)); + } switch (filterCOPs[j]) { case (COMPARE_EQ): - // Skipping extra filter pass generated by IS NULL + // Filter against NULL value if (memcmp(&filterValues[j], &nullValue, sizeof(nullValue)) == 0) copFunctorVec.push_back(std::mem_fn(&VT::nullEmptyCmpEq)); else @@ -1329,9 +1428,10 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* { primitives::RIDType ridOffset = i * VECTOR_SIZE; assert(!HAS_INPUT_RIDS || (HAS_INPUT_RIDS && ridSize >= ridOffset)); - dataVec = simdDataLoadTemplate(simdProcessor, srcArray, - origSrcArray, ridArray, i) - .v; + dataVec = simdDataLoad(simdProcessor, srcArray, + origSrcArray, ridArray, i).v; + if constexpr(KIND==KIND_TEXT) + swapedOrderDataVec = simdSwapedOrderDataLoad(typeHolder, simdProcessor, dataVec).v; nonEmptyMask = simdProcessor.nullEmptyCmpNe(dataVec, emptyFilterArgVec); writeMask = nonEmptyMask; // NULL check @@ -1346,7 +1446,11 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* for (uint32_t j = 0; j < filterCount; ++j) { // filter using compiled filter and preloaded filter argument - filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]); + if constexpr(KIND==KIND_TEXT) + filterMask = copFunctorVec[j](simdProcessor, swapedOrderDataVec, filterArgsVectors[j]); + else + filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]); + filterMask = bopFunctor(prevFilterMask, filterMask); prevFilterMask = filterMask; } @@ -1389,7 +1493,6 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* out->Min = Min; out->Max = Max; } - // process the tail. scalarFiltering changes out contents, e.g. Min/Max, NVALS, RIDs and values array // This tail also sets out::Min/Max, out::validMinMax if validMinMax is set. uint32_t processedSoFar = rid; @@ -1526,7 +1629,8 @@ void filterColumnData(NewColRequestHeader* in, ColResultHeader* out, uint16_t* r #if defined(__x86_64__) // Don't use vectorized filtering for text based data types. - if (KIND <= KIND_FLOAT && WIDTH < 16) + if (WIDTH < 16 && + (KIND != KIND_TEXT || (KIND == KIND_TEXT && in->colType.strnxfrmIsValid()) )) { bool canUseFastFiltering = true; for (uint32_t i = 0; i < filterCount; ++i) @@ -1672,6 +1776,8 @@ template ::value || datatypes::is_uint128_t::value, T, + typename datatypes::make_unsigned::type>::type; const uint16_t ridSize = in->NVALS; uint16_t* ridArray = in->getRIDArrayPtr(W); const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; @@ -1682,16 +1788,12 @@ void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, C dataType == execplan::CalpontSystemCatalog::TEXT) && !isDictTokenScan(in)) { - using UT = typename std::conditional::value, T, - typename datatypes::make_unsigned::type>::type; filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); return; } if (datatypes::isUnsigned(dataType)) { - using UT = typename std::conditional::value || datatypes::is_uint128_t::value, T, - typename datatypes::make_unsigned::type>::type; filterColumnData(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter); return; } diff --git a/tests/simd_processors.cpp b/tests/simd_processors.cpp index 519d02845..8883a8743 100644 --- a/tests/simd_processors.cpp +++ b/tests/simd_processors.cpp @@ -30,6 +30,7 @@ class SimdProcessorTypedTest : public testing::Test { using IntegralType = T; public: + void SetUp() override { } diff --git a/utils/common/collation.h b/utils/common/collation.h index 5d43df31f..721c98d16 100644 --- a/utils/common/collation.h +++ b/utils/common/collation.h @@ -1,5 +1,5 @@ /* - Copyright (C) 2020 MariaDB Corporation + Copyright (C) 2020-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -135,6 +135,8 @@ class Charset { protected: const struct charset_info_st* mCharset; + private: + static constexpr uint flags_ = MY_STRXFRM_PAD_WITH_SPACE | MY_STRXFRM_PAD_TO_MAXLEN; public: Charset(CHARSET_INFO& cs) : mCharset(&cs) @@ -182,9 +184,31 @@ class Charset size_t strnxfrm(uchar* dst, size_t dstlen, uint nweights, const uchar* src, size_t srclen, uint flags) { idbassert(mCharset->coll); - return mCharset->coll->strnxfrm(mCharset, dst, dstlen, nweights, src, srclen, flags); } + // The magic check that tells that bytes are mapped to weights as 1:1 + bool strnxfrmIsValid() const + { + return (mCharset->state & MY_CS_NON1TO1) == 0; + } + template + T strnxfrm(const char* src) const + { + T ret = 0; + size_t len __attribute__((unused)) = mCharset->strnxfrm((char*)&ret, sizeof(T), sizeof(T), + src, sizeof(T), flags_); + assert(len <= sizeof(T)); + return ret; + } + template + T strnxfrm(const utils::ConstString &src) const + { + T ret = 0; + size_t len __attribute__((unused)) = mCharset->strnxfrm((char*)&ret, sizeof(T), sizeof(T), + (char*)src.str(), src.length(), flags_); + assert(len <= sizeof(T)); + return ret; + } }; class CollationAwareHasher : public Charset diff --git a/utils/common/simd_sse.h b/utils/common/simd_sse.h index ccf2ff2b7..d6407f58e 100644 --- a/utils/common/simd_sse.h +++ b/utils/common/simd_sse.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2021 Mariadb Corporation. +/* Copyright (C) 2021-2022 Mariadb Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -116,6 +116,15 @@ struct StorageToFiltering:: using type = T; }; +template +static inline vi128_t constant4i() { + static const union { + int i[4]; + vi128_t xmm; + } u = {{i0,i1,i2,i3}}; + return u.xmm; +} + template class SimdFilterProcessor; @@ -462,7 +471,7 @@ class SimdFilterProcessor< template class SimdFilterProcessor::value && sizeof(CHECK_T) == 8 && + typename std::enable_if::value && std::is_same::value && !std::is_same::value>::type> { public: @@ -569,7 +578,117 @@ class SimdFilterProcessor class SimdFilterProcessor::value && sizeof(CHECK_T) == 4 && + typename std::enable_if::value && std::is_same::value && + !std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set_epi64x(fill, fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpGt(y, x) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + SimdType signVec = constant4i<0,(int32_t)0x80000000,0,(int32_t)0x80000000>(); + SimdType xFlip = _mm_xor_si128(x, signVec); + SimdType yFlip = _mm_xor_si128(y, signVec); + return _mm_movemask_epi8(_mm_cmpgt_epi64(xFlip, yFlip)); + } + + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return cmpGt(y, x); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor::value && std::is_same::value && !std::is_same::value>::type> { public: @@ -674,9 +793,119 @@ class SimdFilterProcessor +class SimdFilterProcessor::value && std::is_same::value && + !std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi32(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpGt(y, x) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + SimdType signVec = constant4i<(int32_t)0x80000000,(int32_t)0x80000000,(int32_t)0x80000000,(int32_t)0x80000000>(); + SimdType xFlip = _mm_xor_si128(x, signVec); + SimdType yFlip = _mm_xor_si128(y, signVec); + return _mm_movemask_epi8(_mm_cmpgt_epi32(xFlip, yFlip)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return cmpGt(y, x); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + template class SimdFilterProcessor< - VT, CHECK_T, typename std::enable_if::value && sizeof(CHECK_T) == 2>::type> + VT, CHECK_T, typename std::enable_if::value && std::is_same::value>::type> { public: constexpr static const uint16_t vecByteSize = 16U; @@ -782,7 +1011,227 @@ class SimdFilterProcessor< template class SimdFilterProcessor< - VT, CHECK_T, typename std::enable_if::value && sizeof(CHECK_T) == 1>::type> + VT, CHECK_T, typename std::enable_if::value && std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = simd::vi128_wr; + using SimdType = simd::vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi16(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + SimdType maxOfTwo = _mm_max_epu16(x, y); // max(x, y), unsigned + return _mm_movemask_epi8(_mm_cmpeq_epi16(x, maxOfTwo)); + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return cmpGe(y, x) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGe(y, x); + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return cmpGe(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor< + VT, CHECK_T, typename std::enable_if::value && std::is_same::value>::type> +{ + public: + constexpr static const uint16_t vecByteSize = 16U; + constexpr static const uint16_t vecBitSize = 128U; + using T = typename datatypes::WidthToSIntegralType::type; + using SimdWrapperType = vi128_wr; + using SimdType = vi128_t; + using FilterType = T; + using StorageType = T; + // Mask calculation for int and float types differs. + // See corresponding intrinsics algos for details. + constexpr static const uint16_t FilterMaskStep = sizeof(T); + // Load value + MCS_FORCE_INLINE SimdType emptyNullLoadValue(const T fill) + { + return loadValue(fill); + } + + MCS_FORCE_INLINE SimdType loadValue(const T fill) + { + return _mm_set1_epi8(fill); + } + + // Load from + MCS_FORCE_INLINE SimdType loadFrom(const char* from) + { + return _mm_loadu_si128(reinterpret_cast(from)); + } + + // Compare + MCS_FORCE_INLINE MT cmpEq(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpGe(SimdType& x, SimdType& y) + { + return cmpLt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpGt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpgt_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpLe(SimdType& x, SimdType& y) + { + return cmpGt(x, y) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpLt(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmplt_epi8(x, y)); + } + + MCS_FORCE_INLINE MT cmpNe(SimdType& x, SimdType& y) + { + return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)) ^ 0xFFFF; + } + + MCS_FORCE_INLINE MT cmpAlwaysFalse(SimdType& x, SimdType& y) + { + return 0; + } + + MCS_FORCE_INLINE MT cmpAlwaysTrue(SimdType& x, SimdType& y) + { + return 0xFFFF; + } + + // permute + /* TODO Available in AVX-512 + MCS_FORCE_INLINE SimdType perm8Bits(SimdType& x, SimdType& idx) + { + return _mm_permutexvar_epi8(x, idx); + } + */ + // misc + MCS_FORCE_INLINE MT convertVectorToBitMask(SimdType& vmask) + { + return _mm_movemask_epi8(vmask); + } + + MCS_FORCE_INLINE MT nullEmptyCmpNe(SimdType& x, SimdType& y) + { + return cmpNe(x, y); + } + + MCS_FORCE_INLINE MT nullEmptyCmpEq(SimdType& x, SimdType& y) + { + return cmpEq(x, y); + } + + MCS_FORCE_INLINE SimdType setToZero() + { + return _mm_setzero_si128(); + } + + // store + MCS_FORCE_INLINE void storeWMask(SimdType& x, SimdType& vmask, char* dst) + { + _mm_maskmoveu_si128(x, vmask, dst); + } + + MCS_FORCE_INLINE void store(char* dst, SimdType& x) + { + _mm_storeu_si128(reinterpret_cast(dst), x); + } +}; + +template +class SimdFilterProcessor< + VT, CHECK_T, typename std::enable_if::value && std::is_same::value>::type> { public: constexpr static const uint16_t vecByteSize = 16U; From 2ec502aaf3d3c0164e128e82c6b59c46c7d68532 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 1 Apr 2022 09:18:34 -0500 Subject: [PATCH 40/55] MCOL-4576: remove S3 options from cpimport. (#2328) --- writeengine/bulk/cpimport.cpp | 17 ++++++++--------- writeengine/splitter/we_cmdargs.cpp | 16 ++++++++-------- writeengine/splitter/we_filereadthread.cpp | 9 +++++++++ 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/writeengine/bulk/cpimport.cpp b/writeengine/bulk/cpimport.cpp index 4b1fbac8d..08e8871be 100644 --- a/writeengine/bulk/cpimport.cpp +++ b/writeengine/bulk/cpimport.cpp @@ -172,12 +172,11 @@ void printUsage() << " -T Timezone used for TIMESTAMP datatype" << endl << " Possible values: \"SYSTEM\" (default)" << endl << " : Offset in the form +/-HH:MM" << endl - << endl - << " -y S3 Authentication Key (for S3 imports)" << endl - << " -K S3 Authentication Secret (for S3 imports)" << endl - << " -t S3 Bucket (for S3 imports)" << endl - << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl - << " -g S3 Regions (for S3 imports)" << endl +// << " -y S3 Authentication Key (for S3 imports)" << endl +// << " -K S3 Authentication Secret (for S3 imports)" << endl +// << " -t S3 Bucket (for S3 imports)" << endl +// << " -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl +// << " -g S3 Regions (for S3 imports)" << endl << " -U username of new data files owner. Default is mysql" << endl; cout << " Example1:" << endl @@ -310,7 +309,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob BulkModeType bulkMode = BULK_MODE_LOCAL; std::string jobUUID; - while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:U:")) != + while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:U:")) != EOF) { switch (option) @@ -676,7 +675,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob BulkLoad::disableConsoleOutput(true); break; } - +/* case 'y': { curJob.setS3Key(optarg); @@ -706,7 +705,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob curJob.setS3Region(optarg); break; } - +*/ case 'U': { curJob.setUsername(optarg); diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index 27efb1d37..9ad76a451 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -561,11 +561,11 @@ void WECmdArgs::usage() << "\t-T\tTimezone used for TIMESTAMP datatype.\n" << "\t\tPossible values: \"SYSTEM\" (default)\n" << "\t\t : Offset in the form +/-HH:MM\n" - << "\t-y\tS3 Authentication Key (for S3 imports)\n" - << "\t-K\tS3 Authentication Secret (for S3 imports)\n" - << "\t-t\tS3 Bucket (for S3 imports)\n" - << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" - << "\t-g\tS3 Region (for S3 imports)\n" +// << "\t-y\tS3 Authentication Key (for S3 imports)\n" +// << "\t-K\tS3 Authentication Secret (for S3 imports)\n" +// << "\t-t\tS3 Bucket (for S3 imports)\n" +// << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" +// << "\t-g\tS3 Region (for S3 imports)\n" << "\t-L\tDirectory for the output .err and .bad files.\n" << "\t\tDefault is " << string(MCSLOGDIR); @@ -598,7 +598,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) if (argc > 0) fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; // argv[0] is splitter but we need cpimport - while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:")) != EOF) + while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:NU:L:")) != EOF) { switch (aCh) { @@ -906,7 +906,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fConsoleOutput = false; break; } - +/* case 'y': //-y S3 Key { fS3Key = optarg; @@ -936,7 +936,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fS3Region = optarg; break; } - +*/ case 'U': //-U username of the files owner { fUsername = optarg; diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index a21d5ef1f..4b64ad3fb 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -481,6 +481,15 @@ void WEFileReadThread::openInFile() use ms3 lib to d/l data into mem use boost::iostreams to wrap the mem in a stream interface point infile's stream buffer to it. + + MCOL-4576: The options to setup S3 with cpimport have been removed and this + code is unreachable. However we may need to resurrect it at some point in some form. + Performance issues with extremely large data files as well as the fact files larger + than system memory will cause an OOM error. Multipart downloads/uploads need to be + implemented or more likely a different streaming solution developed with external API tools + + MCOL-4576 work around is to use 3rd party CLI tools and pipe data file from S3 bucket + into cpimport stdin. 3rd party tooling for large object downloads will be more efficient. */ if (fSdh.getDebugLvl()) From e174696351043558bc4d97d346afda12e8de507f Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 2 Mar 2022 17:13:29 +0000 Subject: [PATCH 41/55] MCOL-5001 This patch merges ExeMgr and PrimProc runtimes EM and PP are most resource-hungry runtimes. The merge enables to control their cummulative resource consumption, thread allocation + enables zero-copy data exchange b/w local EM and PP facilities. --- dbcon/joblist/distributedenginecomm.cpp | 4 +- dbcon/joblist/resourcemanager.cpp | 23 +- dbcon/joblist/resourcemanager.h | 28 +- debian/mariadb-plugin-columnstore.install | 1 - exemgr/CMakeLists.txt | 12 +- exemgr/serviceexemgr.h | 1 + oam/install_scripts/mcs-exemgr.service.in | 9 +- primitives/primproc/CMakeLists.txt | 11 +- .../primproc/activestatementcounter.cpp | 59 ++ primitives/primproc/activestatementcounter.h | 64 ++ primitives/primproc/femsghandler.cpp | 143 +++ primitives/primproc/femsghandler.h | 47 + primitives/primproc/primitiveserver.cpp | 9 +- primitives/primproc/primitiveserver.h | 2 +- primitives/primproc/primproc.cpp | 54 +- primitives/primproc/rssmonfcn.cpp | 69 ++ primitives/primproc/rssmonfcn.h | 36 + primitives/primproc/serviceexemgr.cpp | 457 ++++++++ primitives/primproc/serviceexemgr.h | 348 +++++++ primitives/primproc/sqlfrontsessionthread.cpp | 985 ++++++++++++++++++ primitives/primproc/sqlfrontsessionthread.h | 131 +++ utils/common/spinlock.h | 29 + utils/joiner/joiner.h | 32 +- 23 files changed, 2443 insertions(+), 111 deletions(-) create mode 100644 primitives/primproc/activestatementcounter.cpp create mode 100644 primitives/primproc/activestatementcounter.h create mode 100644 primitives/primproc/femsghandler.cpp create mode 100644 primitives/primproc/femsghandler.h create mode 100644 primitives/primproc/rssmonfcn.cpp create mode 100644 primitives/primproc/rssmonfcn.h create mode 100644 primitives/primproc/serviceexemgr.cpp create mode 100644 primitives/primproc/serviceexemgr.h create mode 100644 primitives/primproc/sqlfrontsessionthread.cpp create mode 100644 primitives/primproc/sqlfrontsessionthread.h diff --git a/dbcon/joblist/distributedenginecomm.cpp b/dbcon/joblist/distributedenginecomm.cpp index 67f40f268..a960e7374 100644 --- a/dbcon/joblist/distributedenginecomm.cpp +++ b/dbcon/joblist/distributedenginecomm.cpp @@ -292,7 +292,7 @@ void DistributedEngineComm::Setup() catch (std::exception& ex) { if (i < newPmCount) - newPmCount--; + newPmCount = newPmCount > 1 ? newPmCount-1 : 1; // We can't afford to reduce newPmCount to 0 writeToLog(__FILE__, __LINE__, "Could not connect to PMS" + std::to_string(connectionId) + ": " + ex.what(), @@ -302,7 +302,7 @@ void DistributedEngineComm::Setup() catch (...) { if (i < newPmCount) - newPmCount--; + newPmCount = newPmCount > 1 ? newPmCount-1 : 1; // We can't afford to reduce newPmCount to 0 writeToLog(__FILE__, __LINE__, "Could not connect to PMS" + std::to_string(connectionId), LOG_TYPE_ERROR); diff --git a/dbcon/joblist/resourcemanager.cpp b/dbcon/joblist/resourcemanager.cpp index da041efa6..be03066c8 100644 --- a/dbcon/joblist/resourcemanager.cpp +++ b/dbcon/joblist/resourcemanager.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2022 Mariadb Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -40,39 +41,25 @@ using namespace config; namespace joblist { -// const string ResourceManager::fExeMgrStr("ExeMgr1"); -const string ResourceManager::fHashJoinStr("HashJoin"); -const string ResourceManager::fHashBucketReuseStr("HashBucketReuse"); -const string ResourceManager::fJobListStr("JobList"); -const string ResourceManager::fPrimitiveServersStr("PrimitiveServers"); -// const string ResourceManager::fSystemConfigStr("SystemConfig"); -const string ResourceManager::fTupleWSDLStr("TupleWSDL"); -const string ResourceManager::fZDLStr("ZDL"); -const string ResourceManager::fExtentMapStr("ExtentMap"); -// const string ResourceManager::fDMLProcStr("DMLProc"); -// const string ResourceManager::fBatchInsertStr("BatchInsert"); -const string ResourceManager::fOrderByLimitStr("OrderByLimit"); -const string ResourceManager::fRowAggregationStr("RowAggregation"); - ResourceManager* ResourceManager::fInstance = NULL; boost::mutex mx; -ResourceManager* ResourceManager::instance(bool runningInExeMgr) +ResourceManager* ResourceManager::instance(bool runningInExeMgr, config::Config* aConfig) { boost::mutex::scoped_lock lk(mx); if (!fInstance) - fInstance = new ResourceManager(runningInExeMgr); + fInstance = new ResourceManager(runningInExeMgr, aConfig); return fInstance; } -ResourceManager::ResourceManager(bool runningInExeMgr) +ResourceManager::ResourceManager(bool runningInExeMgr, config::Config* aConfig) : fExeMgrStr("ExeMgr1") , fSystemConfigStr("SystemConfig") , fDMLProcStr("DMLProc") , fBatchInsertStr("BatchInsert") - , fConfig(Config::makeConfig()) + , fConfig(aConfig == nullptr ? Config::makeConfig() : aConfig) , fNumCores(8) , fHjNumThreads(defaultNumThreads) , fJlProcessorThreadsPerScan(defaultProcessorThreadsPerScan) diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index 64ff17a55..76b08760b 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2022 Mariadb Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -138,11 +139,8 @@ class ResourceManager /** @brief ctor * */ - EXPORT ResourceManager(bool runningInExeMgr = false); - static ResourceManager* instance(bool runningInExeMgr = false); - // ResourceManager(const config::Config *cf); - // ResourceManager(const std::string& config); - // passed by ExeMgr and DistributedEngineComm to MessageQueueServer or -Client + EXPORT ResourceManager(bool runningInExeMgr = false, config::Config *aConfig = nullptr); + static ResourceManager* instance(bool runningInExeMgr = false, config::Config *aConfig = nullptr); config::Config* getConfig() { return fConfig; @@ -577,19 +575,19 @@ class ResourceManager void logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, uint64_t value = 0, uint32_t sessionId = 0); - /*static const*/ std::string fExeMgrStr; - static const std::string fHashJoinStr; - static const std::string fHashBucketReuseStr; - static const std::string fJobListStr; - static const std::string fPrimitiveServersStr; + std::string fExeMgrStr; + inline static const std::string fHashJoinStr = "HashJoin"; + inline static const std::string fHashBucketReuseStr = "HashBucketReuse"; + inline static const std::string fJobListStr = "JobList"; + inline static const std::string fPrimitiveServersStr = "PrimitiveServers"; /*static const*/ std::string fSystemConfigStr; - static const std::string fTupleWSDLStr; - static const std::string fZDLStr; - static const std::string fExtentMapStr; + inline static const std::string fTupleWSDLStr = "TupleWSDL"; + inline static const std::string fZDLStr = "ZDL"; + inline static const std::string fExtentMapStr = "ExtentMap"; /*static const*/ std::string fDMLProcStr; /*static const*/ std::string fBatchInsertStr; - static const std::string fOrderByLimitStr; - static const std::string fRowAggregationStr; + inline static const std::string fOrderByLimitStr = "OrderByLimit"; + inline static const std::string fRowAggregationStr = "RowAggregation"; config::Config* fConfig; static ResourceManager* fInstance; uint32_t fTraceFlags; diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index 39f4e8c72..20241ff57 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -6,7 +6,6 @@ etc/mysql/mariadb.conf.d/columnstore.cnf usr/bin/mcsRebuildEM usr/bin/DDLProc usr/bin/DMLProc -usr/bin/ExeMgr usr/bin/PrimProc usr/bin/StorageManager usr/bin/WriteEngineServer diff --git a/exemgr/CMakeLists.txt b/exemgr/CMakeLists.txt index 729cc98cd..e0d17d205 100644 --- a/exemgr/CMakeLists.txt +++ b/exemgr/CMakeLists.txt @@ -1,18 +1,18 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ) +# include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ExeMgr_SRCS serviceexemgr.cpp sqlfrontsessionthread.cpp rssmonfcn.cpp exemgr.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) +# set(ExeMgr_SRCS serviceexemgr.cpp sqlfrontsessionthread.cpp rssmonfcn.cpp exemgr.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) -add_executable(ExeMgr ${ExeMgr_SRCS}) +# add_executable(ExeMgr ${ExeMgr_SRCS}) -target_link_libraries(ExeMgr ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} cacheutils threadpool) +# target_link_libraries(ExeMgr ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} cacheutils threadpool) -target_include_directories(ExeMgr PRIVATE ${Boost_INCLUDE_DIRS}) +# target_include_directories(ExeMgr PRIVATE ${Boost_INCLUDE_DIRS}) -install(TARGETS ExeMgr DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +# install(TARGETS ExeMgr DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) ########### install files ############### diff --git a/exemgr/serviceexemgr.h b/exemgr/serviceexemgr.h index 544978ee7..afcf07914 100644 --- a/exemgr/serviceexemgr.h +++ b/exemgr/serviceexemgr.h @@ -84,6 +84,7 @@ namespace exemgr } } } + Opt(): m_debug(0), m_e(false), m_fg(false) {}; int getDebugLevel() const { return m_debug; diff --git a/oam/install_scripts/mcs-exemgr.service.in b/oam/install_scripts/mcs-exemgr.service.in index b9a36fcaf..4cbddfc4a 100644 --- a/oam/install_scripts/mcs-exemgr.service.in +++ b/oam/install_scripts/mcs-exemgr.service.in @@ -6,15 +6,18 @@ PartOf=mcs-primproc.service After=network.target mcs-primproc.service [Service] -Type=forking +Type=oneshot User=@DEFAULT_USER@ Group=@DEFAULT_GROUP@ LimitNOFILE=65536 LimitNPROC=65536 -ExecStartPre=/usr/bin/env bash -c "ldconfig -p | grep -m1 libjemalloc > /dev/null || echo 'Please install jemalloc to avoid ColumnStore performance degradation and unexpected service interruptions.'" -ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/ExeMgr" +#ExecStartPre=/usr/bin/env bash -c "ldconfig -p | grep -m1 libjemalloc > /dev/null || echo 'Please install jemalloc to avoid ColumnStore performance degradation and unexpected service interruptions.'" +#ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/ExeMgr" +ExecStart=/bin/echo 'EM dummy start' + +RemainAfterExit=yes Restart=on-failure TimeoutStopSec=2 diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index b8d2e3741..69d83f464 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -19,14 +19,17 @@ set(PrimProc_SRCS pseudocc.cpp rtscommand.cpp umsocketselector.cpp + serviceexemgr.cpp + sqlfrontsessionthread.cpp + rssmonfcn.cpp + activestatementcounter.cpp + femsghandler.cpp ../../utils/common/crashtrace.cpp) add_executable(PrimProc ${PrimProc_SRCS}) add_dependencies(PrimProc loggingcpp) - +target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(PrimProc ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} threadpool cacheutils dbbc processor) -install(TARGETS PrimProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) - - +install(TARGETS PrimProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) \ No newline at end of file diff --git a/primitives/primproc/activestatementcounter.cpp b/primitives/primproc/activestatementcounter.cpp new file mode 100644 index 000000000..b25e61549 --- /dev/null +++ b/primitives/primproc/activestatementcounter.cpp @@ -0,0 +1,59 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +// $Id: activestatementcounter.cpp 940 2013-01-21 14:11:31Z rdempsey $ +// + +#include +#include +using namespace boost; + +#include "activestatementcounter.h" + +void ActiveStatementCounter::incr(bool& counted) +{ + if (counted) + return; + + counted = true; + boost::mutex::scoped_lock lk(fMutex); + + if (upperLimit > 0) + while (fStatementCount >= upperLimit) + { + fStatementsWaiting++; + condvar.wait(lk); + --fStatementsWaiting; + } + + fStatementCount++; +} + +void ActiveStatementCounter::decr(bool& counted) +{ + if (!counted) + return; + + counted = false; + boost::mutex::scoped_lock lk(fMutex); + + if (fStatementCount == 0) + return; + + --fStatementCount; + condvar.notify_one(); +} diff --git a/primitives/primproc/activestatementcounter.h b/primitives/primproc/activestatementcounter.h new file mode 100644 index 000000000..87e7163aa --- /dev/null +++ b/primitives/primproc/activestatementcounter.h @@ -0,0 +1,64 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +// $Id: activestatementcounter.h 940 2013-01-21 14:11:31Z rdempsey $ +// +/** @file */ + +#pragma once + +#include + +#include +#include + +#include "vss.h" + +class ActiveStatementCounter +{ + public: + ActiveStatementCounter(uint32_t limit) : fStatementCount(0), upperLimit(limit), fStatementsWaiting(0) + { + } + + virtual ~ActiveStatementCounter() + { + } + + void incr(bool& counted); + void decr(bool& counted); + uint32_t cur() const + { + return fStatementCount; + } + uint32_t waiting() const + { + return fStatementsWaiting; + } + + private: + ActiveStatementCounter(const ActiveStatementCounter& rhs); + ActiveStatementCounter& operator=(const ActiveStatementCounter& rhs); + + uint32_t fStatementCount; + uint32_t upperLimit; + uint32_t fStatementsWaiting; + boost::mutex fMutex; + boost::condition condvar; + BRM::VSS fVss; +}; + diff --git a/primitives/primproc/femsghandler.cpp b/primitives/primproc/femsghandler.cpp new file mode 100644 index 000000000..0ce22519f --- /dev/null +++ b/primitives/primproc/femsghandler.cpp @@ -0,0 +1,143 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "messagequeue.h" +#include "iosocket.h" + +#include "femsghandler.h" + +using namespace std; +using namespace joblist; +using namespace messageqcpp; + +threadpool::ThreadPool FEMsgHandler::threadPool; + +namespace +{ +class Runner +{ + public: + Runner(FEMsgHandler* f) : target(f) + { + } + void operator()() + { + target->threadFcn(); + } + FEMsgHandler* target; +}; + +} // namespace + +FEMsgHandler::FEMsgHandler() : die(false), running(false), sawData(false), sock(NULL) +{ +} + +FEMsgHandler::FEMsgHandler(boost::shared_ptr j, IOSocket* s) + : die(false), running(false), sawData(false), jl(j) +{ + sock = s; + assert(sock); +} + +FEMsgHandler::~FEMsgHandler() +{ + stop(); + threadPool.join(thr); +} + +void FEMsgHandler::start() +{ + if (!running) + { + running = true; + thr = threadPool.invoke(Runner(this)); + } +} + +void FEMsgHandler::stop() +{ + die = true; + jl.reset(); +} + +void FEMsgHandler::setJobList(boost::shared_ptr j) +{ + jl = j; +} + +void FEMsgHandler::setSocket(IOSocket* i) +{ + sock = i; + assert(sock); +} + +/* Note, the next two fcns strongly depend on ExeMgr's current implementation. There's a + * good chance that if ExeMgr's table send loop is changed, these will need to be + * updated to match. + */ + +/* This is currently only called if InetStreamSocket::write() throws, implying + * a connection error. It might not make sense in other contexts. + */ +bool FEMsgHandler::aborted() +{ + if (sawData) + return true; + + boost::mutex::scoped_lock sl(mutex); + int err; + int connectionNum = sock->getConnectionNum(); + + err = InetStreamSocket::pollConnection(connectionNum, 1000); + + if (err == 1) + { + sawData = true; + return true; + } + + return false; +} + +void FEMsgHandler::threadFcn() +{ + int err = 0; + int connectionNum = sock->getConnectionNum(); + + /* This waits for the next readable event on sock. An abort is signaled + * by sending something (anything at the moment), then dropping the connection. + * This fcn exits on all other events. + */ + while (!die && err == 0) + { + boost::mutex::scoped_lock sl(mutex); + err = InetStreamSocket::pollConnection(connectionNum, 1000); + } + + if (err == 1) + sawData = true; // there's data to read, must be the abort signal + + if (!die && (err == 2 || err == 1)) + { + die = true; + jl->abort(); + jl.reset(); + } + + running = false; +} diff --git a/primitives/primproc/femsghandler.h b/primitives/primproc/femsghandler.h new file mode 100644 index 000000000..baa91ad90 --- /dev/null +++ b/primitives/primproc/femsghandler.h @@ -0,0 +1,47 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include "joblist.h" +#include "inetstreamsocket.h" +#include "threadpool.h" + +class FEMsgHandler +{ + public: + FEMsgHandler(); + FEMsgHandler(boost::shared_ptr, messageqcpp::IOSocket*); + virtual ~FEMsgHandler(); + + void start(); + void stop(); + void setJobList(boost::shared_ptr); + void setSocket(messageqcpp::IOSocket*); + bool aborted(); + + void threadFcn(); + + static threadpool::ThreadPool threadPool; + + private: + bool die, running, sawData; + messageqcpp::IOSocket* sock; + boost::shared_ptr jl; + boost::mutex mutex; + uint64_t thr; +}; diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index ad073ddc7..89c6b70f5 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -945,7 +945,6 @@ struct AsynchLoader QueryContext ver; uint32_t txn; int compType; - uint8_t dataWidth; bool LBIDTrace; uint32_t sessionID; uint32_t* cacheCount; @@ -1420,8 +1419,8 @@ struct BPPHandler /* Uncomment these lines to verify duplicate(). == op might need updating */ // if (*bpp != *dup) - // cerr << "createBPP: duplicate mismatch at index " << i << - // endl; + // cerr << "createBPP: duplicate mismatch at index " << i + // << endl; // idbassert(*bpp == *dup); bppv->add(dup); } @@ -2441,7 +2440,7 @@ PrimitiveServer::~PrimitiveServer() { } -void PrimitiveServer::start(Service* service) +void PrimitiveServer::start(Service* service, utils::USpaceSpinLock& startupRaceLock) { // start all the server threads for (int i = 1; i <= fServerThreads; i++) @@ -2452,7 +2451,7 @@ void PrimitiveServer::start(Service* service) fServerpool.invoke(ServerThread(oss.str(), this)); } - + startupRaceLock.release(); service->NotifyServiceStarted(); fServerpool.wait(); diff --git a/primitives/primproc/primitiveserver.h b/primitives/primproc/primitiveserver.h index 5f35911e4..c79c0f807 100644 --- a/primitives/primproc/primitiveserver.h +++ b/primitives/primproc/primitiveserver.h @@ -124,7 +124,7 @@ class PrimitiveServer /** @brief start the primitive server * */ - void start(Service* p); + void start(Service* p, utils::USpaceSpinLock& startupRaceLock); /** @brief get a pointer the shared processor thread pool */ diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index b65846533..260a3ca96 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -30,15 +30,15 @@ #endif #include #include -#ifndef _MSC_VER #include #include -#else -#include -#endif + #include #include #include +#include +#include +#include //#define NDEBUG #include using namespace std; @@ -72,7 +72,9 @@ using namespace idbdatafile; #include "mariadb_my_sys.h" +#include "spinlock.h" #include "service.h" +#include "serviceexemgr.h" class Opt { @@ -115,6 +117,14 @@ class ServicePrimProc : public Service, public Opt { return m_fg ? Child() : RunForking(); } + std::atomic_flag& getStartupRaceFlag() + { + return startupRaceFlag_; + } + + private: + // Since C++20 flag's init value is false. + std::atomic_flag startupRaceFlag_; }; namespace primitiveprocessor @@ -152,28 +162,11 @@ int toInt(const string& val) void setupSignalHandlers() { #ifndef _MSC_VER - signal(SIGHUP, SIG_IGN); - struct sigaction ign; - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &ign, 0); - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - sigaction(SIGUSR1, &ign, 0); - memset(&ign, 0, sizeof(ign)); ign.sa_handler = SIG_IGN; sigaction(SIGUSR2, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); - sigset_t sigset; sigemptyset(&sigset); sigaddset(&sigset, SIGPIPE); @@ -382,6 +375,18 @@ int ServicePrimProc::Child() NotifyServiceInitializationFailed(); return 2; } + utils::USpaceSpinLock startupRaceLock(getStartupRaceFlag()); + std::thread exeMgrThread( + [this, cf]() + { + exemgr::Opt opt; + exemgr::globServiceExeMgr = new exemgr::ServiceExeMgr(opt, cf); + // primitive delay to avoid 'not connected to PM' log error messages + // from EM. PrimitiveServer::start() releases SpinLock after sockets + // are available. + utils::USpaceSpinLock startupRaceLock(this->getStartupRaceFlag()); + exemgr::globServiceExeMgr->Child(); + }); int serverThreads = 1; int serverQueueSize = 10; @@ -395,7 +400,6 @@ int ServicePrimProc::Child() bool rotatingDestination = false; uint32_t deleteBlocks = 128; bool PTTrace = false; - int temp; string strTemp; int priority = -1; const string primitiveServers("PrimitiveServers"); @@ -414,7 +418,7 @@ int ServicePrimProc::Child() gDebugLevel = primitiveprocessor::NONE; - temp = toInt(cf->getConfig(primitiveServers, "ServerThreads")); + int temp = toInt(cf->getConfig(primitiveServers, "ServerThreads")); if (temp > 0) serverThreads = temp; @@ -761,7 +765,7 @@ int ServicePrimProc::Child() } #endif - server.start(this); + server.start(this, startupRaceLock); cerr << "server.start() exited!" << endl; diff --git a/primitives/primproc/rssmonfcn.cpp b/primitives/primproc/rssmonfcn.cpp new file mode 100644 index 000000000..8a9b62d83 --- /dev/null +++ b/primitives/primproc/rssmonfcn.cpp @@ -0,0 +1,69 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include + +#include "rssmonfcn.h" +#include "serviceexemgr.h" + +namespace exemgr +{ + void RssMonFcn::operator()() const + { + logging::Logger& msgLog = globServiceExeMgr->getLogger(); + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + for (;;) + { + size_t rssMb = rss(); + size_t pct = rssMb * 100 / fMemTotal; + + if (pct > fMaxPct) + { + if (fMaxPct >= 95) + { + std::cerr << "Too much memory allocated!" << std::endl; + logging::Message::Args args; + args.add((int)pct); + args.add((int)fMaxPct); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logRssTooBig, args, logging::LoggingID(16)); + exit(1); + } + + if (statementsRunningCount->cur() == 0) + { + std::cerr << "Too much memory allocated!" << std::endl; + logging::Message::Args args; + args.add((int)pct); + args.add((int)fMaxPct); + msgLog.logMessage(logging::LOG_TYPE_WARNING, ServiceExeMgr::logRssTooBig, args, logging::LoggingID(16)); + exit(1); + } + + std::cerr << "Too much memory allocated, but stmts running" << std::endl; + } + + // Update sessionMemMap entries lower than current mem % use + globServiceExeMgr->updateSessionMap(pct); + + pause_(); + } + } + void startRssMon(size_t maxPct, int pauseSeconds) + { + new std::thread(RssMonFcn(maxPct, pauseSeconds)); + } +} // namespace \ No newline at end of file diff --git a/primitives/primproc/rssmonfcn.h b/primitives/primproc/rssmonfcn.h new file mode 100644 index 000000000..341038e91 --- /dev/null +++ b/primitives/primproc/rssmonfcn.h @@ -0,0 +1,36 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include + +#include "MonitorProcMem.h" + +namespace exemgr +{ +class RssMonFcn : public utils::MonitorProcMem +{ + public: + RssMonFcn(size_t maxPct, int pauseSeconds) : MonitorProcMem(maxPct, 0, 21, pauseSeconds) + { + } + + void operator()() const; +}; + +} // namespace \ No newline at end of file diff --git a/primitives/primproc/serviceexemgr.cpp b/primitives/primproc/serviceexemgr.cpp new file mode 100644 index 000000000..8746b68f7 --- /dev/null +++ b/primitives/primproc/serviceexemgr.cpp @@ -0,0 +1,457 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2019-22 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/********************************************************************** + * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ + * + * + ***********************************************************************/ +/** + * @brief execution plan manager main program + * + * This is the ?main? program for dealing with execution plans and + * result sets. It sits in a loop waiting for CalpontExecutionPlan + * from the FEP. It then passes the CalpontExecutionPlan to the + * JobListFactory from which a JobList is obtained. This is passed + * to to the Query Manager running in the real-time portion of the + * EC. The ExecutionPlanManager waits until the Query Manager + * returns a result set for the job list. These results are passed + * into the CalpontResultFactory, which outputs a CalpontResultSet. + * The ExecutionPlanManager passes the CalpontResultSet into the + * VendorResultFactory which produces a result set tailored to the + * specific DBMS front end in use. The ExecutionPlanManager then + * sends the VendorResultSet back to the Calpont Database Connector + * on the Front-End Processor where it is returned to the DBMS + * front-end. + */ +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" +#include "serviceexemgr.h" +#include "sqlfrontsessionthread.h" + +namespace exemgr +{ +ServiceExeMgr* globServiceExeMgr = nullptr; +void startRssMon(size_t maxPct, int pauseSeconds); + +void added_a_pm(int) +{ + logging::LoggingID logid(21, 0, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add("exeMgr caught SIGHUP. Resetting connections"); + msg.format(args1); + std::cout << msg.msg().c_str() << std::endl; + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); + + auto* dec = exemgr::globServiceExeMgr->getDec(); + if (dec) + { + oam::OamCache* oamCache = oam::OamCache::makeOamCache(); + oamCache->forceReload(); + dec->Setup(); + } +} + +void printTotalUmMemory(int sig) +{ + int64_t num = globServiceExeMgr->getRm().availableMemory(); + std::cout << "Total UM memory available: " << num << std::endl; +} + +void ServiceExeMgr::setupSignalHandlers() +{ + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = SIG_IGN; + + sigaction(SIGPIPE, &ign, 0); + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = exemgr::added_a_pm; + sigaction(SIGHUP, &ign, 0); + ign.sa_handler = exemgr::printTotalUmMemory; + sigaction(SIGUSR1, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); +} + +void cleanTempDir() +{ + using TempDirPurpose = config::Config::TempDirPurpose; + struct Dirs + { + std::string section; + std::string allowed; + TempDirPurpose purpose; + }; + std::vector dirs{{"HashJoin", "AllowDiskBasedJoin", TempDirPurpose::Joins}, + {"RowAggregation", "AllowDiskBasedAggregation", TempDirPurpose::Aggregates}}; + const auto config = config::Config::makeConfig(); + + for (const auto& dir : dirs) + { + std::string allowStr = config->getConfig(dir.section, dir.allowed); + bool allow = (allowStr == "Y" || allowStr == "y"); + + std::string tmpPrefix = config->getTempFileDir(dir.purpose); + + if (allow && tmpPrefix.empty()) + { + std::cerr << "Empty tmp directory name for " << dir.section << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Empty tmp directory name for:"); + args.add(dir.section); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_CRITICAL, message, logid); + } + + tmpPrefix += "/"; + + idbassert(tmpPrefix != "/"); + + /* This is quite scary as ExeMgr usually runs as root */ + try + { + if (allow) + { + boost::filesystem::remove_all(tmpPrefix); + } + boost::filesystem::create_directories(tmpPrefix); + } + catch (const std::exception& ex) + { + std::cerr << ex.what() << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Exception whilst cleaning tmpdir: "); + args.add(ex.what()); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); + } + catch (...) + { + std::cerr << "Caught unknown exception during tmpdir cleanup" << std::endl; + logging::LoggingID logid(16, 0, 0); + logging::Message::Args args; + logging::Message message(8); + args.add("Unknown exception whilst cleaning tmpdir"); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); + } + } +} + +const std::string ServiceExeMgr::prettyPrintMiniInfo(const std::string& in) +{ + // 1. take the std::string and tok it by '\n' + // 2. for each part in each line calc the longest part + // 3. padding to each longest value, output a header and the lines + using CharTraits = std::basic_string::traits_type; + using CharSeparator = boost::char_separator; + using Tokeniser = boost::tokenizer>; + CharSeparator sep1("\n"); + Tokeniser tok1(in, sep1); + std::vector lines; + std::string header = "Desc Mode Table TableOID ReferencedColumns PIO LIO PBE Elapsed Rows"; + const int header_parts = 10; + lines.push_back(header); + + for (auto iter1 = tok1.begin(); iter1 != tok1.end(); ++iter1) + { + if (!iter1->empty()) + lines.push_back(*iter1); + } + + std::vector lens; + + for (int i = 0; i < header_parts; i++) + lens.push_back(0); + + std::vector> lineparts; + std::vector::iterator iter2; + int j; + + for (iter2 = lines.begin(), j = 0; iter2 != lines.end(); ++iter2, j++) + { + CharSeparator sep2(" "); + Tokeniser tok2(*iter2, sep2); + int i; + std::vector parts; + Tokeniser::iterator iter3; + + for (iter3 = tok2.begin(), i = 0; iter3 != tok2.end(); ++iter3, i++) + { + if (i >= header_parts) + break; + + std::string part(*iter3); + + if (j != 0 && i == 8) + part.resize(part.size() - 3); + + assert(i < header_parts); + + if (part.size() > lens[i]) + lens[i] = part.size(); + + parts.push_back(part); + } + + assert(i == header_parts); + lineparts.push_back(parts); + } + + std::ostringstream oss; + + std::vector>::iterator iter1 = lineparts.begin(); + std::vector>::iterator end1 = lineparts.end(); + + oss << "\n"; + + while (iter1 != end1) + { + std::vector::iterator iter2 = iter1->begin(); + std::vector::iterator end2 = iter1->end(); + assert(distance(iter2, end2) == header_parts); + int i = 0; + + while (iter2 != end2) + { + assert(i < header_parts); + oss << std::setw(lens[i]) << std::left << *iter2 << " "; + ++iter2; + i++; + } + + oss << "\n"; + ++iter1; + } + + return oss.str(); +} + +int ServiceExeMgr::Child() +{ + // Make sure CSC thinks it's on a UM or else bucket reuse stuff below will stall + if (!m_e) + setenv("CALPONT_CSC_IDENT", "um", 1); + + setupSignalHandlers(); + int err = 0; + if (!m_debug) + err = setupResources(); + std::string errMsg; + + switch (err) + { + case -1: + case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; + + case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; + + case -4: + errMsg = "Could not install file limits to required value, please see non-root install documentation"; + break; + + default: errMsg = "Couldn't change working directory or unknown error"; break; + } + + cleanTempDir(); + + logging::MsgMap msgMap; + msgMap[logDefaultMsg] = logging::Message(logDefaultMsg); + msgMap[logDbProfStartStatement] = logging::Message(logDbProfStartStatement); + msgMap[logDbProfEndStatement] = logging::Message(logDbProfEndStatement); + msgMap[logStartSql] = logging::Message(logStartSql); + msgMap[logEndSql] = logging::Message(logEndSql); + msgMap[logRssTooBig] = logging::Message(logRssTooBig); + msgMap[logDbProfQueryStats] = logging::Message(logDbProfQueryStats); + msgMap[logExeMgrExcpt] = logging::Message(logExeMgrExcpt); + msgLog_.msgMap(msgMap); + + dec_ = joblist::DistributedEngineComm::instance(rm_, true); + dec_->Open(); + + bool tellUser = true; + + messageqcpp::MessageQueueServer* mqs; + + statementsRunningCount_ = new ActiveStatementCounter(rm_->getEmExecQueueSize()); + const std::string ExeMgr = "ExeMgr1"; + for (;;) + { + try + { + mqs = new messageqcpp::MessageQueueServer(ExeMgr, rm_->getConfig(), messageqcpp::ByteStream::BlockSize, + 64); + break; + } + catch (std::runtime_error& re) + { + std::string what = re.what(); + + if (what.find("Address already in use") != std::string::npos) + { + if (tellUser) + { + std::cerr << "Address already in use, retrying..." << std::endl; + tellUser = false; + } + + sleep(5); + } + else + { + throw; + } + } + } + + // class jobstepThreadPool is used by other processes. We can't call + // resourcemanaager (rm) functions during the static creation of threadpool + // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). + // From the pools perspective, it has no idea if it is ExeMgr doing the + // creation, so it has no idea which way to set the flag. So we set the max here. + joblist::JobStep::jobstepThreadPool.setMaxThreads(rm_->getJLThreadPoolSize()); + joblist::JobStep::jobstepThreadPool.setName("ExeMgrJobList"); + + if (rm_->getJlThreadPoolDebug() == "Y" || rm_->getJlThreadPoolDebug() == "y") + { + joblist::JobStep::jobstepThreadPool.setDebug(true); + joblist::JobStep::jobstepThreadPool.invoke( + threadpool::ThreadPoolMonitor(&joblist::JobStep::jobstepThreadPool)); + } + + int serverThreads = rm_->getEmServerThreads(); + int maxPct = rm_->getEmMaxPct(); + int pauseSeconds = rm_->getEmSecondsBetweenMemChecks(); + int priority = rm_->getEmPriority(); + + FEMsgHandler::threadPool.setMaxThreads(serverThreads); + FEMsgHandler::threadPool.setName("FEMsgHandler"); + + if (maxPct > 0) + { + // Defined in rssmonfcn.cpp + exemgr::startRssMon(maxPct, pauseSeconds); + } + + setpriority(PRIO_PROCESS, 0, priority); + + std::string teleServerHost(rm_->getConfig()->getConfig("QueryTele", "Host")); + + if (!teleServerHost.empty()) + { + int teleServerPort = toInt(rm_->getConfig()->getConfig("QueryTele", "Port")); + + if (teleServerPort > 0) + { + teleServerParms_ = querytele::QueryTeleServerParms(teleServerHost, teleServerPort); + } + } + + std::cout << "Starting ExeMgr: st = " << serverThreads << ", qs = " << rm_->getEmExecQueueSize() + << ", mx = " << maxPct << ", cf = " << rm_->getConfig()->configFile() << std::endl; + + { + BRM::DBRM* dbrm = new BRM::DBRM(); + dbrm->setSystemQueryReady(true); + delete dbrm; + } + + threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); + exeMgrThreadPool.setName("ExeMgrServer"); + + if (rm_->getExeMgrThreadPoolDebug() == "Y" || rm_->getExeMgrThreadPoolDebug() == "y") + { + exeMgrThreadPool.setDebug(true); + exeMgrThreadPool.invoke(threadpool::ThreadPoolMonitor(&exeMgrThreadPool)); + } + + // Load statistics. + try + { + statistics::StatisticsManager::instance()->loadFromFile(); + } + catch (...) + { + std::cerr << "Cannot load statistics from file " << std::endl; + } + + for (;;) + { + messageqcpp::IOSocket ios; + ios = mqs->accept(); + exeMgrThreadPool.invoke(exemgr::SQLFrontSessionThread(ios, dec_, rm_)); + } + + exeMgrThreadPool.wait(); + + return 0; +} +} // namespace exemgr diff --git a/primitives/primproc/serviceexemgr.h b/primitives/primproc/serviceexemgr.h new file mode 100644 index 000000000..606ae48db --- /dev/null +++ b/primitives/primproc/serviceexemgr.h @@ -0,0 +1,348 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" + +namespace exemgr +{ + class Opt + { + public: + int m_debug; + bool m_e; + bool m_fg; + Opt() : m_debug(0), m_e(false), m_fg(false) {}; + Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) + { + int c; + while ((c = getopt(argc, argv, "edf")) != EOF) + { + switch (c) + { + case 'd': m_debug++; break; + + case 'e': m_e = true; break; + + case 'f': m_fg = true; break; + + case '?': + default: break; + } + } + } + int getDebugLevel() const + { + return m_debug; + } + }; + + class ServiceExeMgr : public Service, public Opt + { + using SessionMemMap_t = std::map; + using ThreadCntPerSessionMap_t = std::map; + protected: + void log(logging::LOG_TYPE type, const std::string& str) + { + logging::LoggingID logid(16); + logging::Message::Args args; + logging::Message message(8); + args.add(strerror(errno)); + message.format(args); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(type, message, logid); + } + + public: + ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) + { + bool runningWithExeMgr = true; + rm_ = joblist::ResourceManager::instance(runningWithExeMgr); + } + ServiceExeMgr(const Opt& opt, config::Config* aConfig) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) + { + bool runningWithExeMgr = true; + rm_ = joblist::ResourceManager::instance(runningWithExeMgr, aConfig); + } + void LogErrno() override + { + log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); + } + void ParentLogChildMessage(const std::string& str) override + { + log(logging::LOG_TYPE_INFO, str); + } + int Child() override; + int Run() + { + return m_fg ? Child() : RunForking(); + } + static const constexpr unsigned logDefaultMsg = logging::M0000; + static const constexpr unsigned logDbProfStartStatement = logging::M0028; + static const constexpr unsigned logDbProfEndStatement = logging::M0029; + static const constexpr unsigned logStartSql = logging::M0041; + static const constexpr unsigned logEndSql = logging::M0042; + static const constexpr unsigned logRssTooBig = logging::M0044; + static const constexpr unsigned logDbProfQueryStats = logging::M0047; + static const constexpr unsigned logExeMgrExcpt = logging::M0055; + // If any flags other than the table mode flags are set, produce output to screeen + static const constexpr uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & + ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); + logging::Logger& getLogger() + { + return msgLog_; + } + void updateSessionMap(const size_t pct) + { + std::lock_guard lk(sessionMemMapMutex_); + + for (auto mapIter = sessionMemMap_.begin(); mapIter != sessionMemMap_.end(); ++mapIter) + { + if (pct > mapIter->second) + { + mapIter->second = pct; + } + } + } + ThreadCntPerSessionMap_t& getThreadCntPerSessionMap() + { + return threadCntPerSessionMap_; + } + std::mutex& getThreadCntPerSessionMapMutex() + { + return threadCntPerSessionMapMutex_; + } + void initMaxMemPct(uint32_t sessionId) + { + // WIP + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter == sessionMemMap_.end()) + { + sessionMemMap_[sessionId] = 0; + } + else + { + mapIter->second = 0; + } + } + } + uint64_t getMaxMemPct(const uint32_t sessionId) + { + uint64_t maxMemoryPct = 0; + // WIP + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter != sessionMemMap_.end()) + { + maxMemoryPct = (uint64_t)mapIter->second; + } + } + + return maxMemoryPct; + } + void deleteMaxMemPct(uint32_t sessionId) + { + if (sessionId < 0x80000000) + { + std::lock_guard lk(sessionMemMapMutex_); + auto mapIter = sessionMemMap_.find(sessionId); + + if (mapIter != sessionMemMap_.end()) + { + sessionMemMap_.erase(sessionId); + } + } + } + //...Increment the number of threads using the specified sessionId + void incThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex_); + auto mapIter = threadCntPerSessionMap_.find(sessionId); + + if (mapIter == threadCntPerSessionMap_.end()) + threadCntPerSessionMap_.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); + else + mapIter->second++; + } + //...Decrement the number of threads using the specified sessionId. + //...When the thread count for a sessionId reaches 0, the corresponding + //...CalpontSystemCatalog objects are deleted. + //...The user query and its associated catalog query have a different + //...session Id where the highest bit is flipped. + //...The object with id(sessionId | 0x80000000) cannot be removed before + //...user query session completes because the sysdata may be used for + //...debugging/stats purpose, such as result graph, etc. + void decThreadCntPerSession(uint32_t sessionId) + { + std::lock_guard lk(threadCntPerSessionMapMutex_); + auto mapIter = threadCntPerSessionMap_.find(sessionId); + + if (mapIter != threadCntPerSessionMap_.end()) + { + if (--mapIter->second == 0) + { + threadCntPerSessionMap_.erase(mapIter); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); + execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); + } + } + } + ActiveStatementCounter* getStatementsRunningCount() + { + return statementsRunningCount_; + } + joblist::DistributedEngineComm* getDec() + { + return dec_; + } + int toInt(const std::string& val) + { + if (val.length() == 0) + return -1; + + return static_cast(config::Config::fromText(val)); + } + const std::string prettyPrintMiniInfo(const std::string& in); + + const std::string timeNow() + { + time_t outputTime = time(0); + struct tm ltm; + char buf[32]; // ctime(3) says at least 26 + size_t len = 0; + asctime_r(localtime_r(&outputTime, <m), buf); + len = strlen(buf); + + if (len > 0) + --len; + + if (buf[len] == '\n') + buf[len] = 0; + + return buf; + } + querytele::QueryTeleServerParms& getTeleServerParms() + { + return teleServerParms_; + } + joblist::ResourceManager& getRm() + { + return *rm_; + } + private: + void setupSignalHandlers(); + int8_t setupCwd() + { + std::string workdir = rm_->getScWorkingDir(); + int8_t rc = chdir(workdir.c_str()); + + if (rc < 0 || access(".", W_OK) != 0) + rc = chdir("/tmp"); + + return (rc < 0) ? -5 : rc; + } + int setupResources() + { + struct rlimit rlim; + + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -1; + } + rlim.rlim_cur = rlim.rlim_max = 65536; + + if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -2; + } + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) + { + return -3; + } + if (rlim.rlim_cur != 65536) + { + return -4; + } + return 0; + } + + logging::Logger msgLog_; + SessionMemMap_t sessionMemMap_; // track memory% usage during a query + std::mutex sessionMemMapMutex_; + //...The FrontEnd may establish more than 1 connection (which results in + // more than 1 ExeMgr thread) per session. These threads will share + // the same CalpontSystemCatalog object for that session. Here, we + // define a std::map to track how many threads are sharing each session, so + // that we know when we can safely delete a CalpontSystemCatalog object + // shared by multiple threads per session. + ThreadCntPerSessionMap_t threadCntPerSessionMap_; + std::mutex threadCntPerSessionMapMutex_; + ActiveStatementCounter* statementsRunningCount_; + joblist::DistributedEngineComm* dec_; + joblist::ResourceManager* rm_; + // Its attributes are set in Child() + querytele::QueryTeleServerParms teleServerParms_; + }; + extern ServiceExeMgr* globServiceExeMgr; +} \ No newline at end of file diff --git a/primitives/primproc/sqlfrontsessionthread.cpp b/primitives/primproc/sqlfrontsessionthread.cpp new file mode 100644 index 000000000..3b5094e7a --- /dev/null +++ b/primitives/primproc/sqlfrontsessionthread.cpp @@ -0,0 +1,985 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "sqlfrontsessionthread.h" + +namespace exemgr +{ + uint64_t SQLFrontSessionThread::getMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->getMaxMemPct(sessionId); + } + void SQLFrontSessionThread::deleteMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->deleteMaxMemPct(sessionId); + } + void SQLFrontSessionThread::incThreadCntPerSession(uint32_t sessionId) + { + return globServiceExeMgr->incThreadCntPerSession(sessionId); + } + void SQLFrontSessionThread::decThreadCntPerSession(uint32_t sessionId) + { + return globServiceExeMgr->decThreadCntPerSession(sessionId); + } + + void SQLFrontSessionThread::initMaxMemPct(uint32_t sessionId) + { + return globServiceExeMgr->initMaxMemPct(sessionId); + } + //...Get and log query stats to specified output stream + const std::string SQLFrontSessionThread::formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned) + { + std::ostringstream os; + + // Get stats if not already acquired for current query + if (!fStatsRetrieved) + { + if (wantExtendedStats) + { + // wait for the ei data to be written by another thread (brain-dead) + struct timespec req = {0, 250000}; // 250 usec + nanosleep(&req, 0); + } + + // Get % memory usage during current query for sessionId + jl->querySummary(wantExtendedStats); + fStats = jl->queryStats(); + fStats.fMaxMemPct = getMaxMemPct(fStats.fSessionID); + fStats.fRows = rowsReturned; + fStatsRetrieved = true; + } + + std::string queryMode; + queryMode = (vtableModeOn ? "Distributed" : "Standard"); + + // Log stats to specified output stream + os << label << ": MaxMemPct-" << fStats.fMaxMemPct << "; NumTempFiles-" << fStats.fNumFiles + << "; TempFileSpace-" << roundBytes(fStats.fFileBytes) << "; ApproxPhyI/O-" << fStats.fPhyIO + << "; CacheI/O-" << fStats.fCacheIO << "; BlocksTouched-" << fStats.fMsgRcvCnt; + + if (includeNewLine) + os << std::endl << " "; // insert line break + else + os << "; "; // continue without line break + + os << "PartitionBlocksEliminated-" << fStats.fCPBlocksSkipped << "; MsgBytesIn-" + << roundBytes(fStats.fMsgBytesIn) << "; MsgBytesOut-" << roundBytes(fStats.fMsgBytesOut) << "; Mode-" + << queryMode; + + return os.str(); + } + + //... Round off to human readable format (KB, MB, or GB). + const std::string SQLFrontSessionThread::roundBytes(uint64_t value) const + { + const char* units[] = {"B", "KB", "MB", "GB", "TB"}; + uint64_t i = 0, up = 0; + uint64_t roundedValue = value; + + while (roundedValue > 1024 && i < 4) + { + up = (roundedValue & 512); + roundedValue /= 1024; + i++; + } + + if (up) + roundedValue++; + + std::ostringstream oss; + oss << roundedValue << units[i]; + return oss.str(); + } + + //...Round off to nearest (1024*1024) MB + uint64_t SQLFrontSessionThread::roundMB(uint64_t value) const + { + uint64_t roundedValue = value >> 20; + + if (value & 524288) + roundedValue++; + + return roundedValue; + } + + void SQLFrontSessionThread::setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms) + { + for (execplan::CalpontSelectExecutionPlan::RMParmVec::const_iterator it = parms.begin(); + it != parms.end(); ++it) + { + switch (it->id) + { + case execplan::PMSMALLSIDEMEMORY: + { + globServiceExeMgr->getRm().addHJPmMaxSmallSideMap(it->sessionId, it->value); + break; + } + + case execplan::UMSMALLSIDEMEMORY: + { + globServiceExeMgr->getRm().addHJUmMaxSmallSideMap(it->sessionId, it->value); + break; + } + + default:; + } + } + } + + void SQLFrontSessionThread::buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc) + { + const execplan::CalpontSelectExecutionPlan::ColumnMap& colMap = csep.columnMap(); + std::string schemaName; + + for (auto it = colMap.begin(); it != colMap.end(); ++it) + { + const auto sc = dynamic_cast((it->second).get()); + + if (sc) + { + schemaName = sc->schemaName(); + + // only the first time a schema is got will actually query + // system catalog. System catalog keeps a schema name std::map. + // if a schema exists, the call getSchemaInfo returns without + // doing anything. + if (!schemaName.empty()) + csc->getSchemaInfo(schemaName); + } + } + + execplan::CalpontSelectExecutionPlan::SelectList::const_iterator subIt; + + for (subIt = csep.derivedTableList().begin(); subIt != csep.derivedTableList().end(); ++subIt) + { + buildSysCache(*(dynamic_cast(subIt->get())), csc); + } + } + + void SQLFrontSessionThread::writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg) + { + messageqcpp::ByteStream emsgBs; + messageqcpp::ByteStream tbs; + tbs << code; + fIos.write(tbs); + emsgBs << emsg; + fIos.write(emsgBs); + } + + void SQLFrontSessionThread::analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted) + { + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + messageqcpp::ByteStream::quadbyte qb; + execplan::MCSAnalyzeTableExecutionPlan caep; + + bs = fIos.read(); + caep.unserialize(bs); + + statementsRunningCount->incr(stmtCounted); + jl = joblist::JobListFactory::makeJobList(&caep, fRm, false, true); + + // Joblist is empty. + if (jl->status() == logging::statisticsJobListEmpty) + { + if (caep.traceOn()) + std::cout << "JobList is empty " << std::endl; + + jl.reset(); + bs.restart(); + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + statementsRunningCount->decr(stmtCounted); + return; + } + + if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) + { + std::cout << "fEc setup " << std::endl; + fEc->Setup(); + } + + if (jl->status() == 0) + { + std::string emsg; + + if (jl->putEngineComm(fEc) != 0) + throw std::runtime_error(jl->errMsg()); + } + else + { + throw std::runtime_error("ExeMgr: could not build a JobList!"); + } + + // Execute a joblist. + jl->doQuery(); + + FEMsgHandler msgHandler(jl, &fIos); + + msgHandler.start(); + auto rowCount = jl->projectTable(100, bs); + msgHandler.stop(); + + auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); + + if (caep.traceOn()) + std::cout << "Row count " << rowCount << std::endl; + + // Process `RowGroup`, increase an epoch and save statistics to the file. + auto* statisticsManager = statistics::StatisticsManager::instance(); + statisticsManager->analyzeColumnKeyTypes(outRG, caep.traceOn()); + statisticsManager->incEpoch(); + statisticsManager->saveToFile(); + + // Distribute statistics across all ExeMgr clients if possible. + statistics::StatisticsDistributor::instance()->distributeStatistics(); + + // Send the signal back to front-end. + bs.restart(); + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + statementsRunningCount->decr(stmtCounted); + } + + void SQLFrontSessionThread::analyzeTableHandleStats(messageqcpp::ByteStream& bs) + { + messageqcpp::ByteStream::quadbyte qb; +#ifdef DEBUG_STATISTICS + std::cout << "Get distributed statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + bs = fIos.read(); +#ifdef DEBUG_STATISTICS + std::cout << "Read the hash from statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + uint64_t dataHashRec; + bs >> dataHashRec; + + uint64_t dataHash = statistics::StatisticsManager::instance()->computeHashFromStats(); + // The stats are the same. + if (dataHash == dataHashRec) + { +#ifdef DEBUG_STATISTICS + std::cout << "The hash is the same as rec hash on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + return; + } + + bs.restart(); + qb = ANALYZE_TABLE_NEED_STATS; + bs << qb; + fIos.write(bs); + + bs.restart(); + bs = fIos.read(); +#ifdef DEBUG_STATISTICS + std::cout << "Read statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; +#endif + statistics::StatisticsManager::instance()->unserialize(bs); + statistics::StatisticsManager::instance()->saveToFile(); + +#ifdef DEBUG_STATISTICS + std::cout << "Write flag on ExeMgr(Client) to ExeMgr(Server)" << std::endl; +#endif + qb = ANALYZE_TABLE_SUCCESS; + bs << qb; + fIos.write(bs); + bs.reset(); + } + + void SQLFrontSessionThread::operator()() + { + messageqcpp::ByteStream bs, inbs; + execplan::CalpontSelectExecutionPlan csep; + csep.sessionID(0); + joblist::SJLP jl; + bool incSQLFrontSessionThreadCnt = true; + std::mutex jlMutex; + std::condition_variable jlCleanupDone; + int destructing = 0; + int gDebug = globServiceExeMgr->getDebugLevel(); + logging::Logger& msgLog = globServiceExeMgr->getLogger(); + + bool selfJoin = false; + bool tryTuples = false; + bool usingTuples = false; + bool stmtCounted = false; + auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); + + try + { + for (;;) + { + selfJoin = false; + tryTuples = false; + usingTuples = false; + + if (jl) + { + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + std::unique_lock scoped(jlMutex); + destructing++; + std::thread bgdtor( + [jl, &jlMutex, &jlCleanupDone, &destructing] + { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this runs first + bgdtor.detach(); + } + + bs = fIos.read(); + + if (bs.length() == 0) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### Got a close(1) for session id " << csep.sessionID() << std::endl; + + // connection closed by client + fIos.close(); + break; + } + else if (bs.length() < 4) // Not a CalpontSelectExecutionPlan + { + if (gDebug) + std::cout << "### Got a not-a-plan for session id " << csep.sessionID() << " with length " + << bs.length() << std::endl; + + fIos.close(); + break; + } + else if (bs.length() == 4) // possible tuple flag + { + messageqcpp::ByteStream::quadbyte qb; + bs >> qb; + + if (qb == 4) // UM wants new tuple i/f + { + if (gDebug) + std::cout << "### UM wants tuples" << std::endl; + + tryTuples = true; + // now wait for the CSEP... + bs = fIos.read(); + } + else if (qb == 5) // somebody wants stats + { + bs.restart(); + qb = statementsRunningCount->cur(); + bs << qb; + qb = statementsRunningCount->waiting(); + bs << qb; + fIos.write(bs); + fIos.close(); + break; + } + else if (qb == ANALYZE_TABLE_EXECUTE) + { + analyzeTableExecute(bs, jl, stmtCounted); + continue; + } + else if (qb == ANALYZE_TABLE_REC_STATS) + { + analyzeTableHandleStats(bs); + continue; + } + else + { + if (gDebug) + std::cout << "### Got a not-a-plan value " << qb << std::endl; + + fIos.close(); + break; + } + } + + new_plan: + try + { + csep.unserialize(bs); + } + catch (logging::IDBExcept& ex) + { + // We can get here on illegal function parameter data type, e.g. + // SELECT blob_column|1 FROM t1; + statementsRunningCount->decr(stmtCounted); + writeCodeAndError(ex.errorCode(), std::string(ex.what())); + continue; + } + + querytele::QueryTeleStats qts; + + if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) + { + qts.query_uuid = csep.uuid(); + qts.msg_type = querytele::QueryTeleStats::QT_START; + qts.start_time = querytele::QueryTeleClient::timeNowms(); + qts.query = csep.data(); + qts.session_id = csep.sessionID(); + qts.query_type = csep.queryType(); + qts.system_name = fOamCachePtr->getSystemName(); + qts.module_name = fOamCachePtr->getModuleName(); + qts.local_query = csep.localQuery(); + qts.schema_name = csep.schemaName(); + fTeleClient.postQueryTele(qts); + } + + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", got a CSEP" << std::endl; + + setRMParms(csep.rmParms()); + // Re-establish lost PP connections. + if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) + { + fEc->Setup(); + } + // @bug 1021. try to get schema cache for a come in query. + // skip system catalog queries. + if (!csep.isInternal()) + { + boost::shared_ptr csc = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(csep.sessionID()); + buildSysCache(csep, csc); + } + + // As soon as we have a session id for this thread, update the + // thread count per session; only do this once per thread. + // Mask 0x80000000 is for associate user query and csc query + if (incSQLFrontSessionThreadCnt) + { + // WIP + incThreadCntPerSession(csep.sessionID() | 0x80000000); + incSQLFrontSessionThreadCnt = false; + } + + bool needDbProfEndStatementMsg = false; + logging::Message::Args args; + std::string sqlText = csep.data(); + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + + // Initialize stats for this query, including + // init sessionMemMap entry for this session to 0 memory %. + // We will need this later for traceOn() or if we receive a + // table request with qb=3 (see below). This is also recorded + // as query start time. + initStats(csep.sessionID(), sqlText); + fStats.fQueryType = csep.queryType(); + + // Log start and end statement if tracing is enabled. Keep in + // mind the trace flag won't be set for system catalog queries. + if (csep.traceOn()) + { + args.reset(); + args.add((int)csep.statementID()); + args.add((int)csep.verID().currentScn); + args.add(sqlText); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfStartStatement, args, li); + needDbProfEndStatementMsg = true; + } + + // Don't log subsequent self joins after first. + if (selfJoin) + sqlText = ""; + + std::ostringstream oss; + oss << sqlText << "; |" << csep.schemaName() << "|"; + logging::SQLLogger sqlLog(oss.str(), li); + + statementsRunningCount->incr(stmtCounted); + + if (tryTuples) + { + try // @bug2244: try/catch around fIos.write() calls responding to makeTupleList + { + jl = joblist::JobListFactory::makeJobList(&csep, fRm, true, true); + // assign query stats + jl->queryStats(fStats); + + if ((jl->status()) == 0 && (jl->putEngineComm(fEc) == 0)) + { + usingTuples = true; + + // Tell the FE that we're sending tuples back, not TableBands + writeCodeAndError(0, "NOERROR"); + auto tjlp = dynamic_cast(jl.get()); + assert(tjlp); + messageqcpp::ByteStream tbs; + tbs << tjlp->getOutputRowGroup(); + fIos.write(tbs); + } + else + { + const std::string emsg = jl->errMsg(); + statementsRunningCount->decr(stmtCounted); + writeCodeAndError(jl->status(), emsg); + std::cerr << "ExeMgr: could not build a tuple joblist: " << emsg << std::endl; + continue; + } + } + catch (std::exception& ex) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: error writing makeJoblist " + "response; " + << ex.what(); + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: unknown error writing makeJoblist " + "response; "; + throw std::runtime_error(errMsg.str()); + } + + if (!usingTuples) + { + if (gDebug) + std::cout << "### UM wanted tuples but it didn't work out :-(" << std::endl; + } + else + { + if (gDebug) + std::cout << "### UM wanted tuples and we'll do our best;-)" << std::endl; + } + } + else + { + usingTuples = false; + jl = joblist::JobListFactory::makeJobList(&csep, fRm, false, true); + + if (jl->status() == 0) + { + std::string emsg; + + if (jl->putEngineComm(fEc) != 0) + throw std::runtime_error(jl->errMsg()); + } + else + { + throw std::runtime_error("ExeMgr: could not build a JobList!"); + } + } + + jl->doQuery(); + + execplan::CalpontSystemCatalog::OID tableOID; + bool swallowRows = false; + joblist::DeliveredTableMap tm; + uint64_t totalBytesSent = 0; + uint64_t totalRowCount = 0; + + // Project each table as the FE asks for it + for (;;) + { + bs = fIos.read(); + + if (bs.length() == 0) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### Got a close(2) for session id " << csep.sessionID() << std::endl; + + break; + } + + if (gDebug && bs.length() > 4) + std::cout << "### For session id " << csep.sessionID() << ", got too many bytes = " << bs.length() + << std::endl; + + // TODO: Holy crud! Can this be right? + //@bug 1444 Yes, if there is a self-join + if (bs.length() > 4) + { + selfJoin = true; + statementsRunningCount->decr(stmtCounted); + goto new_plan; + } + + assert(bs.length() == 4); + + messageqcpp::ByteStream::quadbyte qb; + + try // @bug2244: try/catch around fIos.write() calls responding to qb command + { + bs >> qb; + + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", got a command = " << qb + << std::endl; + + if (qb == 0) + { + // No more tables, query is done + break; + } + else if (qb == 1) + { + // super-secret flag indicating that the UM is going to scarf down all the rows in the + // query. + swallowRows = true; + tm = jl->deliveredTables(); + continue; + } + else if (qb == 2) + { + // UM just wants any table + assert(swallowRows); + auto iter = tm.begin(); + + if (iter == tm.end()) + { + if (gDebug > 1 || (gDebug && !csep.isInternal())) + std::cout << "### For session id " << csep.sessionID() << ", returning end flag" + << std::endl; + + bs.restart(); + bs << (messageqcpp::ByteStream::byte)1; + fIos.write(bs); + continue; + } + + tableOID = iter->first; + } + else if (qb == 3) // special option-UM wants job stats std::string + { + std::string statsString; + + // Log stats std::string to be sent back to front end + statsString = formatQueryStats( + jl, "Query Stats", false, + !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), + (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), totalRowCount); + + bs.restart(); + bs << statsString; + + if ((csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG) != 0) + { + bs << jl->extendedInfo(); + bs << globServiceExeMgr->prettyPrintMiniInfo(jl->miniInfo()); + } + else + { + std::string empty; + bs << empty; + bs << empty; + } + + // send stats to connector for inserting to the querystats table + fStats.serialize(bs); + fIos.write(bs); + continue; + } + // for table mode handling + else if (qb == 4) + { + statementsRunningCount->decr(stmtCounted); + bs = fIos.read(); + goto new_plan; + } + else // (qb > 3) + { + // Return table bands for the requested tableOID + tableOID = static_cast(qb); + } + } + catch (std::exception& ex) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: error writing qb response " + "for qb cmd " + << qb << "; " << ex.what(); + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + errMsg << "ExeMgr: unknown error writing qb response " + "for qb cmd " + << qb; + throw std::runtime_error(errMsg.str()); + } + + if (swallowRows) + tm.erase(tableOID); + + FEMsgHandler msgHandler(jl, &fIos); + + if (tableOID == 100) + msgHandler.start(); + + //...Loop serializing table bands projected for the tableOID + for (;;) + { + uint32_t rowCount; + + rowCount = jl->projectTable(tableOID, bs); + + msgHandler.stop(); + + if (jl->status()) + { + const auto errInfo = logging::IDBErrorInfo::instance(); + + if (jl->errMsg().length() != 0) + bs << jl->errMsg(); + else + bs << errInfo->errorMsg(jl->status()); + } + + try // @bug2244: try/catch around fIos.write() calls projecting rows + { + if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) + { + // Skip the write to the front end until the last empty band. Used to time queries + // through without any front end waiting. + if (tableOID < 3000 || rowCount == 0) + fIos.write(bs); + } + else + { + fIos.write(bs); + } + } + catch (std::exception& ex) + { + msgHandler.stop(); + std::ostringstream errMsg; + errMsg << "ExeMgr: error projecting rows " + "for tableOID: " + << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount << "; " + << ex.what(); + jl->abort(); + + while (rowCount) + rowCount = jl->projectTable(tableOID, bs); + + if (tableOID == 100 && msgHandler.aborted()) + { + /* TODO: modularize the cleanup code, as well as + * the rest of this fcn */ + + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + fIos.close(); + return; + } + + // std::cout << "connection drop\n"; + throw std::runtime_error(errMsg.str()); + } + catch (...) + { + std::ostringstream errMsg; + msgHandler.stop(); + errMsg << "ExeMgr: unknown error projecting rows " + "for tableOID: " + << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount; + jl->abort(); + + while (rowCount) + rowCount = jl->projectTable(tableOID, bs); + + throw std::runtime_error(errMsg.str()); + } + + totalRowCount += rowCount; + totalBytesSent += bs.length(); + + if (rowCount == 0) + { + msgHandler.stop(); + // No more bands, table is done + bs.reset(); + + // @bug 2083 decr active statement count here for table mode. + if (!usingTuples) + statementsRunningCount->decr(stmtCounted); + + break; + } + else + { + bs.restart(); + } + } // End of loop to project and serialize table bands for a table + } // End of loop to process tables + + // @bug 828 + if (csep.traceOn()) + jl->graph(csep.sessionID()); + + if (needDbProfEndStatementMsg) + { + std::string ss; + std::ostringstream prefix; + prefix << "ses:" << csep.sessionID() << " Query Totals"; + + // Log stats std::string to standard out + ss = formatQueryStats(jl, prefix.str(), true, + !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), + (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), + totalRowCount); + //@Bug 1306. Added timing info for real time tracking. + std::cout << ss << " at " << globServiceExeMgr->timeNow() << std::endl; + + // log query stats to debug log file + args.reset(); + args.add((int)csep.statementID()); + args.add(fStats.fMaxMemPct); + args.add(fStats.fNumFiles); + args.add(fStats.fFileBytes); // log raw byte count instead of MB + args.add(fStats.fPhyIO); + args.add(fStats.fCacheIO); + args.add(fStats.fMsgRcvCnt); + args.add(fStats.fMsgBytesIn); + args.add(fStats.fMsgBytesOut); + args.add(fStats.fCPBlocksSkipped); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfQueryStats, args, li); + //@bug 1327 + deleteMaxMemPct(csep.sessionID()); + // Calling reset here, will cause joblist destructor to be + // called, which "joins" the threads. We need to do that + // here to make sure all syslogging from all the threads + // are complete; and that our logDbProfEndStatement will + // appear "last" in the syslog for this SQL statement. + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + int stmtID = csep.statementID(); + std::unique_lock scoped(jlMutex); + destructing++; + std::thread bgdtor( + [jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing, &msgLog] + { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + logging::Message::Args args; + args.add(stmtID); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfEndStatement, args, li); + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this happens first + bgdtor.detach(); + } + else + // delete sessionMemMap entry for this session's memory % use + deleteMaxMemPct(csep.sessionID()); + + std::string endtime(globServiceExeMgr->timeNow()); + + if ((csep.traceFlags() & globServiceExeMgr->flagsWantOutput) && (csep.sessionID() < 0x80000000)) + { + std::cout << "For session " << csep.sessionID() << ": " << totalBytesSent << " bytes sent back at " + << endtime << std::endl; + + // @bug 663 - Implemented caltraceon(16) to replace the + // $FIFO_SINK compiler definition in pColStep. + // This option consumes rows in the project steps. + if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS4) + { + std::cout << std::endl; + std::cout << "**** No data returned to DM. Rows consumed " + "in ProjectSteps - caltrace(16) is on (FIFO_SINK)." + " ****" + << std::endl; + std::cout << std::endl; + } + else if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) + { + std::cout << std::endl; + std::cout << "**** No data returned to DM - caltrace(8) is " + "on (SWALLOW_ROWS_EXEMGR). ****" + << std::endl; + std::cout << std::endl; + } + } + + statementsRunningCount->decr(stmtCounted); + + if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) + { + qts.msg_type = querytele::QueryTeleStats::QT_SUMMARY; + qts.max_mem_pct = fStats.fMaxMemPct; + qts.num_files = fStats.fNumFiles; + qts.phy_io = fStats.fPhyIO; + qts.cache_io = fStats.fCacheIO; + qts.msg_rcv_cnt = fStats.fMsgRcvCnt; + qts.cp_blocks_skipped = fStats.fCPBlocksSkipped; + qts.msg_bytes_in = fStats.fMsgBytesIn; + qts.msg_bytes_out = fStats.fMsgBytesOut; + qts.rows = totalRowCount; + qts.end_time = querytele::QueryTeleClient::timeNowms(); + qts.session_id = csep.sessionID(); + qts.query_type = csep.queryType(); + qts.query = csep.data(); + qts.system_name = fOamCachePtr->getSystemName(); + qts.module_name = fOamCachePtr->getModuleName(); + qts.local_query = csep.localQuery(); + fTeleClient.postQueryTele(qts); + } + } + + // Release CSC object (for sessionID) that was added by makeJobList() + // Mask 0x80000000 is for associate user query and csc query. + // (actual joblist destruction happens at the top of this loop) + decThreadCntPerSession(csep.sessionID() | 0x80000000); + } + catch (std::exception& ex) + { + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + std::cerr << "### ExeMgr ses:" << csep.sessionID() << " caught: " << ex.what() << std::endl; + logging::Message::Args args; + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + args.add(ex.what()); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logExeMgrExcpt, args, li); + fIos.close(); + } + catch (...) + { + decThreadCntPerSession(csep.sessionID() | 0x80000000); + statementsRunningCount->decr(stmtCounted); + std::cerr << "### Exception caught!" << std::endl; + logging::Message::Args args; + logging::LoggingID li(16, csep.sessionID(), csep.txnID()); + args.add("ExeMgr caught unknown exception"); + msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logExeMgrExcpt, args, li); + fIos.close(); + } + + // make sure we don't leave scope while joblists are being destroyed + std::unique_lock scoped(jlMutex); + while (destructing > 0) + jlCleanupDone.wait(scoped); +} +}; // namespace exemgr \ No newline at end of file diff --git a/primitives/primproc/sqlfrontsessionthread.h b/primitives/primproc/sqlfrontsessionthread.h new file mode 100644 index 000000000..ca605ea79 --- /dev/null +++ b/primitives/primproc/sqlfrontsessionthread.h @@ -0,0 +1,131 @@ +/* Copyright (C) 2022 Mariadb Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include +#include +#include +#include + +#undef root_name +#include + +#include "calpontselectexecutionplan.h" +#include "mcsanalyzetableexecutionplan.h" +#include "activestatementcounter.h" +#include "distributedenginecomm.h" +#include "resourcemanager.h" +#include "configcpp.h" +#include "queryteleserverparms.h" +#include "iosocket.h" +#include "joblist.h" +#include "joblistfactory.h" +#include "oamcache.h" +#include "simplecolumn.h" +#include "bytestream.h" +#include "telestats.h" +#include "messageobj.h" +#include "messagelog.h" +#include "sqllogger.h" +#include "femsghandler.h" +#include "idberrorinfo.h" +#include "MonitorProcMem.h" +#include "liboamcpp.h" +#include "crashtrace.h" +#include "service.h" + +#include +#include +#include + +#include "dbrm.h" + +#include "mariadb_my_sys.h" +#include "statistics.h" +#include "serviceexemgr.h" + +namespace exemgr +{ + class SQLFrontSessionThread + { + public: + SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, + joblist::ResourceManager* rm) + : fIos(ios) + , fEc(ec) + , fRm(rm) + , fStatsRetrieved(false) + , fTeleClient(globServiceExeMgr->getTeleServerParms()) + , fOamCachePtr(oam::OamCache::makeOamCache()) + { + } + + private: + messageqcpp::IOSocket fIos; + joblist::DistributedEngineComm* fEc; + joblist::ResourceManager* fRm; + querystats::QueryStats fStats; + + // Variables used to store return stats + bool fStatsRetrieved; + + querytele::QueryTeleClient fTeleClient; + + oam::OamCache* fOamCachePtr; // this ptr is copyable... + + //...Reinitialize stats for start of a new query + void initStats(uint32_t sessionId, std::string& sqlText) + { + initMaxMemPct(sessionId); + + fStats.reset(); + fStats.setStartTime(); + fStats.fSessionID = sessionId; + fStats.fQuery = sqlText; + fStatsRetrieved = false; + } + //...Get % memory usage during latest query for sesssionId. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static uint64_t getMaxMemPct(uint32_t sessionId); + //...Delete sessionMemMap entry for the specified session's memory % use. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void deleteMaxMemPct(uint32_t sessionId); + //...Get and log query stats to specified output stream + const std::string formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); + static void incThreadCntPerSession(uint32_t sessionId); + static void decThreadCntPerSession(uint32_t sessionId); + //...Init sessionMemMap entry for specified session to 0 memory %. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void initMaxMemPct(uint32_t sessionId); + //... Round off to human readable format (KB, MB, or GB). + const std::string roundBytes(uint64_t value) const; + void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); + void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc); + void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); + void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); + void analyzeTableHandleStats(messageqcpp::ByteStream& bs); + uint64_t roundMB(uint64_t value) const; + public: + void operator()(); + }; +} \ No newline at end of file diff --git a/utils/common/spinlock.h b/utils/common/spinlock.h index 2549230c7..8fab8661f 100644 --- a/utils/common/spinlock.h +++ b/utils/common/spinlock.h @@ -2,6 +2,10 @@ #include +// To be refactored. There are two issues with this implentation. +// 1. It can be replaced with a lock despite the memory_order_acquire/release mem order. Needs +// std::atomic_flag instead. +// 2. https://www.realworldtech.com/forum/?threadid=189711&curpostid=189723 namespace utils { inline void getSpinlock(std::atomic& lock) @@ -23,4 +27,29 @@ inline void releaseSpinlock(std::atomic& lock) lock.store(false, std::memory_order_release); } +// c++20 offers a combination of wait/notify methods but +// I prefer to use a simpler version of uspace spin lock. +class USpaceSpinLock +{ + public: + USpaceSpinLock(std::atomic_flag& flag) : flag_(flag) + { + while (flag_.test_and_set(std::memory_order_acquire)) + { + ; + } + }; + ~USpaceSpinLock() + { + release(); + }; + inline void release() + { + flag_.clear(std::memory_order_release); + } + + private: + std::atomic_flag& flag_; +}; + } // namespace utils diff --git a/utils/joiner/joiner.h b/utils/joiner/joiner.h index e61370dd6..58661489b 100644 --- a/utils/joiner/joiner.h +++ b/utils/joiner/joiner.h @@ -20,41 +20,11 @@ #include #include #include -#ifdef _MSC_VER -#include -#else + #include -#endif #include "../common/simpleallocator.h" - -#ifndef _HASHFIX_ -#define _HASHFIX_ -#ifndef __LP64__ -#if __GNUC__ == 4 && __GNUC_MINOR__ < 2 -// This is needed for /usr/include/c++/4.1.1/tr1/functional on 32-bit compiles -// tr1_hashtable_define_trivial_hash(long long unsigned int); -namespace std -{ -namespace tr1 -{ -template <> -struct hash : public std::unary_function -{ - std::size_t operator()(long long unsigned int val) const - { - return static_cast(val); - } -}; -} // namespace tr1 -} // namespace std -#endif -#endif -#endif - -#define NO_DATALISTS #include "../joblist/elementtype.h" -#undef NO_DATALISTS namespace joiner { From 7453db6b02cf65d974704fedb51fdbfeea664174 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 4 Apr 2022 12:50:42 +0000 Subject: [PATCH 42/55] MCOL-5001 This patch removes ExeMgr traces --- CMakeLists.txt | 1 - debian/mariadb-plugin-columnstore.install | 1 - exemgr/CMakeLists.txt | 20 - exemgr/activestatementcounter.cpp | 59 -- exemgr/activestatementcounter.h | 64 -- exemgr/exemgr.cpp | 102 -- exemgr/femsghandler.cpp | 143 --- exemgr/femsghandler.h | 47 - exemgr/rssmonfcn.cpp | 69 -- exemgr/rssmonfcn.h | 36 - exemgr/serviceexemgr.cpp | 475 --------- exemgr/serviceexemgr.h | 343 ------ exemgr/sqlfrontsessionthread.cpp | 985 ------------------ exemgr/sqlfrontsessionthread.h | 131 --- oam/install_scripts/CMakeLists.txt | 14 +- .../columnstore-post-install.in | 26 +- .../columnstore-pre-uninstall.in | 14 +- .../mariadb-columnstore-start.sh.in | 1 - .../mariadb-columnstore-stop.sh | 1 - oam/install_scripts/mcs-exemgr.service.in | 23 - .../mcs-writeengineserver.service.in | 6 +- 21 files changed, 30 insertions(+), 2531 deletions(-) delete mode 100644 exemgr/CMakeLists.txt delete mode 100644 exemgr/activestatementcounter.cpp delete mode 100644 exemgr/activestatementcounter.h delete mode 100644 exemgr/exemgr.cpp delete mode 100644 exemgr/femsghandler.cpp delete mode 100644 exemgr/femsghandler.h delete mode 100644 exemgr/rssmonfcn.cpp delete mode 100644 exemgr/rssmonfcn.h delete mode 100644 exemgr/serviceexemgr.cpp delete mode 100644 exemgr/serviceexemgr.h delete mode 100644 exemgr/sqlfrontsessionthread.cpp delete mode 100644 exemgr/sqlfrontsessionthread.h delete mode 100644 oam/install_scripts/mcs-exemgr.service.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 1be5206fd..a2fcc7954 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,7 +343,6 @@ ADD_SUBDIRECTORY(dbcon/ddlpackage) ADD_SUBDIRECTORY(dbcon/ddlpackageproc) ADD_SUBDIRECTORY(dbcon/dmlpackage) ADD_SUBDIRECTORY(dbcon/dmlpackageproc) -ADD_SUBDIRECTORY(exemgr) ADD_SUBDIRECTORY(ddlproc) ADD_SUBDIRECTORY(dmlproc) ADD_SUBDIRECTORY(oamapps) diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index 20241ff57..a921e36f2 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -105,7 +105,6 @@ usr/share/columnstore/mariadb-columnstore.service usr/share/columnstore/mcs-controllernode.service usr/share/columnstore/mcs-ddlproc.service usr/share/columnstore/mcs-dmlproc.service -usr/share/columnstore/mcs-exemgr.service usr/share/columnstore/mcs-loadbrm.service usr/share/columnstore/mcs-primproc.service usr/share/columnstore/mcs-storagemanager.service diff --git a/exemgr/CMakeLists.txt b/exemgr/CMakeLists.txt deleted file mode 100644 index e0d17d205..000000000 --- a/exemgr/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ - -# include_directories( ${ENGINE_COMMON_INCLUDES} ) - - -########### next target ############### - -# set(ExeMgr_SRCS serviceexemgr.cpp sqlfrontsessionthread.cpp rssmonfcn.cpp exemgr.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) - -# add_executable(ExeMgr ${ExeMgr_SRCS}) - -# target_link_libraries(ExeMgr ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} cacheutils threadpool) - -# target_include_directories(ExeMgr PRIVATE ${Boost_INCLUDE_DIRS}) - -# install(TARGETS ExeMgr DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) - - -########### install files ############### - - diff --git a/exemgr/activestatementcounter.cpp b/exemgr/activestatementcounter.cpp deleted file mode 100644 index b25e61549..000000000 --- a/exemgr/activestatementcounter.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: activestatementcounter.cpp 940 2013-01-21 14:11:31Z rdempsey $ -// - -#include -#include -using namespace boost; - -#include "activestatementcounter.h" - -void ActiveStatementCounter::incr(bool& counted) -{ - if (counted) - return; - - counted = true; - boost::mutex::scoped_lock lk(fMutex); - - if (upperLimit > 0) - while (fStatementCount >= upperLimit) - { - fStatementsWaiting++; - condvar.wait(lk); - --fStatementsWaiting; - } - - fStatementCount++; -} - -void ActiveStatementCounter::decr(bool& counted) -{ - if (!counted) - return; - - counted = false; - boost::mutex::scoped_lock lk(fMutex); - - if (fStatementCount == 0) - return; - - --fStatementCount; - condvar.notify_one(); -} diff --git a/exemgr/activestatementcounter.h b/exemgr/activestatementcounter.h deleted file mode 100644 index 87e7163aa..000000000 --- a/exemgr/activestatementcounter.h +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: activestatementcounter.h 940 2013-01-21 14:11:31Z rdempsey $ -// -/** @file */ - -#pragma once - -#include - -#include -#include - -#include "vss.h" - -class ActiveStatementCounter -{ - public: - ActiveStatementCounter(uint32_t limit) : fStatementCount(0), upperLimit(limit), fStatementsWaiting(0) - { - } - - virtual ~ActiveStatementCounter() - { - } - - void incr(bool& counted); - void decr(bool& counted); - uint32_t cur() const - { - return fStatementCount; - } - uint32_t waiting() const - { - return fStatementsWaiting; - } - - private: - ActiveStatementCounter(const ActiveStatementCounter& rhs); - ActiveStatementCounter& operator=(const ActiveStatementCounter& rhs); - - uint32_t fStatementCount; - uint32_t upperLimit; - uint32_t fStatementsWaiting; - boost::mutex fMutex; - boost::condition condvar; - BRM::VSS fVss; -}; - diff --git a/exemgr/exemgr.cpp b/exemgr/exemgr.cpp deleted file mode 100644 index fa1acec8d..000000000 --- a/exemgr/exemgr.cpp +++ /dev/null @@ -1,102 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2019 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/********************************************************************** - * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ - * - * - ***********************************************************************/ -/** - * @brief execution plan manager main program - * - * This is the ?main? program for dealing with execution plans and - * result sets. It sits in a loop waiting for CalpontExecutionPlan - * from the FEP. It then passes the CalpontExecutionPlan to the - * JobListFactory from which a JobList is obtained. This is passed - * to to the Query Manager running in the real-time portion of the - * EC. The ExecutionPlanManager waits until the Query Manager - * returns a result set for the job list. These results are passed - * into the CalpontResultFactory, which outputs a CalpontResultSet. - * The ExecutionPlanManager passes the CalpontResultSet into the - * VendorResultFactory which produces a result set tailored to the - * specific DBMS front end in use. The ExecutionPlanManager then - * sends the VendorResultSet back to the Calpont Database Connector - * on the Front-End Processor where it is returned to the DBMS - * front-end. - */ -#include -#include -#include -#include - -#undef root_name -#include - -#include "calpontselectexecutionplan.h" -#include "mcsanalyzetableexecutionplan.h" -#include "activestatementcounter.h" -#include "distributedenginecomm.h" -#include "resourcemanager.h" -#include "configcpp.h" -#include "queryteleserverparms.h" -#include "iosocket.h" -#include "joblist.h" -#include "joblistfactory.h" -#include "oamcache.h" -#include "simplecolumn.h" -#include "bytestream.h" -#include "telestats.h" -#include "messageobj.h" -#include "messagelog.h" -#include "sqllogger.h" -#include "femsghandler.h" -#include "idberrorinfo.h" -#include "MonitorProcMem.h" -#include "liboamcpp.h" -#include "crashtrace.h" -#include "service.h" - -#include -#include -#include - -#include "dbrm.h" - -#include "mariadb_my_sys.h" -#include "statistics.h" -#include "serviceexemgr.h" - -int main(int argc, char* argv[]) -{ - opterr = 0; - exemgr::Opt opt(argc, argv); - - // Set locale language - setlocale(LC_ALL, ""); - setlocale(LC_NUMERIC, "C"); - - // This is unset due to the way we start it - // program_invocation_short_name = const_cast("ExeMgr"); - - // Initialize the charset library - MY_INIT(argv[0]); - // global ptr defined in serviceexemgr.cpp - exemgr::globServiceExeMgr = new exemgr::ServiceExeMgr(opt); - return exemgr::globServiceExeMgr->Run(); -} - diff --git a/exemgr/femsghandler.cpp b/exemgr/femsghandler.cpp deleted file mode 100644 index 0ce22519f..000000000 --- a/exemgr/femsghandler.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#include "messagequeue.h" -#include "iosocket.h" - -#include "femsghandler.h" - -using namespace std; -using namespace joblist; -using namespace messageqcpp; - -threadpool::ThreadPool FEMsgHandler::threadPool; - -namespace -{ -class Runner -{ - public: - Runner(FEMsgHandler* f) : target(f) - { - } - void operator()() - { - target->threadFcn(); - } - FEMsgHandler* target; -}; - -} // namespace - -FEMsgHandler::FEMsgHandler() : die(false), running(false), sawData(false), sock(NULL) -{ -} - -FEMsgHandler::FEMsgHandler(boost::shared_ptr j, IOSocket* s) - : die(false), running(false), sawData(false), jl(j) -{ - sock = s; - assert(sock); -} - -FEMsgHandler::~FEMsgHandler() -{ - stop(); - threadPool.join(thr); -} - -void FEMsgHandler::start() -{ - if (!running) - { - running = true; - thr = threadPool.invoke(Runner(this)); - } -} - -void FEMsgHandler::stop() -{ - die = true; - jl.reset(); -} - -void FEMsgHandler::setJobList(boost::shared_ptr j) -{ - jl = j; -} - -void FEMsgHandler::setSocket(IOSocket* i) -{ - sock = i; - assert(sock); -} - -/* Note, the next two fcns strongly depend on ExeMgr's current implementation. There's a - * good chance that if ExeMgr's table send loop is changed, these will need to be - * updated to match. - */ - -/* This is currently only called if InetStreamSocket::write() throws, implying - * a connection error. It might not make sense in other contexts. - */ -bool FEMsgHandler::aborted() -{ - if (sawData) - return true; - - boost::mutex::scoped_lock sl(mutex); - int err; - int connectionNum = sock->getConnectionNum(); - - err = InetStreamSocket::pollConnection(connectionNum, 1000); - - if (err == 1) - { - sawData = true; - return true; - } - - return false; -} - -void FEMsgHandler::threadFcn() -{ - int err = 0; - int connectionNum = sock->getConnectionNum(); - - /* This waits for the next readable event on sock. An abort is signaled - * by sending something (anything at the moment), then dropping the connection. - * This fcn exits on all other events. - */ - while (!die && err == 0) - { - boost::mutex::scoped_lock sl(mutex); - err = InetStreamSocket::pollConnection(connectionNum, 1000); - } - - if (err == 1) - sawData = true; // there's data to read, must be the abort signal - - if (!die && (err == 2 || err == 1)) - { - die = true; - jl->abort(); - jl.reset(); - } - - running = false; -} diff --git a/exemgr/femsghandler.h b/exemgr/femsghandler.h deleted file mode 100644 index baa91ad90..000000000 --- a/exemgr/femsghandler.h +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include "joblist.h" -#include "inetstreamsocket.h" -#include "threadpool.h" - -class FEMsgHandler -{ - public: - FEMsgHandler(); - FEMsgHandler(boost::shared_ptr, messageqcpp::IOSocket*); - virtual ~FEMsgHandler(); - - void start(); - void stop(); - void setJobList(boost::shared_ptr); - void setSocket(messageqcpp::IOSocket*); - bool aborted(); - - void threadFcn(); - - static threadpool::ThreadPool threadPool; - - private: - bool die, running, sawData; - messageqcpp::IOSocket* sock; - boost::shared_ptr jl; - boost::mutex mutex; - uint64_t thr; -}; diff --git a/exemgr/rssmonfcn.cpp b/exemgr/rssmonfcn.cpp deleted file mode 100644 index 8a9b62d83..000000000 --- a/exemgr/rssmonfcn.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright (C) 2022 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#include - -#include "rssmonfcn.h" -#include "serviceexemgr.h" - -namespace exemgr -{ - void RssMonFcn::operator()() const - { - logging::Logger& msgLog = globServiceExeMgr->getLogger(); - auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); - for (;;) - { - size_t rssMb = rss(); - size_t pct = rssMb * 100 / fMemTotal; - - if (pct > fMaxPct) - { - if (fMaxPct >= 95) - { - std::cerr << "Too much memory allocated!" << std::endl; - logging::Message::Args args; - args.add((int)pct); - args.add((int)fMaxPct); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logRssTooBig, args, logging::LoggingID(16)); - exit(1); - } - - if (statementsRunningCount->cur() == 0) - { - std::cerr << "Too much memory allocated!" << std::endl; - logging::Message::Args args; - args.add((int)pct); - args.add((int)fMaxPct); - msgLog.logMessage(logging::LOG_TYPE_WARNING, ServiceExeMgr::logRssTooBig, args, logging::LoggingID(16)); - exit(1); - } - - std::cerr << "Too much memory allocated, but stmts running" << std::endl; - } - - // Update sessionMemMap entries lower than current mem % use - globServiceExeMgr->updateSessionMap(pct); - - pause_(); - } - } - void startRssMon(size_t maxPct, int pauseSeconds) - { - new std::thread(RssMonFcn(maxPct, pauseSeconds)); - } -} // namespace \ No newline at end of file diff --git a/exemgr/rssmonfcn.h b/exemgr/rssmonfcn.h deleted file mode 100644 index 341038e91..000000000 --- a/exemgr/rssmonfcn.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright (C) 2022 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include - -#include "MonitorProcMem.h" - -namespace exemgr -{ -class RssMonFcn : public utils::MonitorProcMem -{ - public: - RssMonFcn(size_t maxPct, int pauseSeconds) : MonitorProcMem(maxPct, 0, 21, pauseSeconds) - { - } - - void operator()() const; -}; - -} // namespace \ No newline at end of file diff --git a/exemgr/serviceexemgr.cpp b/exemgr/serviceexemgr.cpp deleted file mode 100644 index 33ef3d8d2..000000000 --- a/exemgr/serviceexemgr.cpp +++ /dev/null @@ -1,475 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2019-22 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/********************************************************************** - * $Id: main.cpp 1000 2013-07-24 21:05:51Z pleblanc $ - * - * - ***********************************************************************/ -/** - * @brief execution plan manager main program - * - * This is the ?main? program for dealing with execution plans and - * result sets. It sits in a loop waiting for CalpontExecutionPlan - * from the FEP. It then passes the CalpontExecutionPlan to the - * JobListFactory from which a JobList is obtained. This is passed - * to to the Query Manager running in the real-time portion of the - * EC. The ExecutionPlanManager waits until the Query Manager - * returns a result set for the job list. These results are passed - * into the CalpontResultFactory, which outputs a CalpontResultSet. - * The ExecutionPlanManager passes the CalpontResultSet into the - * VendorResultFactory which produces a result set tailored to the - * specific DBMS front end in use. The ExecutionPlanManager then - * sends the VendorResultSet back to the Calpont Database Connector - * on the Front-End Processor where it is returned to the DBMS - * front-end. - */ -#include -#include -#include -#include - -#undef root_name -#include - -#include "calpontselectexecutionplan.h" -#include "mcsanalyzetableexecutionplan.h" -#include "activestatementcounter.h" -#include "distributedenginecomm.h" -#include "resourcemanager.h" -#include "configcpp.h" -#include "queryteleserverparms.h" -#include "iosocket.h" -#include "joblist.h" -#include "joblistfactory.h" -#include "oamcache.h" -#include "simplecolumn.h" -#include "bytestream.h" -#include "telestats.h" -#include "messageobj.h" -#include "messagelog.h" -#include "sqllogger.h" -#include "femsghandler.h" -#include "idberrorinfo.h" -#include "MonitorProcMem.h" -#include "liboamcpp.h" -#include "crashtrace.h" -#include "service.h" - -#include -#include -#include - -#include "dbrm.h" - -#include "mariadb_my_sys.h" -#include "statistics.h" -#include "serviceexemgr.h" -#include "sqlfrontsessionthread.h" - - -namespace exemgr -{ - ServiceExeMgr* globServiceExeMgr = nullptr; - void startRssMon(size_t maxPct, int pauseSeconds); - - void added_a_pm(int) - { - logging::LoggingID logid(21, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("exeMgr caught SIGHUP. Resetting connections"); - msg.format(args1); - std::cout << msg.msg().c_str() << std::endl; - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid); - - auto* dec = exemgr::globServiceExeMgr->getDec(); - if (dec) - { - oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - oamCache->forceReload(); - dec->Setup(); - } - } - - void printTotalUmMemory(int sig) - { - int64_t num = globServiceExeMgr->getRm().availableMemory(); - std::cout << "Total UM memory available: " << num << std::endl; - } - - void ServiceExeMgr::setupSignalHandlers() - { - struct sigaction ign; - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = SIG_IGN; - - sigaction(SIGPIPE, &ign, 0); - - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = exemgr::added_a_pm; - sigaction(SIGHUP, &ign, 0); - ign.sa_handler = exemgr::printTotalUmMemory; - sigaction(SIGUSR1, &ign, 0); - memset(&ign, 0, sizeof(ign)); - ign.sa_handler = fatalHandler; - sigaction(SIGSEGV, &ign, 0); - sigaction(SIGABRT, &ign, 0); - sigaction(SIGFPE, &ign, 0); - } - - void cleanTempDir() - { - using TempDirPurpose = config::Config::TempDirPurpose; - struct Dirs - { - std::string section; - std::string allowed; - TempDirPurpose purpose; - }; - std::vector dirs{{"HashJoin", "AllowDiskBasedJoin", TempDirPurpose::Joins}, - {"RowAggregation", "AllowDiskBasedAggregation", TempDirPurpose::Aggregates}}; - const auto config = config::Config::makeConfig(); - - for (const auto& dir : dirs) - { - std::string allowStr = config->getConfig(dir.section, dir.allowed); - bool allow = (allowStr == "Y" || allowStr == "y"); - - std::string tmpPrefix = config->getTempFileDir(dir.purpose); - - if (allow && tmpPrefix.empty()) - { - std::cerr << "Empty tmp directory name for " << dir.section << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Empty tmp directory name for:"); - args.add(dir.section); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_CRITICAL, message, logid); - } - - tmpPrefix += "/"; - - idbassert(tmpPrefix != "/"); - - /* This is quite scary as ExeMgr usually runs as root */ - try - { - if (allow) - { - boost::filesystem::remove_all(tmpPrefix); - } - boost::filesystem::create_directories(tmpPrefix); - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Exception whilst cleaning tmpdir: "); - args.add(ex.what()); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); - } - catch (...) - { - std::cerr << "Caught unknown exception during tmpdir cleanup" << std::endl; - logging::LoggingID logid(16, 0, 0); - logging::Message::Args args; - logging::Message message(8); - args.add("Unknown exception whilst cleaning tmpdir"); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(logging::LOG_TYPE_WARNING, message, logid); - } - } - } - - const std::string ServiceExeMgr::prettyPrintMiniInfo(const std::string& in) - { - // 1. take the std::string and tok it by '\n' - // 2. for each part in each line calc the longest part - // 3. padding to each longest value, output a header and the lines - typedef boost::tokenizer > my_tokenizer; - boost::char_separator sep1("\n"); - my_tokenizer tok1(in, sep1); - std::vector lines; - std::string header = "Desc Mode Table TableOID ReferencedColumns PIO LIO PBE Elapsed Rows"; - const int header_parts = 10; - lines.push_back(header); - - for (my_tokenizer::const_iterator iter1 = tok1.begin(); iter1 != tok1.end(); ++iter1) - { - if (!iter1->empty()) - lines.push_back(*iter1); - } - - std::vector lens; - - for (int i = 0; i < header_parts; i++) - lens.push_back(0); - - std::vector > lineparts; - std::vector::iterator iter2; - int j; - - for (iter2 = lines.begin(), j = 0; iter2 != lines.end(); ++iter2, j++) - { - boost::char_separator sep2(" "); - my_tokenizer tok2(*iter2, sep2); - int i; - std::vector parts; - my_tokenizer::iterator iter3; - - for (iter3 = tok2.begin(), i = 0; iter3 != tok2.end(); ++iter3, i++) - { - if (i >= header_parts) - break; - - std::string part(*iter3); - - if (j != 0 && i == 8) - part.resize(part.size() - 3); - - assert(i < header_parts); - - if (part.size() > lens[i]) - lens[i] = part.size(); - - parts.push_back(part); - } - - assert(i == header_parts); - lineparts.push_back(parts); - } - - std::ostringstream oss; - - std::vector >::iterator iter1 = lineparts.begin(); - std::vector >::iterator end1 = lineparts.end(); - - oss << "\n"; - - while (iter1 != end1) - { - std::vector::iterator iter2 = iter1->begin(); - std::vector::iterator end2 = iter1->end(); - assert(distance(iter2, end2) == header_parts); - int i = 0; - - while (iter2 != end2) - { - assert(i < header_parts); - oss << std::setw(lens[i]) << std::left << *iter2 << " "; - ++iter2; - i++; - } - - oss << "\n"; - ++iter1; - } - - return oss.str(); - } - - int ServiceExeMgr::Child() - { - // Make sure CSC thinks it's on a UM or else bucket reuse stuff below will stall - if (!m_e) - setenv("CALPONT_CSC_IDENT", "um", 1); - - setupSignalHandlers(); - int err = 0; - if (!m_debug) - err = setupResources(); - std::string errMsg; - - switch (err) - { - case -1: - case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break; - - case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break; - - case -4: - errMsg = "Could not install file limits to required value, please see non-root install documentation"; - break; - - default: errMsg = "Couldn't change working directory or unknown error"; break; - } - - err = setupCwd(); - - if (err < 0) - { - oam::Oam oam; - logging::Message::Args args; - logging::Message message; - args.add(errMsg); - message.format(args); - logging::LoggingID lid(16); - logging::MessageLog ml(lid); - ml.logCriticalMessage(message); - std::cerr << errMsg << std::endl; - - NotifyServiceInitializationFailed(); - return 2; - } - - cleanTempDir(); - - logging::MsgMap msgMap; - msgMap[logDefaultMsg] = logging::Message(logDefaultMsg); - msgMap[logDbProfStartStatement] = logging::Message(logDbProfStartStatement); - msgMap[logDbProfEndStatement] = logging::Message(logDbProfEndStatement); - msgMap[logStartSql] = logging::Message(logStartSql); - msgMap[logEndSql] = logging::Message(logEndSql); - msgMap[logRssTooBig] = logging::Message(logRssTooBig); - msgMap[logDbProfQueryStats] = logging::Message(logDbProfQueryStats); - msgMap[logExeMgrExcpt] = logging::Message(logExeMgrExcpt); - msgLog_.msgMap(msgMap); - - dec_ = joblist::DistributedEngineComm::instance(rm_, true); - dec_->Open(); - - bool tellUser = true; - - messageqcpp::MessageQueueServer* mqs; - - statementsRunningCount_ = new ActiveStatementCounter(rm_->getEmExecQueueSize()); - const std::string ExeMgr = "ExeMgr1"; - for (;;) - { - try - { - mqs = new messageqcpp::MessageQueueServer(ExeMgr, rm_->getConfig(), messageqcpp::ByteStream::BlockSize, 64); - break; - } - catch (std::runtime_error& re) - { - std::string what = re.what(); - - if (what.find("Address already in use") != std::string::npos) - { - if (tellUser) - { - std::cerr << "Address already in use, retrying..." << std::endl; - tellUser = false; - } - - sleep(5); - } - else - { - throw; - } - } - } - - // class jobstepThreadPool is used by other processes. We can't call - // resourcemanaager (rm) functions during the static creation of threadpool - // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). - // From the pools perspective, it has no idea if it is ExeMgr doing the - // creation, so it has no idea which way to set the flag. So we set the max here. - joblist::JobStep::jobstepThreadPool.setMaxThreads(rm_->getJLThreadPoolSize()); - joblist::JobStep::jobstepThreadPool.setName("ExeMgrJobList"); - - if (rm_->getJlThreadPoolDebug() == "Y" || rm_->getJlThreadPoolDebug() == "y") - { - joblist::JobStep::jobstepThreadPool.setDebug(true); - joblist::JobStep::jobstepThreadPool.invoke( - threadpool::ThreadPoolMonitor(&joblist::JobStep::jobstepThreadPool)); - } - - int serverThreads = rm_->getEmServerThreads(); - int maxPct = rm_->getEmMaxPct(); - int pauseSeconds = rm_->getEmSecondsBetweenMemChecks(); - int priority = rm_->getEmPriority(); - - FEMsgHandler::threadPool.setMaxThreads(serverThreads); - FEMsgHandler::threadPool.setName("FEMsgHandler"); - - if (maxPct > 0) - { - // Defined in rssmonfcn.cpp - exemgr::startRssMon(maxPct, pauseSeconds); - } - - setpriority(PRIO_PROCESS, 0, priority); - - std::string teleServerHost(rm_->getConfig()->getConfig("QueryTele", "Host")); - - if (!teleServerHost.empty()) - { - int teleServerPort = toInt(rm_->getConfig()->getConfig("QueryTele", "Port")); - - if (teleServerPort > 0) - { - teleServerParms_ = querytele::QueryTeleServerParms(teleServerHost, teleServerPort); - } - } - - NotifyServiceStarted(); - - std::cout << "Starting ExeMgr: st = " << serverThreads << ", qs = " << rm_->getEmExecQueueSize() - << ", mx = " << maxPct << ", cf = " << rm_->getConfig()->configFile() << std::endl; - - { - BRM::DBRM* dbrm = new BRM::DBRM(); - dbrm->setSystemQueryReady(true); - delete dbrm; - } - - threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); - exeMgrThreadPool.setName("ExeMgrServer"); - - if (rm_->getExeMgrThreadPoolDebug() == "Y" || rm_->getExeMgrThreadPoolDebug() == "y") - { - exeMgrThreadPool.setDebug(true); - exeMgrThreadPool.invoke(threadpool::ThreadPoolMonitor(&exeMgrThreadPool)); - } - - // Load statistics. - try - { - statistics::StatisticsManager::instance()->loadFromFile(); - } - catch (...) - { - std::cerr << "Cannot load statistics from file " << std::endl; - } - - for (;;) - { - messageqcpp::IOSocket ios; - ios = mqs->accept(); - exeMgrThreadPool.invoke(exemgr::SQLFrontSessionThread(ios, dec_, rm_)); - } - - exeMgrThreadPool.wait(); - - return 0; - } -} // namespace diff --git a/exemgr/serviceexemgr.h b/exemgr/serviceexemgr.h deleted file mode 100644 index afcf07914..000000000 --- a/exemgr/serviceexemgr.h +++ /dev/null @@ -1,343 +0,0 @@ -/* Copyright (C) 2022 Mariadb Corporation. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include -#include -#include -#include - -#undef root_name -#include - -#include "calpontselectexecutionplan.h" -#include "mcsanalyzetableexecutionplan.h" -#include "activestatementcounter.h" -#include "distributedenginecomm.h" -#include "resourcemanager.h" -#include "configcpp.h" -#include "queryteleserverparms.h" -#include "iosocket.h" -#include "joblist.h" -#include "joblistfactory.h" -#include "oamcache.h" -#include "simplecolumn.h" -#include "bytestream.h" -#include "telestats.h" -#include "messageobj.h" -#include "messagelog.h" -#include "sqllogger.h" -#include "femsghandler.h" -#include "idberrorinfo.h" -#include "MonitorProcMem.h" -#include "liboamcpp.h" -#include "crashtrace.h" -#include "service.h" - -#include -#include -#include - -#include "dbrm.h" - -#include "mariadb_my_sys.h" -#include "statistics.h" - -namespace exemgr -{ - class Opt - { - public: - int m_debug; - bool m_e; - bool m_fg; - Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) - { - int c; - while ((c = getopt(argc, argv, "edf")) != EOF) - { - switch (c) - { - case 'd': m_debug++; break; - - case 'e': m_e = true; break; - - case 'f': m_fg = true; break; - - case '?': - default: break; - } - } - } - Opt(): m_debug(0), m_e(false), m_fg(false) {}; - int getDebugLevel() const - { - return m_debug; - } - }; - - class ServiceExeMgr : public Service, public Opt - { - using SessionMemMap_t = std::map; - using ThreadCntPerSessionMap_t = std::map; - protected: - void log(logging::LOG_TYPE type, const std::string& str) - { - logging::LoggingID logid(16); - logging::Message::Args args; - logging::Message message(8); - args.add(strerror(errno)); - message.format(args); - logging::Logger logger(logid.fSubsysID); - logger.logMessage(type, message, logid); - } - - public: - ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)) - { - bool runningWithExeMgr = true; - rm_ = joblist::ResourceManager::instance(runningWithExeMgr); - } - void LogErrno() override - { - log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno))); - } - void ParentLogChildMessage(const std::string& str) override - { - log(logging::LOG_TYPE_INFO, str); - } - int Child() override; - int Run() - { - return m_fg ? Child() : RunForking(); - } - static const constexpr unsigned logDefaultMsg = logging::M0000; - static const constexpr unsigned logDbProfStartStatement = logging::M0028; - static const constexpr unsigned logDbProfEndStatement = logging::M0029; - static const constexpr unsigned logStartSql = logging::M0041; - static const constexpr unsigned logEndSql = logging::M0042; - static const constexpr unsigned logRssTooBig = logging::M0044; - static const constexpr unsigned logDbProfQueryStats = logging::M0047; - static const constexpr unsigned logExeMgrExcpt = logging::M0055; - // If any flags other than the table mode flags are set, produce output to screeen - static const constexpr uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH & - ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF); - logging::Logger& getLogger() - { - return msgLog_; - } - void updateSessionMap(const size_t pct) - { - std::lock_guard lk(sessionMemMapMutex_); - - for (auto mapIter = sessionMemMap_.begin(); mapIter != sessionMemMap_.end(); ++mapIter) - { - if (pct > mapIter->second) - { - mapIter->second = pct; - } - } - } - ThreadCntPerSessionMap_t& getThreadCntPerSessionMap() - { - return threadCntPerSessionMap_; - } - std::mutex& getThreadCntPerSessionMapMutex() - { - return threadCntPerSessionMapMutex_; - } - void initMaxMemPct(uint32_t sessionId) - { - // WIP - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex_); - auto mapIter = sessionMemMap_.find(sessionId); - - if (mapIter == sessionMemMap_.end()) - { - sessionMemMap_[sessionId] = 0; - } - else - { - mapIter->second = 0; - } - } - } - uint64_t getMaxMemPct(const uint32_t sessionId) - { - uint64_t maxMemoryPct = 0; - // WIP - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex_); - auto mapIter = sessionMemMap_.find(sessionId); - - if (mapIter != sessionMemMap_.end()) - { - maxMemoryPct = (uint64_t)mapIter->second; - } - } - - return maxMemoryPct; - } - void deleteMaxMemPct(uint32_t sessionId) - { - if (sessionId < 0x80000000) - { - std::lock_guard lk(sessionMemMapMutex_); - auto mapIter = sessionMemMap_.find(sessionId); - - if (mapIter != sessionMemMap_.end()) - { - sessionMemMap_.erase(sessionId); - } - } - } - //...Increment the number of threads using the specified sessionId - void incThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex_); - auto mapIter = threadCntPerSessionMap_.find(sessionId); - - if (mapIter == threadCntPerSessionMap_.end()) - threadCntPerSessionMap_.insert(ThreadCntPerSessionMap_t::value_type(sessionId, 1)); - else - mapIter->second++; - } - //...Decrement the number of threads using the specified sessionId. - //...When the thread count for a sessionId reaches 0, the corresponding - //...CalpontSystemCatalog objects are deleted. - //...The user query and its associated catalog query have a different - //...session Id where the highest bit is flipped. - //...The object with id(sessionId | 0x80000000) cannot be removed before - //...user query session completes because the sysdata may be used for - //...debugging/stats purpose, such as result graph, etc. - void decThreadCntPerSession(uint32_t sessionId) - { - std::lock_guard lk(threadCntPerSessionMapMutex_); - auto mapIter = threadCntPerSessionMap_.find(sessionId); - - if (mapIter != threadCntPerSessionMap_.end()) - { - if (--mapIter->second == 0) - { - threadCntPerSessionMap_.erase(mapIter); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId); - execplan::CalpontSystemCatalog::removeCalpontSystemCatalog((sessionId ^ 0x80000000)); - } - } - } - ActiveStatementCounter* getStatementsRunningCount() - { - return statementsRunningCount_; - } - joblist::DistributedEngineComm* getDec() - { - return dec_; - } - int toInt(const std::string& val) - { - if (val.length() == 0) - return -1; - - return static_cast(config::Config::fromText(val)); - } - const std::string prettyPrintMiniInfo(const std::string& in); - - const std::string timeNow() - { - time_t outputTime = time(0); - struct tm ltm; - char buf[32]; // ctime(3) says at least 26 - size_t len = 0; - asctime_r(localtime_r(&outputTime, <m), buf); - len = strlen(buf); - - if (len > 0) - --len; - - if (buf[len] == '\n') - buf[len] = 0; - - return buf; - } - querytele::QueryTeleServerParms& getTeleServerParms() - { - return teleServerParms_; - } - joblist::ResourceManager& getRm() - { - return *rm_; - } - private: - void setupSignalHandlers(); - int8_t setupCwd() - { - std::string workdir = rm_->getScWorkingDir(); - int8_t rc = chdir(workdir.c_str()); - - if (rc < 0 || access(".", W_OK) != 0) - rc = chdir("/tmp"); - - return (rc < 0) ? -5 : rc; - } - int setupResources() - { - struct rlimit rlim; - - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -1; - } - rlim.rlim_cur = rlim.rlim_max = 65536; - - if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -2; - } - if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) - { - return -3; - } - if (rlim.rlim_cur != 65536) - { - return -4; - } - return 0; - } - - logging::Logger msgLog_; - SessionMemMap_t sessionMemMap_; // track memory% usage during a query - std::mutex sessionMemMapMutex_; - //...The FrontEnd may establish more than 1 connection (which results in - // more than 1 ExeMgr thread) per session. These threads will share - // the same CalpontSystemCatalog object for that session. Here, we - // define a std::map to track how many threads are sharing each session, so - // that we know when we can safely delete a CalpontSystemCatalog object - // shared by multiple threads per session. - ThreadCntPerSessionMap_t threadCntPerSessionMap_; - std::mutex threadCntPerSessionMapMutex_; - ActiveStatementCounter* statementsRunningCount_; - joblist::DistributedEngineComm* dec_; - joblist::ResourceManager* rm_; - // Its attributes are set in Child() - querytele::QueryTeleServerParms teleServerParms_; - }; - extern ServiceExeMgr* globServiceExeMgr; -} \ No newline at end of file diff --git a/exemgr/sqlfrontsessionthread.cpp b/exemgr/sqlfrontsessionthread.cpp deleted file mode 100644 index 3b5094e7a..000000000 --- a/exemgr/sqlfrontsessionthread.cpp +++ /dev/null @@ -1,985 +0,0 @@ -/* Copyright (C) 2022 Mariadb Corporation. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#include "sqlfrontsessionthread.h" - -namespace exemgr -{ - uint64_t SQLFrontSessionThread::getMaxMemPct(uint32_t sessionId) - { - return globServiceExeMgr->getMaxMemPct(sessionId); - } - void SQLFrontSessionThread::deleteMaxMemPct(uint32_t sessionId) - { - return globServiceExeMgr->deleteMaxMemPct(sessionId); - } - void SQLFrontSessionThread::incThreadCntPerSession(uint32_t sessionId) - { - return globServiceExeMgr->incThreadCntPerSession(sessionId); - } - void SQLFrontSessionThread::decThreadCntPerSession(uint32_t sessionId) - { - return globServiceExeMgr->decThreadCntPerSession(sessionId); - } - - void SQLFrontSessionThread::initMaxMemPct(uint32_t sessionId) - { - return globServiceExeMgr->initMaxMemPct(sessionId); - } - //...Get and log query stats to specified output stream - const std::string SQLFrontSessionThread::formatQueryStats( - joblist::SJLP& jl, // joblist associated with query - const std::string& label, // header label to print in front of log output - bool includeNewLine, // include line breaks in query stats std::string - bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned) - { - std::ostringstream os; - - // Get stats if not already acquired for current query - if (!fStatsRetrieved) - { - if (wantExtendedStats) - { - // wait for the ei data to be written by another thread (brain-dead) - struct timespec req = {0, 250000}; // 250 usec - nanosleep(&req, 0); - } - - // Get % memory usage during current query for sessionId - jl->querySummary(wantExtendedStats); - fStats = jl->queryStats(); - fStats.fMaxMemPct = getMaxMemPct(fStats.fSessionID); - fStats.fRows = rowsReturned; - fStatsRetrieved = true; - } - - std::string queryMode; - queryMode = (vtableModeOn ? "Distributed" : "Standard"); - - // Log stats to specified output stream - os << label << ": MaxMemPct-" << fStats.fMaxMemPct << "; NumTempFiles-" << fStats.fNumFiles - << "; TempFileSpace-" << roundBytes(fStats.fFileBytes) << "; ApproxPhyI/O-" << fStats.fPhyIO - << "; CacheI/O-" << fStats.fCacheIO << "; BlocksTouched-" << fStats.fMsgRcvCnt; - - if (includeNewLine) - os << std::endl << " "; // insert line break - else - os << "; "; // continue without line break - - os << "PartitionBlocksEliminated-" << fStats.fCPBlocksSkipped << "; MsgBytesIn-" - << roundBytes(fStats.fMsgBytesIn) << "; MsgBytesOut-" << roundBytes(fStats.fMsgBytesOut) << "; Mode-" - << queryMode; - - return os.str(); - } - - //... Round off to human readable format (KB, MB, or GB). - const std::string SQLFrontSessionThread::roundBytes(uint64_t value) const - { - const char* units[] = {"B", "KB", "MB", "GB", "TB"}; - uint64_t i = 0, up = 0; - uint64_t roundedValue = value; - - while (roundedValue > 1024 && i < 4) - { - up = (roundedValue & 512); - roundedValue /= 1024; - i++; - } - - if (up) - roundedValue++; - - std::ostringstream oss; - oss << roundedValue << units[i]; - return oss.str(); - } - - //...Round off to nearest (1024*1024) MB - uint64_t SQLFrontSessionThread::roundMB(uint64_t value) const - { - uint64_t roundedValue = value >> 20; - - if (value & 524288) - roundedValue++; - - return roundedValue; - } - - void SQLFrontSessionThread::setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms) - { - for (execplan::CalpontSelectExecutionPlan::RMParmVec::const_iterator it = parms.begin(); - it != parms.end(); ++it) - { - switch (it->id) - { - case execplan::PMSMALLSIDEMEMORY: - { - globServiceExeMgr->getRm().addHJPmMaxSmallSideMap(it->sessionId, it->value); - break; - } - - case execplan::UMSMALLSIDEMEMORY: - { - globServiceExeMgr->getRm().addHJUmMaxSmallSideMap(it->sessionId, it->value); - break; - } - - default:; - } - } - } - - void SQLFrontSessionThread::buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, - boost::shared_ptr csc) - { - const execplan::CalpontSelectExecutionPlan::ColumnMap& colMap = csep.columnMap(); - std::string schemaName; - - for (auto it = colMap.begin(); it != colMap.end(); ++it) - { - const auto sc = dynamic_cast((it->second).get()); - - if (sc) - { - schemaName = sc->schemaName(); - - // only the first time a schema is got will actually query - // system catalog. System catalog keeps a schema name std::map. - // if a schema exists, the call getSchemaInfo returns without - // doing anything. - if (!schemaName.empty()) - csc->getSchemaInfo(schemaName); - } - } - - execplan::CalpontSelectExecutionPlan::SelectList::const_iterator subIt; - - for (subIt = csep.derivedTableList().begin(); subIt != csep.derivedTableList().end(); ++subIt) - { - buildSysCache(*(dynamic_cast(subIt->get())), csc); - } - } - - void SQLFrontSessionThread::writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg) - { - messageqcpp::ByteStream emsgBs; - messageqcpp::ByteStream tbs; - tbs << code; - fIos.write(tbs); - emsgBs << emsg; - fIos.write(emsgBs); - } - - void SQLFrontSessionThread::analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted) - { - auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); - messageqcpp::ByteStream::quadbyte qb; - execplan::MCSAnalyzeTableExecutionPlan caep; - - bs = fIos.read(); - caep.unserialize(bs); - - statementsRunningCount->incr(stmtCounted); - jl = joblist::JobListFactory::makeJobList(&caep, fRm, false, true); - - // Joblist is empty. - if (jl->status() == logging::statisticsJobListEmpty) - { - if (caep.traceOn()) - std::cout << "JobList is empty " << std::endl; - - jl.reset(); - bs.restart(); - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - statementsRunningCount->decr(stmtCounted); - return; - } - - if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) - { - std::cout << "fEc setup " << std::endl; - fEc->Setup(); - } - - if (jl->status() == 0) - { - std::string emsg; - - if (jl->putEngineComm(fEc) != 0) - throw std::runtime_error(jl->errMsg()); - } - else - { - throw std::runtime_error("ExeMgr: could not build a JobList!"); - } - - // Execute a joblist. - jl->doQuery(); - - FEMsgHandler msgHandler(jl, &fIos); - - msgHandler.start(); - auto rowCount = jl->projectTable(100, bs); - msgHandler.stop(); - - auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); - - if (caep.traceOn()) - std::cout << "Row count " << rowCount << std::endl; - - // Process `RowGroup`, increase an epoch and save statistics to the file. - auto* statisticsManager = statistics::StatisticsManager::instance(); - statisticsManager->analyzeColumnKeyTypes(outRG, caep.traceOn()); - statisticsManager->incEpoch(); - statisticsManager->saveToFile(); - - // Distribute statistics across all ExeMgr clients if possible. - statistics::StatisticsDistributor::instance()->distributeStatistics(); - - // Send the signal back to front-end. - bs.restart(); - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - statementsRunningCount->decr(stmtCounted); - } - - void SQLFrontSessionThread::analyzeTableHandleStats(messageqcpp::ByteStream& bs) - { - messageqcpp::ByteStream::quadbyte qb; -#ifdef DEBUG_STATISTICS - std::cout << "Get distributed statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - bs = fIos.read(); -#ifdef DEBUG_STATISTICS - std::cout << "Read the hash from statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - uint64_t dataHashRec; - bs >> dataHashRec; - - uint64_t dataHash = statistics::StatisticsManager::instance()->computeHashFromStats(); - // The stats are the same. - if (dataHash == dataHashRec) - { -#ifdef DEBUG_STATISTICS - std::cout << "The hash is the same as rec hash on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - return; - } - - bs.restart(); - qb = ANALYZE_TABLE_NEED_STATS; - bs << qb; - fIos.write(bs); - - bs.restart(); - bs = fIos.read(); -#ifdef DEBUG_STATISTICS - std::cout << "Read statistics on ExeMgr(Client) from ExeMgr(Server) " << std::endl; -#endif - statistics::StatisticsManager::instance()->unserialize(bs); - statistics::StatisticsManager::instance()->saveToFile(); - -#ifdef DEBUG_STATISTICS - std::cout << "Write flag on ExeMgr(Client) to ExeMgr(Server)" << std::endl; -#endif - qb = ANALYZE_TABLE_SUCCESS; - bs << qb; - fIos.write(bs); - bs.reset(); - } - - void SQLFrontSessionThread::operator()() - { - messageqcpp::ByteStream bs, inbs; - execplan::CalpontSelectExecutionPlan csep; - csep.sessionID(0); - joblist::SJLP jl; - bool incSQLFrontSessionThreadCnt = true; - std::mutex jlMutex; - std::condition_variable jlCleanupDone; - int destructing = 0; - int gDebug = globServiceExeMgr->getDebugLevel(); - logging::Logger& msgLog = globServiceExeMgr->getLogger(); - - bool selfJoin = false; - bool tryTuples = false; - bool usingTuples = false; - bool stmtCounted = false; - auto* statementsRunningCount = globServiceExeMgr->getStatementsRunningCount(); - - try - { - for (;;) - { - selfJoin = false; - tryTuples = false; - usingTuples = false; - - if (jl) - { - // puts the real destruction in another thread to avoid - // making the whole session wait. It can take several seconds. - std::unique_lock scoped(jlMutex); - destructing++; - std::thread bgdtor( - [jl, &jlMutex, &jlCleanupDone, &destructing] - { - std::unique_lock scoped(jlMutex); - const_cast(jl).reset(); // this happens second; does real destruction - if (--destructing == 0) - jlCleanupDone.notify_one(); - }); - jl.reset(); // this runs first - bgdtor.detach(); - } - - bs = fIos.read(); - - if (bs.length() == 0) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### Got a close(1) for session id " << csep.sessionID() << std::endl; - - // connection closed by client - fIos.close(); - break; - } - else if (bs.length() < 4) // Not a CalpontSelectExecutionPlan - { - if (gDebug) - std::cout << "### Got a not-a-plan for session id " << csep.sessionID() << " with length " - << bs.length() << std::endl; - - fIos.close(); - break; - } - else if (bs.length() == 4) // possible tuple flag - { - messageqcpp::ByteStream::quadbyte qb; - bs >> qb; - - if (qb == 4) // UM wants new tuple i/f - { - if (gDebug) - std::cout << "### UM wants tuples" << std::endl; - - tryTuples = true; - // now wait for the CSEP... - bs = fIos.read(); - } - else if (qb == 5) // somebody wants stats - { - bs.restart(); - qb = statementsRunningCount->cur(); - bs << qb; - qb = statementsRunningCount->waiting(); - bs << qb; - fIos.write(bs); - fIos.close(); - break; - } - else if (qb == ANALYZE_TABLE_EXECUTE) - { - analyzeTableExecute(bs, jl, stmtCounted); - continue; - } - else if (qb == ANALYZE_TABLE_REC_STATS) - { - analyzeTableHandleStats(bs); - continue; - } - else - { - if (gDebug) - std::cout << "### Got a not-a-plan value " << qb << std::endl; - - fIos.close(); - break; - } - } - - new_plan: - try - { - csep.unserialize(bs); - } - catch (logging::IDBExcept& ex) - { - // We can get here on illegal function parameter data type, e.g. - // SELECT blob_column|1 FROM t1; - statementsRunningCount->decr(stmtCounted); - writeCodeAndError(ex.errorCode(), std::string(ex.what())); - continue; - } - - querytele::QueryTeleStats qts; - - if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) - { - qts.query_uuid = csep.uuid(); - qts.msg_type = querytele::QueryTeleStats::QT_START; - qts.start_time = querytele::QueryTeleClient::timeNowms(); - qts.query = csep.data(); - qts.session_id = csep.sessionID(); - qts.query_type = csep.queryType(); - qts.system_name = fOamCachePtr->getSystemName(); - qts.module_name = fOamCachePtr->getModuleName(); - qts.local_query = csep.localQuery(); - qts.schema_name = csep.schemaName(); - fTeleClient.postQueryTele(qts); - } - - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", got a CSEP" << std::endl; - - setRMParms(csep.rmParms()); - // Re-establish lost PP connections. - if (UNLIKELY(fEc->getNumConnections() != fEc->connectedPmServers())) - { - fEc->Setup(); - } - // @bug 1021. try to get schema cache for a come in query. - // skip system catalog queries. - if (!csep.isInternal()) - { - boost::shared_ptr csc = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(csep.sessionID()); - buildSysCache(csep, csc); - } - - // As soon as we have a session id for this thread, update the - // thread count per session; only do this once per thread. - // Mask 0x80000000 is for associate user query and csc query - if (incSQLFrontSessionThreadCnt) - { - // WIP - incThreadCntPerSession(csep.sessionID() | 0x80000000); - incSQLFrontSessionThreadCnt = false; - } - - bool needDbProfEndStatementMsg = false; - logging::Message::Args args; - std::string sqlText = csep.data(); - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - - // Initialize stats for this query, including - // init sessionMemMap entry for this session to 0 memory %. - // We will need this later for traceOn() or if we receive a - // table request with qb=3 (see below). This is also recorded - // as query start time. - initStats(csep.sessionID(), sqlText); - fStats.fQueryType = csep.queryType(); - - // Log start and end statement if tracing is enabled. Keep in - // mind the trace flag won't be set for system catalog queries. - if (csep.traceOn()) - { - args.reset(); - args.add((int)csep.statementID()); - args.add((int)csep.verID().currentScn); - args.add(sqlText); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfStartStatement, args, li); - needDbProfEndStatementMsg = true; - } - - // Don't log subsequent self joins after first. - if (selfJoin) - sqlText = ""; - - std::ostringstream oss; - oss << sqlText << "; |" << csep.schemaName() << "|"; - logging::SQLLogger sqlLog(oss.str(), li); - - statementsRunningCount->incr(stmtCounted); - - if (tryTuples) - { - try // @bug2244: try/catch around fIos.write() calls responding to makeTupleList - { - jl = joblist::JobListFactory::makeJobList(&csep, fRm, true, true); - // assign query stats - jl->queryStats(fStats); - - if ((jl->status()) == 0 && (jl->putEngineComm(fEc) == 0)) - { - usingTuples = true; - - // Tell the FE that we're sending tuples back, not TableBands - writeCodeAndError(0, "NOERROR"); - auto tjlp = dynamic_cast(jl.get()); - assert(tjlp); - messageqcpp::ByteStream tbs; - tbs << tjlp->getOutputRowGroup(); - fIos.write(tbs); - } - else - { - const std::string emsg = jl->errMsg(); - statementsRunningCount->decr(stmtCounted); - writeCodeAndError(jl->status(), emsg); - std::cerr << "ExeMgr: could not build a tuple joblist: " << emsg << std::endl; - continue; - } - } - catch (std::exception& ex) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: error writing makeJoblist " - "response; " - << ex.what(); - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: unknown error writing makeJoblist " - "response; "; - throw std::runtime_error(errMsg.str()); - } - - if (!usingTuples) - { - if (gDebug) - std::cout << "### UM wanted tuples but it didn't work out :-(" << std::endl; - } - else - { - if (gDebug) - std::cout << "### UM wanted tuples and we'll do our best;-)" << std::endl; - } - } - else - { - usingTuples = false; - jl = joblist::JobListFactory::makeJobList(&csep, fRm, false, true); - - if (jl->status() == 0) - { - std::string emsg; - - if (jl->putEngineComm(fEc) != 0) - throw std::runtime_error(jl->errMsg()); - } - else - { - throw std::runtime_error("ExeMgr: could not build a JobList!"); - } - } - - jl->doQuery(); - - execplan::CalpontSystemCatalog::OID tableOID; - bool swallowRows = false; - joblist::DeliveredTableMap tm; - uint64_t totalBytesSent = 0; - uint64_t totalRowCount = 0; - - // Project each table as the FE asks for it - for (;;) - { - bs = fIos.read(); - - if (bs.length() == 0) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### Got a close(2) for session id " << csep.sessionID() << std::endl; - - break; - } - - if (gDebug && bs.length() > 4) - std::cout << "### For session id " << csep.sessionID() << ", got too many bytes = " << bs.length() - << std::endl; - - // TODO: Holy crud! Can this be right? - //@bug 1444 Yes, if there is a self-join - if (bs.length() > 4) - { - selfJoin = true; - statementsRunningCount->decr(stmtCounted); - goto new_plan; - } - - assert(bs.length() == 4); - - messageqcpp::ByteStream::quadbyte qb; - - try // @bug2244: try/catch around fIos.write() calls responding to qb command - { - bs >> qb; - - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", got a command = " << qb - << std::endl; - - if (qb == 0) - { - // No more tables, query is done - break; - } - else if (qb == 1) - { - // super-secret flag indicating that the UM is going to scarf down all the rows in the - // query. - swallowRows = true; - tm = jl->deliveredTables(); - continue; - } - else if (qb == 2) - { - // UM just wants any table - assert(swallowRows); - auto iter = tm.begin(); - - if (iter == tm.end()) - { - if (gDebug > 1 || (gDebug && !csep.isInternal())) - std::cout << "### For session id " << csep.sessionID() << ", returning end flag" - << std::endl; - - bs.restart(); - bs << (messageqcpp::ByteStream::byte)1; - fIos.write(bs); - continue; - } - - tableOID = iter->first; - } - else if (qb == 3) // special option-UM wants job stats std::string - { - std::string statsString; - - // Log stats std::string to be sent back to front end - statsString = formatQueryStats( - jl, "Query Stats", false, - !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), - (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), totalRowCount); - - bs.restart(); - bs << statsString; - - if ((csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG) != 0) - { - bs << jl->extendedInfo(); - bs << globServiceExeMgr->prettyPrintMiniInfo(jl->miniInfo()); - } - else - { - std::string empty; - bs << empty; - bs << empty; - } - - // send stats to connector for inserting to the querystats table - fStats.serialize(bs); - fIos.write(bs); - continue; - } - // for table mode handling - else if (qb == 4) - { - statementsRunningCount->decr(stmtCounted); - bs = fIos.read(); - goto new_plan; - } - else // (qb > 3) - { - // Return table bands for the requested tableOID - tableOID = static_cast(qb); - } - } - catch (std::exception& ex) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: error writing qb response " - "for qb cmd " - << qb << "; " << ex.what(); - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - errMsg << "ExeMgr: unknown error writing qb response " - "for qb cmd " - << qb; - throw std::runtime_error(errMsg.str()); - } - - if (swallowRows) - tm.erase(tableOID); - - FEMsgHandler msgHandler(jl, &fIos); - - if (tableOID == 100) - msgHandler.start(); - - //...Loop serializing table bands projected for the tableOID - for (;;) - { - uint32_t rowCount; - - rowCount = jl->projectTable(tableOID, bs); - - msgHandler.stop(); - - if (jl->status()) - { - const auto errInfo = logging::IDBErrorInfo::instance(); - - if (jl->errMsg().length() != 0) - bs << jl->errMsg(); - else - bs << errInfo->errorMsg(jl->status()); - } - - try // @bug2244: try/catch around fIos.write() calls projecting rows - { - if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) - { - // Skip the write to the front end until the last empty band. Used to time queries - // through without any front end waiting. - if (tableOID < 3000 || rowCount == 0) - fIos.write(bs); - } - else - { - fIos.write(bs); - } - } - catch (std::exception& ex) - { - msgHandler.stop(); - std::ostringstream errMsg; - errMsg << "ExeMgr: error projecting rows " - "for tableOID: " - << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount << "; " - << ex.what(); - jl->abort(); - - while (rowCount) - rowCount = jl->projectTable(tableOID, bs); - - if (tableOID == 100 && msgHandler.aborted()) - { - /* TODO: modularize the cleanup code, as well as - * the rest of this fcn */ - - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - fIos.close(); - return; - } - - // std::cout << "connection drop\n"; - throw std::runtime_error(errMsg.str()); - } - catch (...) - { - std::ostringstream errMsg; - msgHandler.stop(); - errMsg << "ExeMgr: unknown error projecting rows " - "for tableOID: " - << tableOID << "; rowCnt: " << rowCount << "; prevTotRowCnt: " << totalRowCount; - jl->abort(); - - while (rowCount) - rowCount = jl->projectTable(tableOID, bs); - - throw std::runtime_error(errMsg.str()); - } - - totalRowCount += rowCount; - totalBytesSent += bs.length(); - - if (rowCount == 0) - { - msgHandler.stop(); - // No more bands, table is done - bs.reset(); - - // @bug 2083 decr active statement count here for table mode. - if (!usingTuples) - statementsRunningCount->decr(stmtCounted); - - break; - } - else - { - bs.restart(); - } - } // End of loop to project and serialize table bands for a table - } // End of loop to process tables - - // @bug 828 - if (csep.traceOn()) - jl->graph(csep.sessionID()); - - if (needDbProfEndStatementMsg) - { - std::string ss; - std::ostringstream prefix; - prefix << "ses:" << csep.sessionID() << " Query Totals"; - - // Log stats std::string to standard out - ss = formatQueryStats(jl, prefix.str(), true, - !(csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF), - (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_LOG), - totalRowCount); - //@Bug 1306. Added timing info for real time tracking. - std::cout << ss << " at " << globServiceExeMgr->timeNow() << std::endl; - - // log query stats to debug log file - args.reset(); - args.add((int)csep.statementID()); - args.add(fStats.fMaxMemPct); - args.add(fStats.fNumFiles); - args.add(fStats.fFileBytes); // log raw byte count instead of MB - args.add(fStats.fPhyIO); - args.add(fStats.fCacheIO); - args.add(fStats.fMsgRcvCnt); - args.add(fStats.fMsgBytesIn); - args.add(fStats.fMsgBytesOut); - args.add(fStats.fCPBlocksSkipped); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfQueryStats, args, li); - //@bug 1327 - deleteMaxMemPct(csep.sessionID()); - // Calling reset here, will cause joblist destructor to be - // called, which "joins" the threads. We need to do that - // here to make sure all syslogging from all the threads - // are complete; and that our logDbProfEndStatement will - // appear "last" in the syslog for this SQL statement. - // puts the real destruction in another thread to avoid - // making the whole session wait. It can take several seconds. - int stmtID = csep.statementID(); - std::unique_lock scoped(jlMutex); - destructing++; - std::thread bgdtor( - [jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing, &msgLog] - { - std::unique_lock scoped(jlMutex); - const_cast(jl).reset(); // this happens second; does real destruction - logging::Message::Args args; - args.add(stmtID); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, ServiceExeMgr::logDbProfEndStatement, args, li); - if (--destructing == 0) - jlCleanupDone.notify_one(); - }); - jl.reset(); // this happens first - bgdtor.detach(); - } - else - // delete sessionMemMap entry for this session's memory % use - deleteMaxMemPct(csep.sessionID()); - - std::string endtime(globServiceExeMgr->timeNow()); - - if ((csep.traceFlags() & globServiceExeMgr->flagsWantOutput) && (csep.sessionID() < 0x80000000)) - { - std::cout << "For session " << csep.sessionID() << ": " << totalBytesSent << " bytes sent back at " - << endtime << std::endl; - - // @bug 663 - Implemented caltraceon(16) to replace the - // $FIFO_SINK compiler definition in pColStep. - // This option consumes rows in the project steps. - if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS4) - { - std::cout << std::endl; - std::cout << "**** No data returned to DM. Rows consumed " - "in ProjectSteps - caltrace(16) is on (FIFO_SINK)." - " ****" - << std::endl; - std::cout << std::endl; - } - else if (csep.traceFlags() & execplan::CalpontSelectExecutionPlan::TRACE_NO_ROWS3) - { - std::cout << std::endl; - std::cout << "**** No data returned to DM - caltrace(8) is " - "on (SWALLOW_ROWS_EXEMGR). ****" - << std::endl; - std::cout << std::endl; - } - } - - statementsRunningCount->decr(stmtCounted); - - if (!csep.isInternal() && (csep.queryType() == "SELECT" || csep.queryType() == "INSERT_SELECT")) - { - qts.msg_type = querytele::QueryTeleStats::QT_SUMMARY; - qts.max_mem_pct = fStats.fMaxMemPct; - qts.num_files = fStats.fNumFiles; - qts.phy_io = fStats.fPhyIO; - qts.cache_io = fStats.fCacheIO; - qts.msg_rcv_cnt = fStats.fMsgRcvCnt; - qts.cp_blocks_skipped = fStats.fCPBlocksSkipped; - qts.msg_bytes_in = fStats.fMsgBytesIn; - qts.msg_bytes_out = fStats.fMsgBytesOut; - qts.rows = totalRowCount; - qts.end_time = querytele::QueryTeleClient::timeNowms(); - qts.session_id = csep.sessionID(); - qts.query_type = csep.queryType(); - qts.query = csep.data(); - qts.system_name = fOamCachePtr->getSystemName(); - qts.module_name = fOamCachePtr->getModuleName(); - qts.local_query = csep.localQuery(); - fTeleClient.postQueryTele(qts); - } - } - - // Release CSC object (for sessionID) that was added by makeJobList() - // Mask 0x80000000 is for associate user query and csc query. - // (actual joblist destruction happens at the top of this loop) - decThreadCntPerSession(csep.sessionID() | 0x80000000); - } - catch (std::exception& ex) - { - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - std::cerr << "### ExeMgr ses:" << csep.sessionID() << " caught: " << ex.what() << std::endl; - logging::Message::Args args; - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - args.add(ex.what()); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logExeMgrExcpt, args, li); - fIos.close(); - } - catch (...) - { - decThreadCntPerSession(csep.sessionID() | 0x80000000); - statementsRunningCount->decr(stmtCounted); - std::cerr << "### Exception caught!" << std::endl; - logging::Message::Args args; - logging::LoggingID li(16, csep.sessionID(), csep.txnID()); - args.add("ExeMgr caught unknown exception"); - msgLog.logMessage(logging::LOG_TYPE_CRITICAL, ServiceExeMgr::logExeMgrExcpt, args, li); - fIos.close(); - } - - // make sure we don't leave scope while joblists are being destroyed - std::unique_lock scoped(jlMutex); - while (destructing > 0) - jlCleanupDone.wait(scoped); -} -}; // namespace exemgr \ No newline at end of file diff --git a/exemgr/sqlfrontsessionthread.h b/exemgr/sqlfrontsessionthread.h deleted file mode 100644 index ca605ea79..000000000 --- a/exemgr/sqlfrontsessionthread.h +++ /dev/null @@ -1,131 +0,0 @@ -/* Copyright (C) 2022 Mariadb Corporation. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include -#include -#include -#include - -#undef root_name -#include - -#include "calpontselectexecutionplan.h" -#include "mcsanalyzetableexecutionplan.h" -#include "activestatementcounter.h" -#include "distributedenginecomm.h" -#include "resourcemanager.h" -#include "configcpp.h" -#include "queryteleserverparms.h" -#include "iosocket.h" -#include "joblist.h" -#include "joblistfactory.h" -#include "oamcache.h" -#include "simplecolumn.h" -#include "bytestream.h" -#include "telestats.h" -#include "messageobj.h" -#include "messagelog.h" -#include "sqllogger.h" -#include "femsghandler.h" -#include "idberrorinfo.h" -#include "MonitorProcMem.h" -#include "liboamcpp.h" -#include "crashtrace.h" -#include "service.h" - -#include -#include -#include - -#include "dbrm.h" - -#include "mariadb_my_sys.h" -#include "statistics.h" -#include "serviceexemgr.h" - -namespace exemgr -{ - class SQLFrontSessionThread - { - public: - SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, - joblist::ResourceManager* rm) - : fIos(ios) - , fEc(ec) - , fRm(rm) - , fStatsRetrieved(false) - , fTeleClient(globServiceExeMgr->getTeleServerParms()) - , fOamCachePtr(oam::OamCache::makeOamCache()) - { - } - - private: - messageqcpp::IOSocket fIos; - joblist::DistributedEngineComm* fEc; - joblist::ResourceManager* fRm; - querystats::QueryStats fStats; - - // Variables used to store return stats - bool fStatsRetrieved; - - querytele::QueryTeleClient fTeleClient; - - oam::OamCache* fOamCachePtr; // this ptr is copyable... - - //...Reinitialize stats for start of a new query - void initStats(uint32_t sessionId, std::string& sqlText) - { - initMaxMemPct(sessionId); - - fStats.reset(); - fStats.setStartTime(); - fStats.fSessionID = sessionId; - fStats.fQuery = sqlText; - fStatsRetrieved = false; - } - //...Get % memory usage during latest query for sesssionId. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static uint64_t getMaxMemPct(uint32_t sessionId); - //...Delete sessionMemMap entry for the specified session's memory % use. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void deleteMaxMemPct(uint32_t sessionId); - //...Get and log query stats to specified output stream - const std::string formatQueryStats( - joblist::SJLP& jl, // joblist associated with query - const std::string& label, // header label to print in front of log output - bool includeNewLine, // include line breaks in query stats std::string - bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); - static void incThreadCntPerSession(uint32_t sessionId); - static void decThreadCntPerSession(uint32_t sessionId); - //...Init sessionMemMap entry for specified session to 0 memory %. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void initMaxMemPct(uint32_t sessionId); - //... Round off to human readable format (KB, MB, or GB). - const std::string roundBytes(uint64_t value) const; - void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); - void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, - boost::shared_ptr csc); - void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); - void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); - void analyzeTableHandleStats(messageqcpp::ByteStream& bs); - uint64_t roundMB(uint64_t value) const; - public: - void operator()(); - }; -} \ No newline at end of file diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index b010b033e..80ca95be9 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -10,7 +10,6 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore-start.sh.in" "${ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-workernode.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-workernode.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-controllernode.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-controllernode.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-primproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-primproc.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-exemgr.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-exemgr.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-writeengineserver.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-writeengineserver.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-dmlproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-dmlproc.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-ddlproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-ddlproc.service" @ONLY) @@ -21,12 +20,12 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.py.in" "${CMAKE_CURRENT_ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY) -install(PROGRAMS columnstore-post-install - columnstore-pre-uninstall - columnstore_run.sh - post-mysql-install - post-mysqld-install - columnstoreSyslogSetup.sh +install(PROGRAMS columnstore-post-install + columnstore-pre-uninstall + columnstore_run.sh + post-mysql-install + post-mysqld-install + columnstoreSyslogSetup.sh mcs-stop-controllernode.sh mcs-loadbrm.py mcs-savebrm.py @@ -45,7 +44,6 @@ install(FILES mariadb-columnstore.service mcs-workernode.service mcs-controllernode.service mcs-primproc.service - mcs-exemgr.service mcs-writeengineserver.service mcs-dmlproc.service mcs-ddlproc.service diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index dea0eba12..8f8d33832 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -70,7 +70,7 @@ quiet=0 stop_mysqld=0 if [ -z "$(pgrep -x mariadbd)" ];then - # Startup mysqld + # Startup mysqld systemctl cat mariadb.service > /dev/null 2>&1 if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl start mariadb.service @@ -165,8 +165,6 @@ if [ $user = "root" ]; then cp @ENGINE_SUPPORTDIR@/mcs-ddlproc.service /lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-dmlproc.service /usr/lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-dmlproc.service /lib/systemd/system/. >/dev/null 2>&1 - cp @ENGINE_SUPPORTDIR@/mcs-exemgr.service /usr/lib/systemd/system/. >/dev/null 2>&1 - cp @ENGINE_SUPPORTDIR@/mcs-exemgr.service /lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-primproc.service /usr/lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-primproc.service /lib/systemd/system/. >/dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/mcs-workernode.service /usr/lib/systemd/system/mcs-workernode@.service >/dev/null 2>&1 @@ -203,7 +201,7 @@ if [ $user = "root" ]; then update-rc.d columnstore defaults 99 > /dev/null 2>&1 else echo "" - echo "Package 'systemctl', 'chkconfig' or 'update-rc.d' not installed, contact your sysadmin if you want to setup to autostart for columnstore" + echo "Package 'systemctl', 'chkconfig' or 'update-rc.d' not installed, contact your sysadmin if you want to setup to autostart for columnstore" fi fi fi @@ -225,7 +223,7 @@ if [ $user = "root" ]; then fi else chown $user:$user @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml - + cat < /dev/null 2>&1 diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index 941d9aa7a..82e2bd570 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -35,7 +35,7 @@ systemctl cat mariadb-columnstore.service > /dev/null 2>&1 if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl stop mariadb-columnstore >/dev/null 2>&1 else - PROGS='PrimProc ExeMgr DMLProc DDLProc WriteEngineServer StorageManager controllernode workernode' + PROGS='PrimProc ExeMgr DMLProc DDLProc WriteEngineServer StorageManager controllernode workernode' kill $(pidof $PROGS) > /dev/null sleep 3 kill -9 $(pidof $PROGS) > /dev/null @@ -43,7 +43,7 @@ else fi if [ -n "$(pgrep -x ProcMon)" ] || [ -n "$(pgrep -x ProcMgr)" ];then - # Old system must be running, kill ProcMon/ProcMgr + # Old system must be running, kill ProcMon/ProcMgr pkill ProcMon pkill ProcMgr fi @@ -94,8 +94,12 @@ if [ -n "$systemctl" ] && [ $(running_systemd) -eq 0 ]; then rm -f /lib/systemd/system/mcs-ddlproc.service rm -f /usr/lib/systemd/system/mcs-dmlproc.service rm -f /lib/systemd/system/mcs-dmlproc.service - rm -f /usr/lib/systemd/system/mcs-exemgr.service - rm -f /lib/systemd/system/mcs-exemgr.service + if [[ -f /usr/lib/systemd/system/mcs-exemgr.service ]] + rm -f /usr/lib/systemd/system/mcs-exemgr.service + fi + if [[ -f /lib/systemd/system/mcs-exemgr.service ]] + rm -f /lib/systemd/system/mcs-exemgr.service + fi rm -f /usr/lib/systemd/system/mcs-primproc.service rm -f /lib/systemd/system/mcs-primproc.service rm -f /usr/lib/systemd/system/mcs-workernode@.service @@ -123,7 +127,7 @@ else updaterc=`which update-rc.d 2>/dev/null` if [ -n "$updaterc" ]; then update-rc.d -f columnstore remove > /dev/null 2>&1 - rm -f /etc/init.d/columnstore > /dev/null 2>&1 + rm -f /etc/init.d/columnstore > /dev/null 2>&1 fi fi fi diff --git a/oam/install_scripts/mariadb-columnstore-start.sh.in b/oam/install_scripts/mariadb-columnstore-start.sh.in index 351512d62..f41b6647d 100644 --- a/oam/install_scripts/mariadb-columnstore-start.sh.in +++ b/oam/install_scripts/mariadb-columnstore-start.sh.in @@ -12,7 +12,6 @@ flock -n "$fd_lock" || exit 0 /bin/systemctl start mcs-controllernode /bin/systemctl start mcs-primproc /bin/systemctl start mcs-writeengineserver -/bin/systemctl start mcs-exemgr /bin/systemctl start mcs-dmlproc /bin/systemctl start mcs-ddlproc su -s /bin/sh -c '@ENGINE_BINDIR@/dbbuilder 7' @DEFAULT_USER@ 1> @ENGINE_LOGDIR@/install/dbbuilder.log diff --git a/oam/install_scripts/mariadb-columnstore-stop.sh b/oam/install_scripts/mariadb-columnstore-stop.sh index 9bce80618..9e1a879ab 100644 --- a/oam/install_scripts/mariadb-columnstore-stop.sh +++ b/oam/install_scripts/mariadb-columnstore-stop.sh @@ -4,7 +4,6 @@ /bin/systemctl stop mcs-dmlproc /bin/systemctl stop mcs-ddlproc -/bin/systemctl stop mcs-exemgr /bin/systemctl stop mcs-writeengineserver /bin/systemctl stop mcs-primproc /bin/systemctl stop mcs-controllernode diff --git a/oam/install_scripts/mcs-exemgr.service.in b/oam/install_scripts/mcs-exemgr.service.in deleted file mode 100644 index 4cbddfc4a..000000000 --- a/oam/install_scripts/mcs-exemgr.service.in +++ /dev/null @@ -1,23 +0,0 @@ -[Unit] -Description=mcs-exemgr - -# restart/start mcs-exemgr on restart/start of mcs-primproc -PartOf=mcs-primproc.service -After=network.target mcs-primproc.service - -[Service] -Type=oneshot - -User=@DEFAULT_USER@ -Group=@DEFAULT_GROUP@ -LimitNOFILE=65536 -LimitNPROC=65536 - -#ExecStartPre=/usr/bin/env bash -c "ldconfig -p | grep -m1 libjemalloc > /dev/null || echo 'Please install jemalloc to avoid ColumnStore performance degradation and unexpected service interruptions.'" -#ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/ExeMgr" -ExecStart=/bin/echo 'EM dummy start' - -RemainAfterExit=yes - -Restart=on-failure -TimeoutStopSec=2 diff --git a/oam/install_scripts/mcs-writeengineserver.service.in b/oam/install_scripts/mcs-writeengineserver.service.in index 32564fc61..2e1470022 100644 --- a/oam/install_scripts/mcs-writeengineserver.service.in +++ b/oam/install_scripts/mcs-writeengineserver.service.in @@ -1,9 +1,9 @@ [Unit] Description=WriteEngineServer -# restart/stop mcs-writeengineserver on restart/stop of mcs-exemgr -PartOf=mcs-exemgr.service -After=network.target mcs-exemgr.service +# restart/stop mcs-writeengineserver on restart/stop of mcs-primproc +PartOf=mcs-primproc.service +After=network.target mcs-primproc.service [Service] Type=forking From 1a2e5412554dfb287e14d2207f0f2df8461f4c63 Mon Sep 17 00:00:00 2001 From: mariadb-RomanNavrotskiy Date: Wed, 13 Apr 2022 21:22:42 +0200 Subject: [PATCH 43/55] set gtest discovery mode to pre_test --- .drone.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index c0ebd5e06..516cad409 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -30,7 +30,7 @@ local builddir = 'verylongdirnameforverystrangecpackbehavior'; local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_XPAND=NO -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO ' + '-DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' + '-DWITH_EMBEDDED_SERVER=OFF -DWITH_WSREP=OFF ' + - '-DBUILD_CONFIG=mysql_release -DWITH_UNITTESTS=YES'; + '-DBUILD_CONFIG=mysql_release -DWITH_UNITTESTS=YES -DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=PRE_TEST'; local gcc_update_alternatives = 'update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 '; From bbb168a846dd3454f33cf1bb835bfd87f9f075ee Mon Sep 17 00:00:00 2001 From: "David.Hall" Date: Mon, 18 Apr 2022 17:00:17 -0500 Subject: [PATCH 44/55] Mcol 4560 (#2337) * MCOL-4560 remove unused xml entries and code that references it. There is reader code and variables for some of these settings, but nobody uses them. --- dbcon/ddlpackage/ddl.y | 2 +- dbcon/dmlpackage/dml.y | 2 +- dbcon/joblist/CMakeLists.txt | 1 - dbcon/joblist/bpp-jl.h | 1 - dbcon/joblist/distributedenginecomm.cpp | 6 +- dbcon/joblist/distributedenginecomm.h | 1 - dbcon/joblist/jlf_common.h | 16 - dbcon/joblist/joblistfactory.cpp | 4 +- dbcon/joblist/pcolscan.cpp | 752 --------------- dbcon/joblist/pcolstep.cpp | 895 ------------------ dbcon/joblist/pdictionary.cpp | 372 -------- dbcon/joblist/pdictionaryscan.cpp | 17 +- dbcon/joblist/primitivemsg.cpp | 74 -- dbcon/joblist/primitivestep.h | 258 +---- dbcon/joblist/resourcemanager.cpp | 107 --- dbcon/joblist/resourcemanager.h | 130 +-- oam/etc/Columnstore.xml | 353 +------ oam/etc/Columnstore.xml.singleserver | 547 ----------- oam/oamcpp/liboamcpp.cpp | 3 - oam/oamcpp/liboamcpp.h | 2 - .../columnstoreSupport/columnstoreSupport.cpp | 28 +- oamapps/columnstoreSupport/mcsSupportUtil.cpp | 14 - primitives/primproc/bpp.h | 1 - primitives/primproc/dictstep.cpp | 1 - primitives/primproc/primproc.cpp | 1 - primitives/primproc/umsocketselector.cpp | 1 - utils/configcpp/tdriver2.cpp | 4 - utils/winport/fixup.cpp | 16 - 28 files changed, 93 insertions(+), 3516 deletions(-) delete mode 100644 dbcon/joblist/primitivemsg.cpp delete mode 100644 oam/etc/Columnstore.xml.singleserver diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index b340fbeaa..c8d36b379 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -95,7 +95,7 @@ void fix_column_length(SchemaObject* elem, const CHARSET_INFO* def_cs) { %} %expect 17 -%pure-parser +%define api.pure %lex-param {void * scanner} %parse-param {struct ddlpackage::pass_to_bison * x} diff --git a/dbcon/dmlpackage/dml.y b/dbcon/dmlpackage/dml.y index 73dcf26ea..3bc62ecb8 100644 --- a/dbcon/dmlpackage/dml.y +++ b/dbcon/dmlpackage/dml.y @@ -90,7 +90,7 @@ char* copy_string(const char *str); } %} -%pure-parser +%define api.pure %lex-param {void * scanner} %parse-param {void * scanner} %debug diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 1bbff02b4..b025005b8 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -36,7 +36,6 @@ set(joblist_LIB_SRCS pcolstep.cpp pdictionary.cpp pdictionaryscan.cpp - primitivemsg.cpp pseudocc-jl.cpp resourcedistributor.cpp resourcemanager.cpp diff --git a/dbcon/joblist/bpp-jl.h b/dbcon/joblist/bpp-jl.h index fc64fae2f..4da0d5e1c 100644 --- a/dbcon/joblist/bpp-jl.h +++ b/dbcon/joblist/bpp-jl.h @@ -28,7 +28,6 @@ // // /** @file */ -// #include "primitivemsg.h" #include "bytestream.h" #include "messagequeue.h" #include "serializeable.h" diff --git a/dbcon/joblist/distributedenginecomm.cpp b/dbcon/joblist/distributedenginecomm.cpp index a960e7374..67d4d7747 100644 --- a/dbcon/joblist/distributedenginecomm.cpp +++ b/dbcon/joblist/distributedenginecomm.cpp @@ -199,7 +199,7 @@ void DistributedEngineComm::reset() } DistributedEngineComm::DistributedEngineComm(ResourceManager* rm, bool isExeMgr) - : fRm(rm), fLBIDShift(fRm->getPsLBID_Shift()), pmCount(0), fIsExeMgr(isExeMgr) +: fRm(rm), pmCount(0), fIsExeMgr(isExeMgr) { Setup(); } @@ -250,10 +250,6 @@ void DistributedEngineComm::Setup() if (newPmCount == 0) writeToLog(__FILE__, __LINE__, "Got a config file with 0 PMs", LOG_TYPE_CRITICAL); - // This needs to make sense when compared to the extent size - // fLBIDShift = static_cast(config::Config::uFromText(fConfig->getConfig(section, - // "LBID_Shift"))); - auto* config = fRm->getConfig(); std::vector pmsAddressesAndPorts; for (size_t i = 1; i <= newPmCount; ++i) diff --git a/dbcon/joblist/distributedenginecomm.h b/dbcon/joblist/distributedenginecomm.h index 51ae885fe..0ec286289 100644 --- a/dbcon/joblist/distributedenginecomm.h +++ b/dbcon/joblist/distributedenginecomm.h @@ -269,7 +269,6 @@ class DistributedEngineComm boost::mutex fMlock; // sessionMessages mutex std::vector > fWlock; // PrimProc socket write mutexes bool fBusy; - unsigned fLBIDShift; volatile uint32_t pmCount; boost::mutex fOnErrMutex; // to lock function scope to reset pmconnections under error condition boost::mutex fSetupMutex; diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index d85c40a4f..61006e9e8 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -185,14 +185,8 @@ struct JobInfo , maxElems(rm->getHjMaxElems()) , flushInterval(rm->getJLFlushInterval()) , fifoSize(rm->getJlFifoSize()) - , fifoSizeLargeSideHj(rm->getHjFifoSizeLargeSide()) - , scanLbidReqLimit(rm->getJlScanLbidReqLimit()) - , scanLbidReqThreshold(rm->getJlScanLbidReqThreshold()) - , tempSaveSize(rm->getScTempSaveSize()) , logger(new Logger()) , traceFlags(0) - , tupleDLMaxSize(rm->getTwMaxSize()) - , tupleMaxBuckets(rm->getTwMaxBuckets()) , projectingTableOID(0) , isExeMgr(false) , trace(false) @@ -226,18 +220,8 @@ struct JobInfo JobStepVectorStack stack; uint32_t flushInterval; uint32_t fifoSize; - uint32_t fifoSizeLargeSideHj; - //...joblist does not use scanLbidReqLimit and SdanLbidReqThreshold. - //...They are actually used by pcolscan and pdictionaryscan, but - //...we have joblist get and report the values here since they - //...are global to the job. - uint32_t scanLbidReqLimit; - uint32_t scanLbidReqThreshold; - uint32_t tempSaveSize; SPJL logger; uint32_t traceFlags; - uint64_t tupleDLMaxSize; - uint32_t tupleMaxBuckets; SErrorInfo errorInfo; execplan::CalpontSystemCatalog::OID* projectingTableOID; // DeliveryWSDLs get a reference to this bool isExeMgr; diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 60985cc8f..26809d394 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -2097,9 +2097,7 @@ SJLP makeJobList_(CalpontExecutionPlan* cplan, ResourceManager* rm, bool isExeMg oss << endl; oss << endl << "job parms: " << endl; oss << "maxBuckets = " << jobInfo.maxBuckets << ", maxElems = " << jobInfo.maxElems - << ", flushInterval = " << jobInfo.flushInterval << ", fifoSize = " << jobInfo.fifoSize - << ", ScanLimit/Threshold = " << jobInfo.scanLbidReqLimit << "/" << jobInfo.scanLbidReqThreshold - << endl; + << ", flushInterval = " << jobInfo.flushInterval << ", fifoSize = " << jobInfo.fifoSize << endl; oss << "UUID: " << jobInfo.uuid << endl; oss << endl << "job filter steps: " << endl; ostream_iterator oIter(oss, "\n"); diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index 6f5d51448..2a3e64fba 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -53,64 +53,6 @@ using namespace execplan; //#define DEBUG 1 //#define DEBUG2 1 -namespace -{ -//// const uint32_t defaultScanLbidReqLimit = 10000; -//// const uint32_t defaultScanLbidReqThreshold = 5000; -// -// struct pColScanStepPrimitive -//{ -// pColScanStepPrimitive(pColScanStep* pColScanStep) : fPColScanStep(pColScanStep) -// {} -// pColScanStep *fPColScanStep; -// void operator()() -// { -// try -// { -// fPColScanStep->sendPrimitiveMessages(); -// } -// catch(std::exception& re) -// { -// string msg = re.what(); -// cerr << "pColScanStep: send thread threw an exception: " << msg << endl; -// -// //Whoa! is this really what we want to do? It's not clear that any good can be had by -// //sticking around, but this seems drastic... -// if (msg.find("there are no primitive processors") != string::npos) -// { -// SPJL logger = fPColScanStep->logger(); -// logger->logMessage(LOG_TYPE_CRITICAL, LogNoPrimProcs, Message::Args(), -//LoggingID(5)); exit(1); -// } -// } -// } -//}; -// -// struct pColScanStepAggregater -//{ -// pColScanStepAggregater(pColScanStep* pColScanStep, uint64_t index) : -// fPColScanStepCol(pColScanStep), fThreadId(index) -// {} -// pColScanStep *fPColScanStepCol; -// uint64_t fThreadId; -// -// void operator()() -// { -// try -// { -// fPColScanStepCol->receivePrimitiveMessages(fThreadId); -// } -// catch(std::exception& re) -// { -// cerr << fPColScanStepCol->toString() << ": receive thread threw an exception: " << -//re.what() << endl; -// } -// } -//}; -// - -} // namespace - namespace joblist { pColScanStep::pColScanStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, @@ -118,20 +60,11 @@ pColScanStep::pColScanStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OI : JobStep(jobInfo) , fRm(jobInfo.rm) , fMsgHeader() - , fNumThreads(fRm->getJlNumScanReceiveThreads()) , fFilterCount(0) , fOid(o) , fTableOid(t) , fColType(ct) , fBOP(BOP_OR) - , sentCount(0) - , recvCount(0) - , fScanLbidReqLimit(fRm->getJlScanLbidReqLimit()) - , fScanLbidReqThreshold(fRm->getJlScanLbidReqThreshold()) - , fStopSending(false) - , fSingleThread(false) - , fPhysicalIO(0) - , fCacheIO(0) , fNumBlksSkipped(0) , fMsgBytesIn(0) , fMsgBytesOut(0) @@ -234,332 +167,6 @@ pColScanStep::pColScanStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OI throw runtime_error("pColScan: Block size and column width must be a power of 2"); } -pColScanStep::~pColScanStep() -{ - // pthread_mutex_destroy(&mutex); - // pthread_mutex_destroy(&dlMutex); - // pthread_mutex_destroy(&cpMutex); - // pthread_cond_destroy(&condvar); - // pthread_cond_destroy(&condvarWakeupProducer); - // delete lbidList; - // delete [] fProducerThread; - // if (fDec) - // fDec->removeQueue(uniqueID); // in case it gets aborted -} - -//------------------------------------------------------------------------------ -// Initialize configurable parameters -//------------------------------------------------------------------------------ -void pColScanStep::initializeConfigParms() -{ - // const string section ( "JobList" ); - // const string sendLimitName ( "ScanLbidReqLimit" ); - // const string sendThresholdName ( "ScanLbidReqThreshold" ); - // const string numReadThreadsName ( "NumScanReceiveThreads" ); - // Config* cf = Config::makeConfig(); - - // string strVal; - - //...Get the tuning parameters that throttle msgs sent to primproc - //...fScanLbidReqLimit puts a cap on how many LBID's we will request from - //... primproc, before pausing to let the consumer thread catch up. - //... Without this limit, there is a chance that PrimProc could flood - //... ExeMgr with thousands of messages that will consume massive - //... amounts of memory for a 100 gigabyte database. - //...fScanLbidReqThreshold is the level at which the number of outstanding - //... LBID reqs must fall below, before the producer can send more LBIDs. - // strVal = cf->getConfig(section, sendLimitName); - // if (strVal.size() > 0) - // fScanLbidReqLimit = static_cast(Config::uFromText(strVal)); - // - // strVal = cf->getConfig(section, sendThresholdName); - // if (strVal.size() > 0) - // fScanLbidReqThreshold = static_cast(Config::uFromText(strVal)); - // - // fNumThreads = 8; - // strVal = cf->getConfig(section, numReadThreadsName); - // if (strVal.size() > 0) - // fNumThreads = static_cast(Config::uFromText(strVal)); - - // fProducerThread = new SPTHD[fNumThreads]; -} - -void pColScanStep::startPrimitiveThread() -{ - // fConsumerThread.reset(new boost::thread(pColScanStepPrimitive(this))); -} - -void pColScanStep::startAggregationThread() -{ - // for (uint32_t i = 0; i < fNumThreads; i++) - // fProducerThread[i].reset(new boost::thread(pColScanStepAggregater(this, i))); -} - -void pColScanStep::run() -{ - // if (traceOn()) - // { - // syslogStartStep(16, // exemgr subsystem - // std::string("pColScanStep")); // step name - // } - // - // //"consume" input datalist. In this case, there is no IDL, we just send one primitive - // startPrimitiveThread(); - // //produce output datalist - // //Don't start this yet...see below - // //startAggregationThread(); -} - -void pColScanStep::join() -{ - // fConsumerThread->join(); - // if ( !fSingleThread ) { - // for (uint32_t i = 0; i < fNumThreads; i++) - // fProducerThread[i]->join(); - // } -} - -void pColScanStep::sendPrimitiveMessages() -{ - // //The presence of an input DL means we (probably) have a pDictionaryScan step feeding this scan step - // // a list of tokens to get the rids for. Convert the input tokens to a filter string. - // if (fInputJobStepAssociation.outSize() > 0) - // { - // addFilters(); - // if (fTableOid >= 3000) - // cout << toString() << endl; - // //If we got no input rids (as opposed to no input DL at all) then there were no matching rows from - // // the previous step, so this step should not return any rows either. This would be the case, for - // // instance, if P_NAME LIKE '%xxxx%' produced no signature matches. - // if (fFilterCount == 0) { - // rDoNothing=true; - // startAggregationThread(); - // return; - // } - // } - // - // startAggregationThread(); - // - // /* for all the blocks that need to be sent - // * build out a message for each block with the primitive message header filled - // * out and all the NOPS and BOP structures as well. - // * Then serialize it to a BytStream and then send it on its way - // */ - // - // LBIDRange_v::iterator it; - // uint64_t fbo; - // - // ISMPacketHeader ism; - // ism.Flags = planFlagsToPrimFlags(fTraceFlags); - // ism.Command=COL_BY_SCAN_RANGE; - // ism.Size = sizeof(ISMPacketHeader) + sizeof(ColByScanRangeRequestHeader) + fFilterString.length(); - // ism.Type=2; - // //bool firstWrite = true; - // - // //...Counter used to track the number of LBIDs we are requesting from - // //...primproc in the current set of msgs, till we reach fScanLbidReqLimit - // uint32_t runningLbidCount = 0; - // bool exitLoop = false; - // const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); - // - // for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - // { - // BRM::LBID_t lbid = (*it).start; - // - // fbo = getFBO(lbid); - // if (hwm < fbo) - // continue; - // - // if (fOid >= 3000 && lbidList->CasualPartitionDataType(fColType.colDataType, fColType.colWidth) ) - // { - // int64_t Min=0; - // int64_t Max=0; - // int64_t SeqNum=0; - // bool MinMaxValid=true; - // bool cpPredicate=true; - // - // // can we consolidate these crit sections? - // cpMutex.lock(); //pthread_mutex_lock(&cpMutex); - // MinMaxValid = lbidList->GetMinMax(Min, Max, SeqNum, lbid, 0); - // - // if (MinMaxValid) - // { - // cpPredicate=lbidList->CasualPartitionPredicate(Min, - // Max, - // &fFilterString, - // fFilterCount, - // fColType, - // fBOP) || - //ignoreCP; - // } - // cpMutex.unlock(); //pthread_mutex_unlock(&cpMutex); - // - // if (cpPredicate==false){ //don't scan this extent - //#ifdef DEBUG - // cout << "Scan Skip " << lbid << endl; - //#endif - // //...Track the number of LBIDs we skip due to Casual Partioning. - // //...We use the same equation we use to initialize remainingLbids - // //...in the code that follows down below this. - // fNumBlksSkipped += ( (hwm > (fbo + it->size - 1)) ? - // (it->size) : (hwm - fbo + 1) ); - // continue; - // } - //#ifdef DEBUG - // else - // cout << "Scan " << lbid << endl; - //#endif - // - // } - // - // LBID_t msgLbidStart = it->start; - // uint32_t remainingLbids = - // ( (hwm > (fbo + it->size - 1)) ? (it->size) : (hwm - fbo + 1) ); - // uint32_t msgLbidCount = 0; - // - // while ( remainingLbids > 0 ) - // { - // //...Break up this range of LBIDs when we reach the msg size - // //...limit for one request (fScanLbidReqLimit) - // if ( (runningLbidCount + remainingLbids) >= fScanLbidReqLimit ) - // { - // msgLbidCount = fScanLbidReqLimit - runningLbidCount; - // sendAPrimitiveMessage(ism, msgLbidStart, msgLbidCount ); - // //...Wait for the consuming thread to catch up if our - // //...backlog of work is >= the allowable threshold limit - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // sentCount += msgLbidCount; - // if (recvWaiting) - // condvar.notify_all(); //pthread_cond_broadcast(&condvar); //signal consumer to - //resume - // - // while ( ((sentCount - recvCount) >= fScanLbidReqThreshold) - // && !fStopSending ) - // { - // sendWaiting = true; - //#ifdef DEBUG2 - // if (fOid >= 3000) - // cout << "pColScanStep producer WAITING: " << - // "st:" << fStepId << - // "; sentCount-" << sentCount << - // "; recvCount-" << recvCount << - // "; threshold-" << fScanLbidReqThreshold << endl; - //#endif - // condvarWakeupProducer.wait(mutex); //pthread_cond_wait ( &condvarWakeupProducer, &mutex - //); #ifdef DEBUG2 if (fOid >= 3000) cout << "pColScanStep producer RESUMING: " << "st:" << fStepId << endl; - //#endif - // sendWaiting = false; - // } - // - // //...Set flag to quit if consumer thread tells us to - // if (fStopSending) - // exitLoop = true; - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // runningLbidCount = 0; - // } - // else - // { - // msgLbidCount = remainingLbids; - // - // sendAPrimitiveMessage(ism, msgLbidStart, msgLbidCount ); - // mutex.lock(); //pthread_mutex_lock(&mutex); - // sentCount += msgLbidCount; - // if (recvWaiting) - // condvar.notify_all(); //pthread_cond_broadcast(&condvar); //signal consumer to - //resume - // - // //...Set flag to quit if consumer thread tells us to - // if (fStopSending) - // exitLoop = true; - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // runningLbidCount += msgLbidCount; - // } - // - // //...If consuming thread has quit, then we should do the same. - // //...This can happen if consuming thread receives empty ByteStream - // if (exitLoop) - // break; - // - // remainingLbids -= msgLbidCount; - // msgLbidStart += msgLbidCount; - // } - // - // if (exitLoop) - // break; - // - // } // end of loop through LBID ranges to be requested from primproc - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // finishedSending = true; - // if (recvWaiting) - // condvar.notify_all(); //pthread_cond_broadcast(&condvar); - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - //// cerr << "send side exiting" << endl; - // - //#ifdef DEBUG2 - // if (fOid >= 3000) - // { - // time_t t = time(0); - // char timeString[50]; - // ctime_r(&t, timeString); - // timeString[strlen(timeString)-1 ] = '\0'; - // cout << "pColScanStep Finished sending primitives for: " << - // fOid << " at " << timeString << endl; - // } - //#endif - // -} - -//------------------------------------------------------------------------------ -// Construct and send a single primitive message to primproc -//------------------------------------------------------------------------------ -void pColScanStep::sendAPrimitiveMessage(ISMPacketHeader& ism, BRM::LBID_t msgLbidStart, - uint32_t msgLbidCount) -{ - // ByteStream bs; - // - // bs.load(reinterpret_cast(&ism), sizeof(ism)); - // - // fMsgHeader.LBID = msgLbidStart; - // fMsgHeader.DataSize = fColType.colWidth; - // fMsgHeader.DataType = fColType.colDataType; - // fMsgHeader.CompType = fColType.compressionType; - // if (fFilterCount > 0) - // fMsgHeader.OutputType = 3; // pairs - // else - // fMsgHeader.OutputType = OT_DATAVALUE; - // fMsgHeader.BOP = fBOP; - // fMsgHeader.NOPS = fFilterCount; - // fMsgHeader.NVALS = 0; - // fMsgHeader.Count = msgLbidCount; - // fMsgHeader.Hdr.SessionID = fSessionId; - // //fMsgHeader.Hdr.StatementID = 0; - // fMsgHeader.Hdr.TransactionID = fTxnId; - // fMsgHeader.Hdr.VerID = fVerId; - // fMsgHeader.Hdr.StepID = fStepId; - // fMsgHeader.Hdr.UniqueID = uniqueID; - // - // bs.append(reinterpret_cast(&fMsgHeader), - // sizeof(fMsgHeader)); - // bs += fFilterString; - // - //#ifdef DEBUG2 - // if (fOid >= 3000) - // cout << "pColScanStep producer st: " << fStepId << - // ": sending req for lbid start " << msgLbidStart << - // "; lbid count " << msgLbidCount << endl; - //#endif - // - // fMsgBytesOut += bs.lengthWithHdrOverhead(); - // fDec->write(bs); - // fMsgsToPm++; -} - struct CPInfo { CPInfo(int64_t MIN, int64_t MAX, uint64_t l) : min(MIN), max(MAX), LBID(l){}; @@ -568,327 +175,6 @@ struct CPInfo uint64_t LBID; }; -void pColScanStep::receivePrimitiveMessages(uint64_t tid) -{ - // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - // DataList_t* dlp = dl->dataList(); - // FifoDataList *fifo = dl->fifoDL(); - // BucketDL *bucket = dynamic_cast *>(dlp); - // ZDL *zdl = dynamic_cast *>(dlp); - // int64_t l_ridsReturned = 0; - // uint64_t l_physicalIO = 0, l_cachedIO = 0; - // uint64_t fbo; - // uint64_t ridBase; - // vector v; - // UintRowGroup rw; - // vector > bsv; - // uint32_t i, k, size, bsLength; - // bool lastThread = false; - // vector cpv; - // - // if (bucket || zdl) - // dlp->setMultipleProducers(true); - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // - // // count the LBIDs - // for (; !rDoNothing; ) { - // - // // sync with the send side - // while (!finishedSending && sentCount == recvCount) { - // recvWaiting++; - // condvar.wait(mutex); //pthread_cond_wait(&condvar, &mutex); - // recvWaiting--; - // } - // if (sentCount == recvCount && finishedSending) { - //// cout << "done recving" << endl; - // break; - // } - // - // fDec->read_some(uniqueID, fNumThreads, bsv); - // for (unsigned int jj=0; jjlengthWithHdrOverhead(); - // } - // - // - // size = bsv.size(); - // - // if (size == 0) { - // /* XXXPAT: Need to give other threads a chance to update recvCount. - // As of 2/25/08, each BS contains multiple responses. With the current - // protocol, the exact # isn't known until they're processed. Threads - // waiting for more input will have to busy wait on either recvCount - // being updated or input. It's only an issue at the tail end of the - // responses, and probably doesn't matter then either. */ - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // usleep(1000); // 1ms Good? - // mutex.lock(); //pthread_mutex_lock(&mutex); - // continue; - // } - // - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) - // dlTimes.setFirstReadTime(); - // if (fOid>=3000) dlTimes.setLastReadTime(); - // - //// cerr << "got a response of " << size << " msgs\n"; - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // uint32_t msgCount = 0; - // for (i = 0; i < size; i++) { - // const ByteStream::byte* bsp = bsv[i]->buf(); - // - // bsLength = bsv[i]->length(); - // k = 0; - // while (k < bsLength) { - // ++msgCount; - //// cout << "got msg " << msgCount << " k = " << k << endl; - // k += sizeof(ISMPacketHeader); - // const ColResultHeader* crh = reinterpret_cast(&bsp[k]); - // // get the ColumnResultHeader out of the bytestream - // k += sizeof(ColResultHeader); - // - // l_cachedIO += crh->CacheIO; - // l_physicalIO += crh->PhysicalIO; - // fbo = getFBO(crh->LBID); - // ridBase = fbo << rpbShift; - // - // for(int j = 0; j < crh->NVALS; j++) - // { - // uint64_t dv; - // uint64_t rid; - // - // if (crh->OutputType == OT_DATAVALUE) { - // if (isEmptyVal(&bsp[k])) { - // k += fColType.colWidth; - // continue; - // } - // rid = j + ridBase; - // } - // else { - // rid = *((const uint16_t *) &bsp[k]) + ridBase; - // k += sizeof(uint16_t); - // } - // - // switch (fColType.colWidth) { - // case 8: dv = *((const uint64_t *) &bsp[k]); k += 8; break; - // case 4: dv = *((const uint32_t *) &bsp[k]); k += 4; break; - // case 2: dv = *((const uint16_t *) &bsp[k]); k += 2; break; - // case 1: dv = *((const uint8_t *) &bsp[k]); ++k; break; - // default: - // throw runtime_error("pColStep: invalid column - //width!"); - // } - // - // v.push_back(ElementType(rid, dv)); - // ++l_ridsReturned; - //#ifdef DEBUG - //// if (fOid >=3000) - //// cout << " -- inserting <" << rid << ", " << dv << ">" << - ///endl; - //#endif - // // per row operations... - // } // for - // - // // per block operations... - // - //#ifdef DEBUG - // cout << "recvPrimMsgs Oid " << fOid - // << " valid " << crh->ValidMinMax - // << " LBID " << crh->LBID - // << " Mn/Mx " << crh->Min << "/" << crh->Max - // << " nvals " << crh->NVALS - // << "/" << l_ridsReturned << endl; - //#endif - // if (fOid >= 3000 && crh->ValidMinMax) - // cpv.push_back(CPInfo(crh->Min, crh->Max, crh->LBID)); - // } - // // per ByteStream operations... - // } - // // per read operations.... - // - // if (bucket) { - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // - // bucket->insert(v); - // } - // else if (zdl) { - // zdl->insert(v); - // } - // else { - // size = v.size(); - // if (size>0) - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // - // dlMutex.lock(); //pthread_mutex_lock(&dlMutex); - // for (i = 0; i < size; ++i) { - // rw.et[rw.count++] = v[i]; - // if (rw.count == rw.ElementsPerGroup) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // - // if (rw.count > 0) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // - // dlMutex.unlock(); //pthread_mutex_unlock(&dlMutex); - // } - // v.clear(); - // - // size = cpv.size(); - // if (size > 0) { - // cpMutex.lock(); //pthread_mutex_lock(&cpMutex); - // for (i = 0; i < size; i++) { - // CPInfo *cpi = &(cpv[i]); - // lbidList->UpdateMinMax(cpi->min, cpi->max, cpi->LBID, fColType.colDataType); - // } - // cpMutex.unlock(); //pthread_mutex_unlock(&cpMutex); - // cpv.clear(); - // } - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // recvCount += msgCount; - // //...If producer is waiting, and we have gone below our threshold value, - // //...then we signal the producer to request more data from primproc - // if ( (sendWaiting) && ( (sentCount - recvCount) < fScanLbidReqThreshold ) ) - // { - //#ifdef DEBUG2 - // if (fOid >= 3000) - // cout << "pColScanStep consumer signaling producer for more data: "<< - // "st:" << fStepId << - // "; sentCount-" << sentCount << - // "; recvCount-" << recvCount << - // "; threshold-" << fScanLbidReqThreshold << endl; - //#endif - // condvarWakeupProducer.notify_one(); //pthread_cond_signal(&condvarWakeupProducer); - // } - // } // end of loop to read LBID responses from primproc - // - // fPhysicalIO += l_physicalIO; - // fCacheIO += l_cachedIO; - // ridsReturned += l_ridsReturned; - //// cerr << "out of the main loop " << recvExited << endl; - // if (++recvExited == fNumThreads) { - // //...Casual partitioning could cause us to do no processing. In that - // //...case these time stamps did not get set. So we set them here. - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // dlTimes.setLastReadTime(); - // dlTimes.setFirstInsertTime(); - // } - // if (fOid>=3000) dlTimes.setEndOfInputTime(); - // - // //@bug 699: Reset StepMsgQueue - // fDec->removeQueue(uniqueID); - // - // if (fifo) - // fifo->endOfInput(); - // else - // dlp->endOfInput(); - // lastThread = true; - // } - // - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // if (fTableOid >= 3000 && lastThread) - // { - // //...Construct timestamp using ctime_r() instead of ctime() not - // //...necessarily due to re-entrancy, but because we want to strip - // //...the newline ('\n') off the end of the formatted string. - // time_t t = time(0); - // char timeString[50]; - // ctime_r(&t, timeString); - // timeString[strlen(timeString)-1 ] = '\0'; - // - // FifoDataList* pFifo = 0; - // uint64_t totalBlockedReadCount = 0; - // uint64_t totalBlockedWriteCount = 0; - // - // //...Sum up the blocked FIFO reads for all input associations - // size_t inDlCnt = fInputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedReadCount += pFifo->blockedReadCount(); - // } - // } - // - // //...Sum up the blocked FIFO writes for all output associations - // size_t outDlCnt = fOutputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedWriteCount += pFifo->blockedWriteCount(); - // } - // } - // - // //...Roundoff inbound msg byte count to nearest KB for display; - // //...no need to do so for outbound, because it should be small. - // uint64_t msgBytesInKB = fMsgBytesIn >> 10; - // if (fMsgBytesIn & 512) - // msgBytesInKB++; - // // @bug 807 - // if (fifo) - // fifo->totalSize(ridsReturned); - // - // if (traceOn()) - // { - // //...Print job step completion information - // ostringstream logStr; - // logStr << "ses:" << fSessionId << - // " st: " << fStepId << " finished at " << - // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << - // fCacheIO << "; MsgsSent-" << fMsgsToPm << "; MsgsRcvd-" << recvCount << - // "; BlockedFifoIn/Out-" << totalBlockedReadCount << - // "/" << totalBlockedWriteCount << - // "; output size-" << ridsReturned << endl << - // "\tPartitionBlocksEliminated-" << fNumBlksSkipped << - // "; MsgBytesIn-" << msgBytesInKB << "KB" << - // "; MsgBytesOut-" << fMsgBytesOut << "B" << endl << - // "\t1st read " << dlTimes.FirstReadTimeString() << - // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << - // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << - // "s" << endl; - // - // logEnd(logStr.str().c_str()); - // - // syslogReadBlockCounts(16, // exemgr subsystem - // fPhysicalIO, // # blocks read from disk - // fCacheIO, // # blocks read from cache - // fNumBlksSkipped); // # casual partition block hits - // syslogProcessingTimes(16, // exemgr subsystem - // dlTimes.FirstReadTime(), // first datalist read - // dlTimes.LastReadTime(), // last datalist read - // dlTimes.FirstInsertTime(), // first datalist write - // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write - // syslogEndStep(16, // exemgr subsystem - // totalBlockedReadCount, // blocked datalist input - // totalBlockedWriteCount, // blocked datalist output - // fMsgBytesIn, // incoming msg byte count - // fMsgBytesOut); // outgoing msg byte count - // } - // - // } - // - // if (fOid >=3000 && lastThread) - // lbidList->UpdateAllPartitionInfo(); - // - //// cerr << "recv thread exiting" << endl; -} - void pColScanStep::addFilter(int8_t COP, float value) { fFilterString << (uint8_t)COP; @@ -937,22 +223,6 @@ void pColScanStep::addFilter(int8_t COP, int64_t value, uint8_t roundFlag) fFilterCount++; } -void pColScanStep::setBOP(int8_t B) -{ - fBOP = B; -} - -void pColScanStep::setSingleThread(bool b) -{ - fSingleThread = b; - fNumThreads = 1; -} - -void pColScanStep::setOutputType(int8_t OutputType) -{ - fOutputType = OutputType; -} - const string pColScanStep::toString() const { ostringstream oss; @@ -983,10 +253,7 @@ uint64_t pColScanStep::getFBO(uint64_t lbid) { lastLBID = extents[i].range.start + (extents[i].range.size << 10) - 1; - // lastLBID = extents[i].range.start + (extents[i].range.size * 1024) - 1; - // cerr << "start: " << extents[i].range.start << " end:" << lastLBID <= (uint64_t)extents[i].range.start && lbid <= lastLBID) - // return (lbid - extents[i].range.start) + (extentSize * i); return (lbid - extents[i].range.start) + (i << divShift); } @@ -996,7 +263,6 @@ uint64_t pColScanStep::getFBO(uint64_t lbid) pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resourceManager()), fMsgHeader() { - fNumThreads = fRm->getJlNumScanReceiveThreads(); fFilterCount = rhs.filterCount(); fFilterString = rhs.filterString(); isFilterFeeder = rhs.getFeederFlag(); @@ -1005,14 +271,6 @@ pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resource fColType = rhs.colType(); fBOP = rhs.BOP(); fIsDict = rhs.isDictCol(); - sentCount = 0; - recvCount = 0; - fScanLbidReqLimit = fRm->getJlScanLbidReqLimit(); - fScanLbidReqThreshold = fRm->getJlScanLbidReqThreshold(); - fStopSending = false; - fSingleThread = false; - fPhysicalIO = 0; - fCacheIO = 0; fNumBlksSkipped = 0; fMsgBytesIn = 0; fMsgBytesOut = 0; @@ -1040,11 +298,6 @@ pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resource numExtents = extents.size(); extentSize = (fRm->getExtentRows() * fColType.colWidth) / BLOCK_SIZE; lbidList = rhs.lbidList; - // pthread_mutex_init(&mutex, NULL); - // pthread_mutex_init(&dlMutex, NULL); - // pthread_mutex_init(&cpMutex, NULL); - // pthread_cond_init(&condvar, NULL); - // pthread_cond_init(&condvarWakeupProducer, NULL); finishedSending = sendWaiting = rDoNothing = false; recvWaiting = 0; recvExited = 0; @@ -1054,12 +307,7 @@ pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), fRm(rhs.resource rpbShift = rhs.rpbShift; divShift = rhs.divShift; - // initializeConfigParms ( ); fTraceFlags = rhs.fTraceFlags; - // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - // if (fDec) - // fDec->addQueue(uniqueID); - // fProducerThread = new SPTHD[fNumThreads]; } void pColScanStep::addFilters() diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index 125491ebd..3a314a7d6 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -58,47 +58,6 @@ using namespace BRM; namespace joblist { -#if 0 -//const uint32_t defaultProjectBlockReqLimit = 32768; -//const uint32_t defaultProjectBlockReqThreshold = 16384; -struct pColStepPrimitive -{ - pColStepPrimitive(pColStep* pColStep) : fPColStep(pColStep) - {} - pColStep* fPColStep; - void operator()() - { - try - { - fPColStep->sendPrimitiveMessages(); - } - catch (exception& re) - { - cerr << "pColStep: send thread threw an exception: " << re.what() << - "\t" << this << endl; - } - } -}; - -struct pColStepAggregator -{ - pColStepAggregator(pColStep* pColStep) : fPColStepCol(pColStep) - {} - pColStep* fPColStepCol; - void operator()() - { - try - { - fPColStepCol->receivePrimitiveMessages(); - } - catch (exception& re) - { - cerr << fPColStepCol->toString() << ": recv thread threw an exception: " << re.what() << endl; - } - } -}; -#endif - pColStep::pColStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, const CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo) : JobStep(jobInfo) @@ -117,14 +76,8 @@ pColStep::pColStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, , fIsDict(false) , isEM(jobInfo.isExeMgr) , ridCount(0) - , fFlushInterval(jobInfo.flushInterval) , fSwallowRows(false) - , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) - , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) - , fStopSending(false) , isFilterFeeder(false) - , fPhysicalIO(0) - , fCacheIO(0) , fNumBlksSkipped(0) , fMsgBytesIn(0) , fMsgBytesOut(0) @@ -135,11 +88,6 @@ pColStep::pColStep(CalpontSystemCatalog::OID o, CalpontSystemCatalog::OID t, int err, i; uint32_t mask; - if (fFlushInterval == 0 || !isEM) - fOutputType = OT_BOTH; - else - fOutputType = OT_TOKEN; - if (fOid < 1000) throw runtime_error("pColStep: invalid column"); @@ -279,15 +227,7 @@ pColStep::pColStep(const pColScanStep& rhs) , recvWaiting(false) , fIsDict(rhs.isDictCol()) , ridCount(0) - , - // Per Cindy, it's save to put fFlushInterval to be 0 - fFlushInterval(0) , fSwallowRows(false) - , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) - , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) - , fStopSending(false) - , fPhysicalIO(0) - , fCacheIO(0) , fNumBlksSkipped(0) , fMsgBytesIn(0) , fMsgBytesOut(0) @@ -390,15 +330,7 @@ pColStep::pColStep(const PassThruStep& rhs) , recvWaiting(false) , fIsDict(rhs.isDictCol()) , ridCount(0) - , - // Per Cindy, it's save to put fFlushInterval to be 0 - fFlushInterval(0) , fSwallowRows(false) - , fProjectBlockReqLimit(fRm->getJlProjectBlockReqLimit()) - , fProjectBlockReqThreshold(fRm->getJlProjectBlockReqThreshold()) - , fStopSending(false) - , fPhysicalIO(0) - , fCacheIO(0) , fNumBlksSkipped(0) , fMsgBytesIn(0) , fMsgBytesOut(0) @@ -474,107 +406,6 @@ pColStep::pColStep(const PassThruStep& rhs) sort(extents.begin(), extents.end(), ExtentSorter()); numExtents = extents.size(); - // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - // if (fDec) - // fDec->addQueue(uniqueID); - // initializeConfigParms ( ); -} - -pColStep::~pColStep() -{ - // join? - // delete lbidList; - // if (fDec) - // fDec->removeQueue(uniqueID); -} - -//------------------------------------------------------------------------------ -// Initialize configurable parameters -//------------------------------------------------------------------------------ -void pColStep::initializeConfigParms() -{ - // const string section ( "JobList" ); - // const string sendLimitName ( "ProjectBlockReqLimit" ); - // const string sendThresholdName ( "ProjectBlockReqThreshold" ); - // Config* cf = Config::makeConfig(); - // - // string strVal; - // uint64_t numVal; - - //...Get the tuning parameters that throttle msgs sent to primproc - //...fFilterRowReqLimit puts a cap on how many rids we will request from - //... primproc, before pausing to let the consumer thread catch up. - //... Without this limit, there is a chance that PrimProc could flood - //... ExeMgr with thousands of messages that will consume massive - //... amounts of memory for a 100 gigabyte database. - //...fFilterRowReqThreshhold is the level at which the number of outstanding - //... rids must fall below, before the producer can send more rids. - - // strVal = cf->getConfig(section, sendLimitName); - // if (strVal.size() > 0) - // { - // errno = 0; - // numVal = Config::uFromText(strVal); - // if ( errno == 0 ) - // fProjectBlockReqLimit = (uint32_t)numVal; - // } - // - // strVal = cf->getConfig(section, sendThresholdName); - // if (strVal.size() > 0) - // { - // errno = 0; - // numVal = Config::uFromText(strVal); - // if ( errno == 0 ) - // fProjectBlockReqThreshold = (uint32_t)numVal; - // } -} - -void pColStep::startPrimitiveThread() -{ - // pThread.reset(new boost::thread(pColStepPrimitive(this))); -} - -void pColStep::startAggregationThread() -{ - // cThread.reset(new boost::thread(pColStepAggregator(this))); -} - -void pColStep::run() -{ - // if (traceOn()) - // { - // syslogStartStep(16, // exemgr subsystem - // std::string("pColStep")); // step name - // } - // - // size_t sz = fInputJobStepAssociation.outSize(); - // idbassert(sz > 0); - // const AnyDataListSPtr& dl = fInputJobStepAssociation.outAt(0); - // DataList_t* dlp = dl->dataList(); - // DataList* strDlp = dl->stringDataList(); - // if ( dlp ) - // setRidList(dlp); - // else - // { - // setStrRidList( strDlp ); - // } - // //Sort can be set through the jobstep or the input JSA if fFlushinterval is 0 - // fToSort = (fFlushInterval) ? 0 : (!fToSort) ? fInputJobStepAssociation.toSort() : fToSort; - // fToSort = 0; - // //pthread_mutex_init(&mutex, NULL); - // //pthread_cond_init(&condvar, NULL); - // //pthread_cond_init(&flushed, NULL); - // startPrimitiveThread(); - // startAggregationThread(); -} - -void pColStep::join() -{ - // pThread->join(); - // cThread->join(); - // //pthread_mutex_destroy(&mutex); - // //pthread_cond_destroy(&condvar); - // //pthread_cond_destroy(&flushed); } void pColStep::addFilter(int8_t COP, float value) @@ -636,732 +467,6 @@ void pColStep::addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag) fFilterCount++; } -void pColStep::setRidList(DataList* dl) -{ - ridList = dl; -} - -void pColStep::setStrRidList(DataList* strDl) -{ - strRidList = strDl; -} - -void pColStep::setBOP(int8_t b) -{ - fBOP = b; -} - -void pColStep::setOutputType(int8_t OutputType) -{ - fOutputType = OutputType; -} - -void pColStep::setSwallowRows(const bool swallowRows) -{ - fSwallowRows = swallowRows; -} - -void pColStep::sendPrimitiveMessages() -{ - // int it = -1; - // int msgRidCount = 0; - // int ridListIdx = 0; - // bool more = false; - // uint64_t absoluteRID = 0; - // int64_t msgLBID = -1; - // int64_t nextLBID = -1; - // int64_t msgLargeBlock = -1; - // int64_t nextLargeBlock = -1; - // uint16_t blockRelativeRID; - // uint32_t msgCount = 0; - // uint32_t sentBlockCount = 0; - // int msgsSkip=0; - // bool scan=false; - // bool scanThisBlock=false; - // ElementType e; - // UintRowGroup rw; - // StringElementType strE; - // StringRowGroup strRw; - // - // ByteStream msgRidList; - // ByteStream primMsg(MAX_BUFFER_SIZE); //the MAX_BUFFER_SIZE as of 8/20 - // - // NewColRequestHeader hdr; - // - // AnyDataListSPtr dl; - // FifoDataList *fifo = NULL; - // StringFifoDataList* strFifo = NULL; - // - // const bool ignoreCP = ((fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP) != 0); - // - // //The presence of more than 1 input DL means we (probably) have a pDictionaryScan step feeding this - //step - // // a list of tokens to get the rids for. Convert the input tokens to a filter string. We also have a - //rid - // // list as the second input dl - // if (fInputJobStepAssociation.outSize() > 1) - // { - // addFilters(); - // if (fTableOid >= 3000) - // cout << toString() << endl; - // //If we got no input rids (as opposed to no input DL at all) then there were no matching rows - //from - // // the previous step, so this step should not return any rows either. This would be the case, - //for - // // instance, if P_NAME LIKE '%xxxx%' produced no signature matches. - // if (fFilterCount == 0) - // { - // goto done; - // } - // } - // - // // determine which ranges/extents to eliminate from this step - // - //#ifdef DEBUG - // if (fOid>=3000) - // cout << "oid " << fOid << endl; - //#endif - // - // scanFlags.resize(numExtents); - // - // for (uint32_t idx=0; idx CasualPartitionPredicate( - // extents[idx].partition.cprange.loVal, - // extents[idx].partition.cprange.hiVal, - // &fFilterString, - // fFilterCount, - // fColType, - // fBOP) || ignoreCP; - // scanFlags[idx]=flag; - //#ifdef DEBUG - // if (fOid >= 3000 && flushInterval == 0) - // cout << (flag ? " will scan " : " will not scan ") - // << "extent with range " << extents[idx].partition.cprange.loVal - // << "-" << extents[idx].partition.cprange.hiVal << endl; - //#endif - // - // } - // - //// if (fOid>=3000) - //// cout << " " << scanFlags[idx]; - // } - //// if (scanFlags.size()>0) - //// cout << endl; - // - // // If there was more than 1 input DL, the first is a list of filters and the second is a list of rids, - // // otherwise the first is the list of rids. - // if (fInputJobStepAssociation.outSize() > 1) - // ridListIdx = 1; - // else - // ridListIdx = 0; - // - // dl = fInputJobStepAssociation.outAt(ridListIdx); - // ridList = dl->dataList(); - // if ( ridList ) - // { - // fifo = dl->fifoDL(); - // - // if (fifo) - // it = fifo->getIterator(); - // else - // it = ridList->getIterator(); - // } - // else - // { - // strRidList = dl->stringDataList(); - // strFifo = dl->stringDL(); - // - // if (strFifo) - // it = strFifo->getIterator(); - // else - // it = strRidList->getIterator(); - // } - // - // if (ridList) - // { - // if (fifo) - // { - // more = fifo->next(it, &rw); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // } - // absoluteRID = rw.et[0].first; - // } - // else - // { - // more = ridList->next(it, &e); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // } - // absoluteRID = e.first; - // rw.count = 1; - // } - // } - // else - // { - // if (strFifo) - // { - // more = strFifo->next(it, &strRw); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // } - // absoluteRID = strRw.et[0].first; - // } - // else - // { - // more = strRidList->next(it, &strE); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // } - // absoluteRID = strE.first; - // strRw.count = 1; - // } - // } - // - // if (more) - // msgLBID = getLBID(absoluteRID, scan); - // scanThisBlock = scan; - // msgLargeBlock = absoluteRID >> blockSizeShift; - // - // while (more || msgRidCount > 0) { - // uint64_t rwCount; - // if ( ridList) - // rwCount = rw.count; - // else - // rwCount = strRw.count; - // - // for (uint64_t i = 0; ((i < rwCount) || (!more && msgRidCount > 0)); ) - // { - // if ( ridList) - // { - // if (fifo) - // absoluteRID = rw.et[i].first; - // else - // absoluteRID = e.first; - // } - // else - // { - // if (strFifo) - // absoluteRID = strRw.et[i].first; - // else - // absoluteRID = strE.first; - // } - // - // if (more) { - // nextLBID = getLBID(absoluteRID, scan); - // nextLargeBlock = absoluteRID >> blockSizeShift; - // } - // - // //XXXPAT: need to prove N & S here - // if (nextLBID == msgLBID && more) { - //// blockRelativeRID = absoluteRID % ridsPerBlock; - // blockRelativeRID = absoluteRID & rpbMask; - // msgRidList << blockRelativeRID; - // msgRidCount++; - // ++i; - // } - // else { - // //Bug 831: move building msg after the check of scanThisBlock - // if (scanThisBlock==true) - // { - // hdr.ism.Interleave=0; - // hdr.ism.Flags=planFlagsToPrimFlags(fTraceFlags); - // hdr.ism.Command=COL_BY_SCAN; - // hdr.ism.Size=sizeof(NewColRequestHeader) + fFilterString.length() + - // msgRidList.length(); - // hdr.ism.Type=2; - // - // hdr.hdr.SessionID = fSessionId; - // //hdr.hdr.StatementID = 0; - // hdr.hdr.TransactionID = fTxnId; - // hdr.hdr.VerID = fVerId; - // hdr.hdr.StepID = fStepId; - // hdr.hdr.UniqueID = uniqueID; - // - // hdr.LBID = msgLBID; - //// idbassert(hdr.LBID >= 0); - // hdr.DataSize = fColType.colWidth; - // hdr.DataType = fColType.colDataType; - // hdr.CompType = fColType.compressionType; - // hdr.OutputType = fOutputType; - // hdr.BOP = fBOP; - // hdr.NOPS = fFilterCount; - // hdr.NVALS = msgRidCount; - // hdr.sort = fToSort; - // - // primMsg.append((const uint8_t *) &hdr, sizeof(NewColRequestHeader)); - // primMsg += fFilterString; - // primMsg += msgRidList; - // ridCount += msgRidCount; - // ++sentBlockCount; - // - //#ifdef DEBUG - // if (flushInterval == 0 && fOid >= 3000) - // cout << "sending a prim msg for LBID " << msgLBID << endl; - //#endif - // ++msgCount; - //// cout << "made a primitive\n"; - // if (msgLargeBlock != nextLargeBlock || !more) { - //// cout << "writing " << msgCount << " primitives\n"; - // fMsgBytesOut += primMsg.lengthWithHdrOverhead(); - // fDec->write(primMsg); - // msgsSent += msgCount; - // msgCount = 0; - // primMsg.restart(); - // msgLargeBlock = nextLargeBlock; - // - // // @bug 769 - Added "&& !fSwallowRows" condition below to fix problem - //with - // // caltraceon(16) not working for tpch01 and some other queries. If a - //query - // // ever held off requesting more blocks, it would lock and never finish. - // //Bug 815 - // if (( sentBlockCount >= fProjectBlockReqLimit) && !fSwallowRows - //&& - // (( msgsSent - msgsRecvd) > fProjectBlockReqThreshold)) - // { - // mutex.lock(); //pthread_mutex_lock(&mutex); - // fStopSending = true; - // - // // @bug 836. Wake up the receiver if he's sleeping. - // if (recvWaiting) - // condvar.notify_one(); - ////pthread_cond_signal(&condvar); flushed.wait(mutex); //pthread_cond_wait(&flushed, &mutex); fStopSending - //= false; mutex.unlock(); //pthread_mutex_unlock(&mutex); sentBlockCount = 0; - // } - // } - // } - // else - // { - // msgsSkip++; - // } - // msgLBID = nextLBID; - // msgRidList.restart(); - // msgRidCount = 0; - // - // mutex.lock(); //pthread_mutex_lock(&mutex); - // - // if (scanThisBlock) { - // if (recvWaiting) - // condvar.notify_one(); //pthread_cond_signal(&condvar); - // #ifdef DEBUG - //// cout << "msgsSent++ = " << msgsSent << endl; - // #endif - // } - // scanThisBlock = scan; - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // // break the for loop - // if (!more) - // break; - // } - // } // for rw.count - // - // if (more) - // { - // if ( ridList ) - // { - // if (fifo) - // { - // rw.count = 0; - // more = fifo->next(it, &rw); - // } - // else - // { - // rw.count = 1; - // more = ridList->next(it, &e); - // } - // } - // else - // { - // if (strFifo) - // { - // strRw.count = 0; - // more = strFifo->next(it, &strRw); - // } - // else - // { - // strRw.count = 1; - // more = strRidList->next(it, &strE); - // } - // } - // } - // } - // - // if (fOid>=3000) dlTimes.setLastReadTime(); - // - // done: - // mutex.lock(); //pthread_mutex_lock(&mutex); - // finishedSending = true; - // if (recvWaiting) - // condvar.notify_one(); //pthread_cond_signal(&condvar); - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - //#ifdef DEBUG - // if (fOid >=3000) - // cout << "pColStep msgSent " - // << msgsSent << "/" << msgsSkip - // << " rids " << ridCount - // << " oid " << fOid << " " << msgLBID << endl; - //#endif - // //...Track the number of LBIDs we skip due to Casual Partioning. - // fNumBlksSkipped += msgsSkip; -} - -void pColStep::receivePrimitiveMessages() -{ - // int64_t ridResults = 0; - // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - // DataList_t* dlp = dl->dataList(); - // uint64_t fbo; - // FifoDataList *fifo = dl->fifoDL(); - // UintRowGroup rw; - // uint64_t ridBase; - // boost::shared_ptr bs; - // uint32_t i = 0, length; - // - // while (1) { - // // sync with the send side - // mutex.lock(); //pthread_mutex_lock(&mutex); - // while (!finishedSending && msgsSent == msgsRecvd) { - // recvWaiting = true; - // #ifdef DEBUG - // cout << "c sleeping" << endl; - // #endif - // // @bug 836. Wake up the sender if he's sleeping. - // if (fStopSending) - // flushed.notify_one(); //pthread_cond_signal(&flushed); - // condvar.wait(mutex); //pthread_cond_wait(&condvar, &mutex); - // #ifdef DEBUG - // cout << "c waking" << endl; - // #endif - // recvWaiting = false; - // } - // if (msgsSent == msgsRecvd) { - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // break; - // } - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // // do the recv - // fDec->read(uniqueID, bs); - // fMsgBytesIn += bs->lengthWithHdrOverhead(); - // - // // no more messages, and buffered messages should be already processed by now. - // if (bs->length() == 0) break; - // - // #ifdef DEBUG - // cout << "msgsRecvd++ = " << msgsRecvd << ". RidResults = " << ridResults << endl; - // cout << "Got a ColResultHeader!: " << bs.length() << " bytes" << endl; - // #endif - // - // const ByteStream::byte* bsp = bs->buf(); - // - // // get the ISMPacketHeader out of the bytestream - // //const ISMPacketHeader* ism = reinterpret_cast(bsp); - // - // // get the ColumnResultHeader out of the bytestream - // const ColResultHeader* crh = reinterpret_cast - // (&bsp[sizeof(ISMPacketHeader)]); - // - // bool firstRead = true; - // length = bs->length(); - // - // i = 0; - // uint32_t msgCount = 0; - // while (i < length) { - // ++msgCount; - // - // i += sizeof(ISMPacketHeader); - // crh = reinterpret_cast(&bsp[i]); - // // double check the sequence number is increased by one each time - // i += sizeof(ColResultHeader); - // - // fCacheIO += crh->CacheIO; - // fPhysicalIO += crh->PhysicalIO; - // - // // From this point on the rest of the bytestream is the data that comes back from the - //primitive server - // // This needs to be fed to a datalist that is retrieved from the outputassociation - //object. - // - // fbo = getFBO(crh->LBID); - // ridBase = fbo << rpbShift; - // - // #ifdef DEBUG - //// cout << " NVALS = " << crh->NVALS << " fbo = " << fbo << " lbid = " << crh->LBID << - ///endl; - // #endif - // - // //Check output type - // if ( fOutputType == OT_RID ) - // { - // ridResults += crh->NVALS; - // } - // - // /* XXXPAT: This clause is executed when ExeMgr calls the new nextBand(BS) fcn. - // - // TODO: both classes have to agree - // on which nextBand() variant will be called. pColStep - // currently has to infer that from flushInterval and the - // Table OID. It would be better to have a more explicit form - // of agreement. - // - // The goal of the nextBand(BS) fcn is to avoid iterating over - // every row except at unserialization. This clause copies - // the raw results from the PrimProc response directly into - // the memory used for the ElementType array. DeliveryStep - // will also treat the ElementType array as raw memory and - // serialize that. TableColumn now parses the packed data - // instead of whole ElementTypes. - // */ - // else if (fOutputType == OT_TOKEN && fFlushInterval > 0 && !fIsDict) { - // - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // ridResults += crh->NVALS; - // - // /* memcpy the bytestream into the output set */ - // uint32_t toCopy, bsPos = 0; - // uint8_t *pos; - // while (bsPos < crh->NVALS) { - // toCopy = (crh->NVALS - bsPos > rw.ElementsPerGroup - rw.count ? - // rw.ElementsPerGroup - rw.count : crh->NVALS - bsPos); - // pos = ((uint8_t *) &rw.et[0]) + (rw.count * fColType.colWidth); - // memcpy(pos, &bsp[i], toCopy * fColType.colWidth); - // bsPos += toCopy; - // i += toCopy * fColType.colWidth; - // rw.count += toCopy; - // if (rw.count == rw.ElementsPerGroup) { - // if (!fSwallowRows) - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // } - // else if ( fOutputType == OT_TOKEN) - // { - // uint64_t dv; - // uint64_t rid; - // - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // ridResults += crh->NVALS; - // for(int j = 0; j < crh->NVALS; ++j) - // { - // // XXXPAT: Only use this when the RID doesn't matter or when - // // the response contains every row. - // - // rid = j + ridBase; - // switch (fColType.colWidth) { - // case 8: dv = *((const uint64_t *) &bsp[i]); i += 8; break; - // case 4: dv = *((const uint32_t *) &bsp[i]); i += 4; break; - // case 2: dv = *((const uint16_t *) &bsp[i]); i += 2; break; - // case 1: dv = *((const uint8_t *) &bsp[i]); ++i; break; - // default: - // throw runtime_error("pColStep: invalid column - //width!"); - // } - // - // // @bug 663 - Don't output any rows if fSwallowRows (caltraceon(16)) is - //on. - // // This options swallows rows in the project steps. - // if (!fSwallowRows) - // { - // if (fifo) - // { - // rw.et[rw.count].first = rid; - // rw.et[rw.count++].second = dv; - // if (rw.count == rw.ElementsPerGroup) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // else - // { - // dlp->insert(ElementType(rid, dv)); - // } - // #ifdef DEBUG - // //cout << " -- inserting <" << rid << ", " << dv << "> " << *prid << - //endl; #endif - // } - // } - // } - // else if ( fOutputType == OT_BOTH ) - // { - // ridResults += crh->NVALS; - // for(int j = 0; j < crh->NVALS; ++j) - // { - // uint64_t dv; - // uint64_t rid; - // - // rid = *((const uint16_t *) &bsp[i]) + ridBase; - // i += sizeof(uint16_t); - // switch (fColType.colWidth) { - // case 8: dv = *((const uint64_t *) &bsp[i]); i += 8; - //break; case 4: dv = *((const uint32_t *) &bsp[i]); i += 4; break; case 2: dv = *((const uint16_t *) - //&bsp[i]); i += 2; break; case 1: dv = *((const uint8_t *) &bsp[i]); ++i; break; default: throw - //runtime_error("pColStep: invalid column width!"); - // } - // - // // @bug 663 - Don't output any rows if fSwallowRows (caltraceon(16)) is - //on. - // // This options swallows rows in the project steps. - // if (!fSwallowRows) { - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // if(fifo) - // { - //// rw.et[rw.count++] = ElementType(rid, dv); - // rw.et[rw.count].first = rid; - // rw.et[rw.count++].second = dv; - // if (rw.count == rw.ElementsPerGroup) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // else - // { - // dlp->insert(ElementType(rid, dv)); - // } - // #ifdef DEBUG - // //cout << " -- inserting <" << rid << ", " << dv << "> " << *prid << - //endl; #endif - // } - // } - // } - // } // unpacking the BS - // - // //Bug 815: Check whether we have enough to process - // //++lockCount; - // mutex.lock(); //pthread_mutex_lock(&mutex); - // if ( fStopSending && ((msgsSent - msgsRecvd ) <= fProjectBlockReqThreshold) ) - // { - // flushed.notify_one(); //pthread_cond_signal(&flushed); - // } - // mutex.unlock(); //pthread_mutex_unlock(&mutex); - // - // firstRead = false; - // msgsRecvd += msgCount; - // } // read loop - // // done reading - // - // if (fifo && rw.count > 0) - // fifo->insert(rw); - // - // //...Casual partitioning could cause us to do no processing. In that - // //...case these time stamps did not get set. So we set them here. - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) { - // dlTimes.setFirstReadTime(); - // dlTimes.setLastReadTime(); - // dlTimes.setFirstInsertTime(); - // } - // if (fOid>=3000) dlTimes.setEndOfInputTime(); - // - // //@bug 699: Reset StepMsgQueue - // fDec->removeQueue(uniqueID); - // - // if (fifo) - // fifo->endOfInput(); - // else - // dlp->endOfInput(); - // - // if (fTableOid >= 3000) - // { - // //...Construct timestamp using ctime_r() instead of ctime() not - // //...necessarily due to re-entrancy, but because we want to strip - // //...the newline ('\n') off the end of the formatted string. - // time_t t = time(0); - // char timeString[50]; - // ctime_r(&t, timeString); - // timeString[strlen(timeString)-1 ] = '\0'; - // - // FifoDataList* pFifo = 0; - // uint64_t totalBlockedReadCount = 0; - // uint64_t totalBlockedWriteCount = 0; - // - // //...Sum up the blocked FIFO reads for all input associations - // size_t inDlCnt = fInputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedReadCount += pFifo->blockedReadCount(); - // } - // } - // - // //...Sum up the blocked FIFO writes for all output associations - // size_t outDlCnt = fOutputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedWriteCount += pFifo->blockedWriteCount(); - // } - // } - // - // //...Roundoff msg byte counts to nearest KB for display - // uint64_t msgBytesInKB = fMsgBytesIn >> 10; - // uint64_t msgBytesOutKB = fMsgBytesOut >> 10; - // if (fMsgBytesIn & 512) - // msgBytesInKB++; - // if (fMsgBytesOut & 512) - // msgBytesOutKB++; - // - // // @bug 828 - // if (fifo) - // fifo->totalSize(ridResults); - // - // if (traceOn()) - // { - // //...Print job step completion information - // ostringstream logStr; - // logStr << "ses:" << fSessionId << - // " st: " << fStepId << " finished at " << - // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << - // fCacheIO << "; MsgsRvcd-" << msgsRecvd << - // "; BlockedFifoIn/Out-" << totalBlockedReadCount << - // "/" << totalBlockedWriteCount << - // "; output size-" << ridResults << endl << - // "\tPartitionBlocksEliminated-" << fNumBlksSkipped << - // "; MsgBytesIn-" << msgBytesInKB << "KB" << - // "; MsgBytesOut-" << msgBytesOutKB << "KB" << endl << - // "\t1st read " << dlTimes.FirstReadTimeString() << - // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << - // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << - // "s" << endl; - // - // logEnd(logStr.str().c_str()); - // - // syslogReadBlockCounts(16, // exemgr sybsystem - // fPhysicalIO, // # blocks read from disk - // fCacheIO, // # blocks read from cache - // fNumBlksSkipped); // # casual partition block hits - // syslogProcessingTimes(16, // exemgr subsystem - // dlTimes.FirstReadTime(), // first datalist read - // dlTimes.LastReadTime(), // last datalist read - // dlTimes.FirstInsertTime(), // first datalist write - // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write - // syslogEndStep(16, // exemgr subsystem - // totalBlockedReadCount, // blocked datalist input - // totalBlockedWriteCount, // blocked datalist output - // fMsgBytesIn, // incoming msg byte count - // fMsgBytesOut); // outgoing msg byte count - // } - // } -} - const string pColStep::toString() const { ostringstream oss; diff --git a/dbcon/joblist/pdictionary.cpp b/dbcon/joblist/pdictionary.cpp index 71a941713..7372a15ed 100644 --- a/dbcon/joblist/pdictionary.cpp +++ b/dbcon/joblist/pdictionary.cpp @@ -105,64 +105,11 @@ pDictionaryStep::pDictionaryStep(CalpontSystemCatalog::OID o, CalpontSystemCatal , fFilterCount(0) , requestList(0) , fInterval(jobInfo.flushInterval) - , fPhysicalIO(0) - , fCacheIO(0) , fMsgBytesIn(0) , fMsgBytesOut(0) , fRm(jobInfo.rm) , hasEqualityFilter(false) { - // uniqueID = UniqueNumberGenerator::instance()->getUnique32(); - - // fColType.compressionType = fColType.ddn.compressionType = ct; -} - -pDictionaryStep::~pDictionaryStep() -{ - // if (fDec) - // fDec->removeQueue(uniqueID); -} - -void pDictionaryStep::startPrimitiveThread() -{ - // pThread.reset(new boost::thread(pDictionaryStepPrimitive(this))); -} - -void pDictionaryStep::startAggregationThread() -{ - // cThread.reset(new boost::thread(pDictStepAggregator(this))); -} - -void pDictionaryStep::run() -{ - // if (traceOn()) - // { - // syslogStartStep(16, // exemgr subsystem - // std::string("pDictionaryStep")); // step name - // } - // - // const AnyDataListSPtr& dl = fInputJobStepAssociation.outAt(0); - // DataList_t* dlp = dl->dataList(); - // setInputList(dlp); - // - // startPrimitiveThread(); - // startAggregationThread(); -} - -void pDictionaryStep::join() -{ - // pThread->join(); - // cThread->join(); -} - -void pDictionaryStep::setInputList(DataList_t* dl) -{ - requestList = dl; -} - -void pDictionaryStep::setBOP(int8_t b) -{ - fBOP = b; } void pDictionaryStep::addFilter(int8_t COP, const string& value) @@ -190,322 +137,6 @@ void pDictionaryStep::addFilter(int8_t COP, const string& value) } } -void pDictionaryStep::sendPrimitiveMessages() -{ - // int it = -1; - // int msgRidCount = 0; - // bool more; - // int64_t sigToken, msgLBID, nextLBID = -1; - // uint16_t sigOrd; - // ByteStream msgRidList, primMsg(65536); //the MAX_BUFFER_SIZE as of 8/20 - // DictSignatureRequestHeader hdr; - // ISMPacketHeader ism; - // OldGetSigParams pt; - // FifoDataList* fifo = fInputJobStepAssociation.outAt(0)->fifoDL(); - // UintRowGroup rw; - // - ///* XXXPAT: Does this primitive need to care about the HWM as a sanity check, given - // that a ridlist is supplied? */ - // - // if (fifo == 0) - // throw logic_error("Use p_colscanrange instead here"); - // - // try{ - // it = fifo->getIterator(); - // }catch(exception& ex) { - // cerr << "pDictionaryStep::sendPrimitiveMessages: caught exception: " << ex.what() << endl; - // }catch(...) { - // cerr << "pDictionaryStep::sendPrimitiveMessages: caught exception" << endl; - // } - // - // more = fifo->next(it, &rw); - // - // sigToken = rw.et[0].second; - // msgLBID = sigToken >> 10; - // while (more || msgRidCount > 0) { - // for (uint64_t i = 0; ((i < rw.count) || (!more && msgRidCount > 0)); ) - // { - // if (more) - // { - // ridCount++; - // sigToken = rw.et[i].second; - // nextLBID = sigToken >> 10; - //#ifdef DEBUG - // cout << "sigToken = " << sigToken << " lbid = " << nextLBID << endl; - //#endif - // } - // - // // @bug 472 - // if (nextLBID == msgLBID && more && msgRidCount < 8000) { //XXXPAT: need to prove N & S - //here sigOrd = sigToken & 0x3ff; pt.rid = (nextLBID >= 0 ? rw.et[i].first : 0x8000000000000000LL | - //rw.et[i].first); pt.offsetIndex = sigOrd; msgRidList.append(reinterpret_cast(&pt), - //sizeof(pt)); msgRidCount++; - // ++i; - //#ifdef DEBUG - // cout << "added signature ordinal " << sigOrd << endl; - //#endif - // } - // else { - //#ifdef DEBUG - // cout << "sending a prim msg" << endl; - //#endif - // - // // send the primitive, start constructing the next msg - // ism.Interleave=0; - // ism.Flags=planFlagsToPrimFlags(fTraceFlags); - // ism.Command=DICT_SIGNATURE; - // ism.Size=sizeof(DictSignatureRequestHeader) + msgRidList.length(); - // ism.Type=2; - // - // hdr.Hdr.SessionID = fSessionId; - // //hdr.Hdr.StatementID = 0; - // hdr.Hdr.TransactionID = fTxnId; - // hdr.Hdr.VerID = fVerId; - // hdr.Hdr.StepID = fStepId; - // hdr.Hdr.UniqueID = uniqueID; - // - // hdr.LBID = msgLBID; - // idbassert(msgRidCount <= 8000); - // hdr.NVALS = msgRidCount; - // hdr.CompType = fColType.ddn.compressionType; - // - // primMsg.load((const uint8_t *) &ism, sizeof(ism)); - // primMsg.append((const uint8_t *) &hdr, sizeof(DictSignatureRequestHeader)); - // primMsg += msgRidList; - // fMsgBytesOut += primMsg.lengthWithHdrOverhead(); - // fDec->write(primMsg); - // - // msgLBID = nextLBID; - // primMsg.restart(); - // msgRidList.restart(); - // msgRidCount = 0; - // - // mutex.lock(); - // msgsSent++; - // if (recvWaiting) - // condvar.notify_one(); - //#ifdef DEBUG - // cout << "msgsSent++ = " << msgsSent << endl; - //#endif - // mutex.unlock(); - // - // if (!more) - // break; - // } - // } // rw.count - // - // if (more) - // { - // rw.count = 0; - // more = fifo->next(it, &rw); - // } - // } - // - // mutex.lock(); - // finishedSending = true; - // if (recvWaiting) - // condvar.notify_one(); - // mutex.unlock(); -} - -void pDictionaryStep::receivePrimitiveMessages() -{ - // int64_t ridResults = 0; - // AnyDataListSPtr dl = fOutputJobStepAssociation.outAt(0); - // StrDataList* dlp = dl->stringDataList(); - // StringFifoDataList *fifo = fOutputJobStepAssociation.outAt(0)->stringDL(); - // StringRowGroup rw; - // - // while (1) { - // - // // sync with the send side - // mutex.lock(); - // - // while (!finishedSending && msgsSent==msgsRecvd) { - // recvWaiting = true; - // condvar.wait(mutex); - // if (msgsSent == msgsRecvd) { - // mutex.unlock(); - // break; - // } - // recvWaiting = false; - // } - // - // if (finishedSending != 0 && msgsRecvd >= msgsSent) { - // goto junk; - // } - // mutex.unlock(); - // - // // do the recv - // - // ByteStream bs = fDec->read(uniqueID); - // fMsgBytesIn += bs.lengthWithHdrOverhead(); - // if (fOid>=3000 && dlTimes.FirstReadTime().tv_sec==0) - // dlTimes.setFirstReadTime(); - // if (fOid>=3000) dlTimes.setLastReadTime(); - // - // msgsRecvd++; - // if (bs.length() == 0) - // break; - // - // const ByteStream::byte* bsp = bs.buf(); - // - // // get the ResultHeader out of the bytestream - // const DictOutput* drh = reinterpret_cast(bsp); - // - // bsp += sizeof(DictOutput); - // - // fCacheIO += drh->CacheIO; - // fPhysicalIO += drh->PhysicalIO; - // - // // From this point on the rest of the bytestream is the data that comes back from the primitive - //server - // // This needs to be fed to a datalist that is retrieved from the outputassociation object. - // - // char d[8192]; - //// memset(d, 0, 8192); - // if (fOid>=3000 && dlTimes.FirstInsertTime().tv_sec==0) - // dlTimes.setFirstInsertTime(); - // for(int j = 0; j < drh->NVALS; j++) - // { - // const uint64_t* ridp = (const uint64_t*)bsp; - // bsp += sizeof(*ridp); - // uint64_t rid = *ridp; - // const uint16_t* lenp = (const uint16_t*)bsp; - // bsp += sizeof(*lenp); - // uint16_t len = *lenp; - // memcpy(d, bsp, len); - // bsp += len; - // d[len] = 0; - // if (rid == 0xFFFFFFFFFFFFFFFFULL) - // { - // strcpy(d, CPNULLSTRMARK.c_str()); - // } - //#ifdef FIFO_SINK - // if (fOid < 3000) - //#endif - // if (fifo) - // { - // rw.et[rw.count++] = StringElementType(rid, d); - // if (rw.count == rw.ElementsPerGroup) - // { - // fifo->insert(rw); - // rw.count = 0; - // } - // } - // else - // { - // dlp->insert(StringElementType(rid, d)); - // } - // - //#ifdef DEBUG - // cout << " -- inserting <" << rid << ", " << d << ">" << endl; - //#endif - // ridResults++; - // - // } - // } - // - // junk: - // - // if (fifo && rw.count > 0) - // fifo->insert(rw); - // - // //@bug 699: Reset StepMsgQueue - // fDec->removeQueue(uniqueID); - // - // if (fOid>=3000) dlTimes.setEndOfInputTime(); - // dlp->endOfInput(); - // - // if (fTableOid >= 3000) - // { - // //...Construct timestamp using ctime_r() instead of ctime() not - // //...necessarily due to re-entrancy, but because we want to strip - // //...the newline ('\n') off the end of the formatted string. - // time_t t = time(0); - // char timeString[50]; - // ctime_r(&t, timeString); - // timeString[strlen(timeString)-1 ] = '\0'; - // - // FifoDataList* pFifo = 0; - // uint64_t totalBlockedReadCount = 0; - // uint64_t totalBlockedWriteCount = 0; - // - // //...Sum up the blocked FIFO reads for all input associations - // size_t inDlCnt = fInputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedReadCount += pFifo->blockedReadCount(); - // } - // } - // - // //...Sum up the blocked FIFO writes for all output associations - // size_t outDlCnt = fOutputJobStepAssociation.outSize(); - // for (size_t iDataList=0; iDataListfifoDL(); - // if (pFifo) - // { - // totalBlockedWriteCount += pFifo->blockedWriteCount(); - // } - // } - // - // - // - // //...Roundoff msg byte counts to nearest KB for display - // uint64_t msgBytesInKB = fMsgBytesIn >> 10; - // uint64_t msgBytesOutKB = fMsgBytesOut >> 10; - // if (fMsgBytesIn & 512) - // msgBytesInKB++; - // if (fMsgBytesOut & 512) - // msgBytesOutKB++; - // - // // @bug 807 - // if (fifo) - // fifo->totalSize(ridResults); - // - // if (traceOn()) - // { - // //...Print job step completion information - // ostringstream logStr; - // logStr << "ses:" << fSessionId << " st: " << fStepId << - // " finished at " << - // timeString << "; PhyI/O-" << fPhysicalIO << "; CacheI/O-" << - // fCacheIO << "; MsgsRcvd-" << msgsRecvd << - // "; BlockedFifoIn/Out-" << totalBlockedReadCount << - // "/" << totalBlockedWriteCount << - // "; output size-" << ridResults << endl << - // "\tMsgBytesIn-" << msgBytesInKB << "KB" << - // "; MsgBytesOut-" << msgBytesOutKB << "KB" << endl << - // "\t1st read " << dlTimes.FirstReadTimeString() << - // "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-" << - // JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(),dlTimes.FirstReadTime()) << - // "s" << endl; - // - // logEnd(logStr.str().c_str()); - // - // syslogReadBlockCounts(16, // exemgr subsystem - // fPhysicalIO, // # blocks read from disk - // fCacheIO, // # blocks read from cache - // 0); // # casual partition block hits - // syslogProcessingTimes(16, // exemgr subsystem - // dlTimes.FirstReadTime(), // first datalist read - // dlTimes.LastReadTime(), // last datalist read - // dlTimes.FirstInsertTime(), // first datalist write - // dlTimes.EndOfInputTime()); // last (endOfInput) datalist write - // syslogEndStep(16, // exemgr subsystem - // totalBlockedReadCount, // blocked datalist input - // totalBlockedWriteCount, // blocked datalist output - // fMsgBytesIn, // incoming msg byte count - // fMsgBytesOut); // outgoing msg byte count - // } - // } - // -} - const string pDictionaryStep::toString() const { ostringstream oss; @@ -546,9 +177,6 @@ void pDictionaryStep::appendFilter(const messageqcpp::ByteStream& filter, unsign addFilter(COP, value); bs.advance(size); } - - // fFilterString += filter; - // fFilterCount += count; } void pDictionaryStep::addFilter(const Filter* f) diff --git a/dbcon/joblist/pdictionaryscan.cpp b/dbcon/joblist/pdictionaryscan.cpp index 94f70ba2e..164ac157c 100644 --- a/dbcon/joblist/pdictionaryscan.cpp +++ b/dbcon/joblist/pdictionaryscan.cpp @@ -138,10 +138,8 @@ pDictionaryScan::pDictionaryScan(CalpontSystemCatalog::OID o, CalpontSystemCatal , fColType(ct) , pThread(0) , cThread(0) - , fScanLbidReqLimit(jobInfo.rm->getJlScanLbidReqLimit()) , fScanLbidReqThreshold(jobInfo.rm->getJlScanLbidReqThreshold()) , fStopSending(false) - , fSingleThread(false) , fPhysicalIO(0) , fCacheIO(0) , fMsgBytesIn(0) @@ -915,4 +913,19 @@ void pDictionaryScan::abort() fDec->shutdownQueue(uniqueID); } +// Unfortuneately we have 32 bits in the execplan flags, but only 16 that can be sent to +// PrimProc, so we have to convert them (throwing some away). +uint16_t pDictionaryScan::planFlagsToPrimFlags(uint32_t planFlags) +{ + uint16_t flags = 0; + + if (planFlags & CalpontSelectExecutionPlan::TRACE_LBIDS) + flags |= PF_LBID_TRACE; + + if (planFlags & CalpontSelectExecutionPlan::PM_PROFILE) + flags |= PF_PM_PROF; + + return flags; +} + } // namespace joblist diff --git a/dbcon/joblist/primitivemsg.cpp b/dbcon/joblist/primitivemsg.cpp deleted file mode 100644 index 47e7b8e2f..000000000 --- a/dbcon/joblist/primitivemsg.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - * $Id: primitivemsg.cpp 9210 2013-01-21 14:10:42Z rdempsey $ - */ - -#include -using namespace std; - -#include "calpontselectexecutionplan.h" -using namespace execplan; - -#include "primitivemsg.h" -#include "primitivestep.h" -using namespace joblist; - -namespace joblist -{ -void PrimitiveMsg::send() -{ - throw logic_error("somehow ended up in PrimitiveMsg::send()!"); -} - -void PrimitiveMsg::buildPrimitiveMessage(ISMPACKETCOMMAND, void*, void*) -{ - throw logic_error("somehow ended up in PrimitiveMsg::buildPrimitiveMessage()!"); -} - -void PrimitiveMsg::receive() -{ - throw logic_error("somehow ended up in PrimitiveMsg::receive()!"); -} - -void PrimitiveMsg::sendPrimitiveMessages() -{ - throw logic_error("somehow ended up in PrimitiveMsg::sendPrimitiveMessages()!"); -} - -void PrimitiveMsg::receivePrimitiveMessages() -{ - throw logic_error("somehow ended up in PrimitiveMsg::receivePrimitiveMessages()!"); -} - -// Unfortuneately we have 32 bits in the execplan flags, but only 16 that can be sent to -// PrimProc, so we have to convert them (throwing some away). -uint16_t PrimitiveMsg::planFlagsToPrimFlags(uint32_t planFlags) -{ - uint16_t flags = 0; - - if (planFlags & CalpontSelectExecutionPlan::TRACE_LBIDS) - flags |= PF_LBID_TRACE; - - if (planFlags & CalpontSelectExecutionPlan::PM_PROFILE) - flags |= PF_PM_PROF; - - return flags; -} - -} // namespace joblist diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index e68837c6c..ec78b893f 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -93,39 +93,9 @@ enum PrimitiveStepType AGGRFILTERSTEP }; -/** @brief class PrimitiveMsg - * - */ -class PrimitiveMsg -{ - public: - /** @brief virtual void Send method - */ - virtual void send(); - /** @brief virtual void Receive method - */ - virtual void receive(); - /** @brief virtual void BuildPrimitiveMessage method - */ - virtual void buildPrimitiveMessage(ISMPACKETCOMMAND cmd, void* filterValues, void* ridArray); - virtual void sendPrimitiveMessages(); - virtual void receivePrimitiveMessages(); - - PrimitiveMsg() - { - } - - virtual ~PrimitiveMsg() - { - } - - uint16_t planFlagsToPrimFlags(uint32_t planFlags); - - private: -}; class pColScanStep; -class pColStep : public JobStep, public PrimitiveMsg +class pColStep : public JobStep { typedef std::pair element_t; @@ -141,48 +111,30 @@ class pColStep : public JobStep, public PrimitiveMsg pColStep(const PassThruStep& rhs); - virtual ~pColStep(); + virtual ~pColStep(){}; /** @brief Starts processing. Set at least the RID list before calling. * * Starts processing. Set at least the RID list before calling this. */ - virtual void run(); + virtual void run(){}; /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(); + virtual void join(){}; virtual const std::string toString() const; virtual bool isDictCol() const { return fIsDict; - }; + } bool isExeMgr() const { return isEM; } - /** @brief Set config parameters for this JobStep. - * - * Set the config parameters this JobStep. - */ - void initializeConfigParms(); - - /** @brief The main loop for the send-side thread - * - * The main loop for the primitive-issuing thread. Don't call it directly. - */ - void sendPrimitiveMessages(); - - /** @brief The main loop for the recv-side thread - * - * The main loop for the receive-side thread. Don't call it directly. - */ - void receivePrimitiveMessages(); - /** @brief Add a filter. Use this interface when the column stores anything but 4-byte floats. * * Add a filter. Use this interface when the column stores anything but 4-byte floats. @@ -197,35 +149,38 @@ class pColStep : public JobStep, public PrimitiveMsg * this class from pColScan. Use pColScan if the every RID should be considered; it's * faster at that. */ - void setRidList(DataList* rids); - + void setRidList(DataList* rids) + { + ridList = rids; + } /** @brief Sets the String DataList to get RID values from. * * Sets the string DataList to get RID values from. Filtering by RID distinguishes * this class from pColScan. Use pColScan if the every RID should be considered; it's * faster at that. */ - void setStrRidList(DataList* strDl); - + void setStrRidList(DataList* strDl) + { + strRidList = strDl; + } /** @brief Set the binary operator for the filter predicate (BOP_AND or BOP_OR). * * Set the binary operator for the filter predicate (BOP_AND or BOP_OR). */ - void setBOP(int8_t BOP); - - /** @brief Set the output type. - * - * Set the output type (1 = RID, 2 = Token, 3 = Both). - */ - void setOutputType(int8_t OutputType); + void setBOP(int8_t BOP) + { + fBOP = BOP; + } /** @brief Set the swallowRows flag. * * * If true, no rows will be inserted to the output datalists. */ - void setSwallowRows(const bool swallowRows); - + void setSwallowRows(const bool swallowRows) + { + fSwallowRows = swallowRows; + } /** @brief Get the swallowRows flag. * * @@ -263,10 +218,6 @@ class pColStep : public JobStep, public PrimitiveMsg return fColType; } void appendFilter(const messageqcpp::ByteStream& filter, unsigned count); - uint32_t flushInterval() const - { - return fFlushInterval; - } bool getFeederFlag() const { return isFilterFeeder; @@ -276,14 +227,6 @@ class pColStep : public JobStep, public PrimitiveMsg { isFilterFeeder = filterFeeder; } - virtual uint64_t phyIOCount() const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount() const - { - return fCacheIO; - } virtual uint64_t msgsRcvdCount() const { return msgsRecvd; @@ -329,14 +272,6 @@ class pColStep : public JobStep, public PrimitiveMsg */ explicit pColStep(); - /** @brief StartPrimitiveThread - * Utility function to start worker thread that sends primitive messages - */ - void startPrimitiveThread(); - /** @brief StartAggregationThread - * Utility function to start worker thread that receives result aggregation from primitive servers - */ - void startAggregationThread(); uint64_t getLBID(uint64_t rid, bool& scan); uint64_t getFBO(uint64_t lbid); @@ -347,7 +282,6 @@ class pColStep : public JobStep, public PrimitiveMsg execplan::CalpontSystemCatalog::ColType fColType; uint32_t fFilterCount; int8_t fBOP; - int8_t fOutputType; uint16_t realWidth; DataList_t* ridList; StrDataList* strRidList; @@ -359,29 +293,18 @@ class pColStep : public JobStep, public PrimitiveMsg bool finishedSending, recvWaiting, fIsDict; bool isEM; int64_t ridCount; - uint32_t fFlushInterval; // @bug 663 - Added fSwallowRows for calpont.caltrace(16) which is TRACE_FLAGS::TRACE_NO_ROWS4. // Running with this one will swallow rows at projection. bool fSwallowRows; - uint32_t fProjectBlockReqLimit; // max number of rids to send in a scan - // request to primproc - uint32_t fProjectBlockReqThreshold; // min level of rids backlog before - // consumer will tell producer to send - // more rids scan requests to primproc - volatile bool fStopSending; bool isFilterFeeder; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count uint64_t fNumBlksSkipped; // total number of block scans skipped due to CP uint64_t fMsgBytesIn; // total byte count for incoming messages uint64_t fMsgBytesOut; // total byte count for outcoming messages BRM::DBRM dbrm; - // boost::shared_ptr cThread; //consumer thread - // boost::shared_ptr pThread; //producer thread boost::mutex mutex; boost::condition condvar; boost::condition flushed; @@ -413,7 +336,7 @@ class pColStep : public JobStep, public PrimitiveMsg * c) send messages to the primitive server as quickly as possible */ -class pColScanStep : public JobStep, public PrimitiveMsg +class pColScanStep : public JobStep { public: /** @brief pColScanStep constructor @@ -422,38 +345,25 @@ class pColScanStep : public JobStep, public PrimitiveMsg const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); pColScanStep(const pColStep& rhs); - ~pColScanStep(); + ~pColScanStep(){} /** @brief Starts processing. * * Starts processing. */ - virtual void run(); + virtual void run(){} /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(); + virtual void join(){} virtual bool isDictCol() const { return fIsDict; }; - /** @brief The main loop for the send-side thread - * - * The main loop for the primitive-issuing thread. Don't call it directly. - */ - void sendPrimitiveMessages(); - - /** @brief The main loop for the recv-side thread - * - * The main loop for the receive-side thread. Don't call it directly. - */ - using PrimitiveMsg::receivePrimitiveMessages; - void receivePrimitiveMessages(uint64_t i = 0); - /** @brief Add a filter when the column is a 4-byte float type * * Add a filter when the column is a 4-byte float type @@ -472,7 +382,11 @@ class pColScanStep : public JobStep, public PrimitiveMsg * Set the binary operator for the filter predicates (BOP_AND or BOP_OR). * It is initialized to OR. */ - void setBOP(int8_t BOP); // AND or OR + void setBOP(int8_t BOP) // AND or OR + { + fBOP = BOP; + } + int8_t BOP() const { return fBOP; @@ -496,17 +410,6 @@ class pColScanStep : public JobStep, public PrimitiveMsg return fFilterString; } - void setSingleThread(bool b); - bool getSingleThread() - { - return fSingleThread; - } - - /** @brief Set the output type. - * - * Set the output type (1 = RID, 2 = Token, 3 = Both).pColScan - */ - void setOutputType(int8_t OutputType); uint32_t filterCount() const { return fFilterCount; @@ -532,18 +435,6 @@ class pColScanStep : public JobStep, public PrimitiveMsg return fRm; } - virtual uint64_t phyIOCount() const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount() const - { - return fCacheIO; - } - virtual uint64_t msgsRcvdCount() const - { - return recvCount; - } virtual uint64_t msgBytesIn() const { return fMsgBytesIn; @@ -599,18 +490,12 @@ class pColScanStep : public JobStep, public PrimitiveMsg // pColScanStep& operator=(const pColScanStep& rhs); typedef boost::shared_ptr SPTHD; - void startPrimitiveThread(); - void startAggregationThread(); - void initializeConfigParms(); - void sendAPrimitiveMessage(ISMPacketHeader& ism, BRM::LBID_t msgLbidStart, uint32_t msgLbidCount); uint64_t getFBO(uint64_t lbid); bool isEmptyVal(const uint8_t* val8) const; ResourceManager* fRm; ColByScanRangeRequestHeader fMsgHeader; SPTHD fConsumerThread; - /// number of threads on the receive side - uint32_t fNumThreads; SPTHD* fProducerThread; messageqcpp::ByteStream fFilterString; @@ -619,16 +504,10 @@ class pColScanStep : public JobStep, public PrimitiveMsg execplan::CalpontSystemCatalog::OID fTableOid; execplan::CalpontSystemCatalog::ColType fColType; int8_t fBOP; - int8_t fOutputType; - uint32_t sentCount; - uint32_t recvCount; BRM::LBIDRange_v lbidRanges; BRM::DBRM dbrm; SP_LBIDList lbidList; - boost::mutex mutex; - boost::mutex dlMutex; - boost::mutex cpMutex; boost::condition condvar; boost::condition condvarWakeupProducer; bool finishedSending, sendWaiting, rDoNothing, fIsDict; @@ -638,17 +517,7 @@ class pColScanStep : public JobStep, public PrimitiveMsg uint32_t extentSize, divShift, ridsPerBlock, rpbShift, numExtents; // config::Config *fConfig; - uint32_t fScanLbidReqLimit; // max number of LBIDs to send in a scan - // request to primproc - uint32_t fScanLbidReqThreshold; // min level of scan LBID backlog before - // consumer will tell producer to send - // more LBID scan requests to primproc - - bool fStopSending; - bool fSingleThread; bool isFilterFeeder; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count uint64_t fNumBlksSkipped; // total number of block scans skipped due to CP uint64_t fMsgBytesIn; // total byte count for incoming messages uint64_t fMsgBytesOut; // total byte count for outcoming messages @@ -667,35 +536,10 @@ class pColScanStep : public JobStep, public PrimitiveMsg friend class TupleBPS; }; -#if 0 -class pIdxStep : public JobStep -{ -public: - /** @brief pIdxStep constructor - * @param in the inputAssociation pointer - * @param out the outputAssociation pointer - * @param ec the DistributedEngineComm pointer - */ - pIdxStep(JobStepAssociation* in, JobStepAssociation* out, DistributedEngineComm* ec); - /** @brief virtual void Run method - */ - virtual void run(); -private: - pIdxStep(); - void startPrimitveThread(); - void startAggregationThread(); - -protected: - DistributedEngineComm* fDec; - JobStepAssociation* fInputJobStepAssociation; - JobStepAssociation* fOutputJobStepAssociation; -}; -#endif - /** @brief class pDictionaryStep * */ -class pDictionaryStep : public JobStep, public PrimitiveMsg +class pDictionaryStep : public JobStep { public: /** @brief pDictionaryStep constructor @@ -704,17 +548,22 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg pDictionaryStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tabelOid, const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - virtual ~pDictionaryStep(); + virtual ~pDictionaryStep(){} /** @brief virtual void Run method */ - virtual void run(); - virtual void join(); + virtual void run(){} + virtual void join(){} // void setOutList(StringDataList* rids); - void setInputList(DataList_t* rids); - void setBOP(int8_t b); - void sendPrimitiveMessages(); - void receivePrimitiveMessages(); + void setInputList(DataList_t* rids) + { + requestList = rids; + } + + void setBOP(int8_t b) + { + fBOP = b; + } virtual const std::string toString() const; @@ -735,14 +584,6 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg { return fTableOid; } - virtual uint64_t phyIOCount() const - { - return fPhysicalIO; - } - virtual uint64_t cacheIOCount() const - { - return fCacheIO; - } virtual uint64_t msgsRcvdCount() const { return msgsRecvd; @@ -780,8 +621,6 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg private: pDictionaryStep(); - void startPrimitiveThread(); - void startAggregationThread(); boost::shared_ptr sysCat; execplan::CalpontSystemCatalog::OID fOid; @@ -804,8 +643,6 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg boost::mutex mutex; boost::condition condvar; uint32_t fInterval; - uint64_t fPhysicalIO; // total physical I/O count - uint64_t fCacheIO; // total cache I/O count uint64_t fMsgBytesIn; // total byte count for incoming messages uint64_t fMsgBytesOut; // total byte count for outcoming messages uint32_t uniqueID; @@ -828,7 +665,7 @@ class pDictionaryStep : public JobStep, public PrimitiveMsg /** @brief class pDictionaryScan * */ -class pDictionaryScan : public JobStep, public PrimitiveMsg +class pDictionaryScan : public JobStep { public: /** @brief pDictionaryScan constructor @@ -953,6 +790,7 @@ class pDictionaryScan : public JobStep, public PrimitiveMsg private: pDictionaryScan(); + uint16_t planFlagsToPrimFlags(uint32_t planFlags); void startPrimitiveThread(); void startAggregationThread(); void initializeConfigParms(); @@ -990,12 +828,10 @@ class pDictionaryScan : public JobStep, public PrimitiveMsg uint64_t extentSize; uint64_t divShift; uint64_t numExtents; - uint32_t fScanLbidReqLimit; // max number of LBIDs to send in a scan // request to primproc uint32_t fScanLbidReqThreshold; // min level of scan LBID backlog before // consumer will tell producer to send bool fStopSending; - bool fSingleThread; uint64_t fPhysicalIO; // total physical I/O count uint64_t fCacheIO; // total cache I/O count uint64_t fMsgBytesIn; // total byte count for incoming messages @@ -1018,7 +854,7 @@ class pDictionaryScan : public JobStep, public PrimitiveMsg void destroyEqualityFilter(); }; -class BatchPrimitive : public JobStep, public PrimitiveMsg, public DECEventListener +class BatchPrimitive : public JobStep, public DECEventListener { public: BatchPrimitive(const JobInfo& jobInfo) : JobStep(jobInfo) @@ -1697,7 +1533,7 @@ class FilterStep : public JobStep /** @brief class PassThruStep * */ -class PassThruStep : public JobStep, public PrimitiveMsg +class PassThruStep : public JobStep { typedef std::pair element_t; diff --git a/dbcon/joblist/resourcemanager.cpp b/dbcon/joblist/resourcemanager.cpp index be03066c8..05db9abbb 100644 --- a/dbcon/joblist/resourcemanager.cpp +++ b/dbcon/joblist/resourcemanager.cpp @@ -64,7 +64,6 @@ ResourceManager::ResourceManager(bool runningInExeMgr, config::Config* aConfig) , fHjNumThreads(defaultNumThreads) , fJlProcessorThreadsPerScan(defaultProcessorThreadsPerScan) , fJlNumScanReceiveThreads(defaultScanReceiveThreads) - , fTwNumThreads(defaultNumThreads) , fJlMaxOutstandingRequests(defaultMaxOutstandingRequests) , fHJUmMaxMemorySmallSideDistributor( fHashJoinStr, "UmMaxMemorySmallSide", @@ -101,7 +100,6 @@ ResourceManager::ResourceManager(bool runningInExeMgr, config::Config* aConfig) { fHjNumThreads = fNumCores; fJlNumScanReceiveThreads = fNumCores; - fTwNumThreads = fNumCores; } // possibly override any calculated values @@ -139,11 +137,6 @@ ResourceManager::ResourceManager(bool runningInExeMgr, config::Config* aConfig) fDECConnectionsPerQuery = (fDECConnectionsPerQuery) ? fDECConnectionsPerQuery : getPsConnectionsPerPrimProc(); - temp = getIntVal(fTupleWSDLStr, "NumThreads", -1); - - if (temp > 0) - fTwNumThreads = temp; - pmJoinMemLimit = getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide); // Need to use different limits if this instance isn't running on the UM, @@ -317,106 +310,6 @@ void ResourceManager::logResourceChangeMessage(logging::LOG_TYPE logType, uint32 log.logMessage(logType, mid, args, logging::LoggingID(5, sessionID)); } -void ResourceManager::emServerThreads() -{ -} -void ResourceManager::emServerQueueSize() -{ -} -void ResourceManager::emSecondsBetweenMemChecks() -{ -} -void ResourceManager::emMaxPct() -{ -} -void ResourceManager::emPriority() -{ -} -void ResourceManager::emExecQueueSize() -{ -} - -void ResourceManager::hjNumThreads() -{ -} -void ResourceManager::hjMaxBuckets() -{ -} -void ResourceManager::hjMaxElems() -{ -} -void ResourceManager::hjFifoSizeLargeSide() -{ -} -void ResourceManager::hjPmMaxMemorySmallSide() -{ -} - -void ResourceManager::jlFlushInterval() -{ -} -void ResourceManager::jlFifoSize() -{ -} -void ResourceManager::jlScanLbidReqLimit() -{ -} -void ResourceManager::jlScanLbidReqThreshold() -{ -} -void ResourceManager::jlProjectBlockReqLimit() -{ -} -void ResourceManager::jlProjectBlockReqThreshold() -{ -} -void ResourceManager::jlNumScanReceiveThreads() -{ -} - -void ResourceManager::psCount() -{ -} -void ResourceManager::psConnectionsPerPrimProc() -{ -} -void ResourceManager::psLBID_Shift() -{ -} - -void ResourceManager::scTempDiskPath() -{ -} -void ResourceManager::scTempSaveSize() -{ -} -void ResourceManager::scWorkingDir() -{ -} - -void ResourceManager::twMaxSize() -{ -} -void ResourceManager::twInitialCapacity() -{ -} -void ResourceManager::twMaxBuckets() -{ -} -void ResourceManager::twNumThreads() -{ -} -void ResourceManager::zdl_MaxElementsInMem() -{ -} -void ResourceManager::zdl_MaxElementsPerBucket() -{ -} - -void ResourceManager::hbrPredicate() -{ -} - bool ResourceManager::getMysqldInfo(std::string& h, std::string& u, std::string& w, unsigned int& p) const { static const std::string hostUserUnassignedValue("unassigned"); diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index 76b08760b..d7f9cfef0 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -52,31 +52,18 @@ const uint32_t defaultNumThreads = 8; // joblistfactory const uint32_t defaultFlushInterval = 8 * 1024; const uint32_t defaultFifoSize = 10; -const uint32_t defaultHJFifoSizeLargeSide = 128; const uint64_t defaultHJMaxElems = 512 * 1024; // hashjoin uses 8192 const int defaultHJMaxBuckets = 32; // hashjoin uses 4 const uint64_t defaultHJPmMaxMemorySmallSide = 1 * 1024 * 1024 * 1024ULL; const uint64_t defaultHJUmMaxMemorySmallSide = 4 * 1024 * 1024 * 1024ULL; -const uint32_t defaultTempSaveSize = defaultHJMaxElems; const uint64_t defaultTotalUmMemory = 8 * 1024 * 1024 * 1024ULL; -const uint64_t defaultHUATotalMem = 8 * 1024 * 1024 * 1024ULL; - -const uint32_t defaultTupleDLMaxSize = 64 * 1024; - const uint32_t defaultJLThreadPoolSize = 100; // pcolscan.cpp -const uint32_t defaultScanLbidReqLimit = 10000; const uint32_t defaultScanLbidReqThreshold = 5000; const uint32_t defaultLogicalBlocksPerScan = 1024; // added for bug 1264. -const uint32_t defaultScanBlockThreshhold = 10000; // in jobstep.h - const uint32_t defaultScanReceiveThreads = 8; -// pcolstep.cpp -const uint32_t defaultProjectBlockReqLimit = 32 * 1024; -const uint32_t defaultProjectBlockReqThreshold = 16 * 1024; // 256 in jobstep.h - // BatchPrimitiveStep const uint32_t defaultRequestSize = 1; const uint32_t defaultMaxOutstandingRequests = 20; @@ -85,15 +72,6 @@ const uint32_t defaultJoinerChunkSize = 16 * 1024 * 1024; // bucketreuse const std::string defaultTempDiskPath = "/tmp"; -const std::string defaultWorkingDir = "."; //"/tmp"; - -// largedatalist -const uint32_t defaultLDLMaxElements = 32 * 1024 * 1024; - -// zdl -const uint64_t defaultMaxElementsInMem = 32 * 1024 * 1024; -const uint64_t defaultNumBuckets = 128; -const uint64_t defaultMaxElementsPerBuckert = 16 * 1024 * 1024; const int defaultEMServerThreads = 50; const int defaultEMSecondsBetweenMemChecks = 1; @@ -101,11 +79,8 @@ const int defaultEMMaxPct = 95; const int defaultEMPriority = 21; // @Bug 3385 const int defaultEMExecQueueSize = 20; -const uint64_t defaultInitialCapacity = 1024 * 1024; -const int defaultTWMaxBuckets = 256; const int defaultPSCount = 0; const int defaultConnectionsPerPrimProc = 1; -const uint32_t defaultLBID_Shift = 13; const uint64_t defaultExtentRows = 8 * 1024 * 1024; // DMLProc @@ -120,13 +95,8 @@ const uint64_t defaultRowsPerBatch = 10000; /* HJ CP feedback, see bug #1465 */ const uint32_t defaultHjCPUniqueLimit = 100; -// Order By and Limit -const uint64_t defaultOrderByLimitMaxMemory = 1 * 1024 * 1024 * 1024ULL; - const uint64_t defaultDECThrottleThreshold = 200000000; // ~200 MB -const uint8_t defaultUseCpimport = 1; - const bool defaultAllowDiskAggregation = false; /** @brief ResourceManager @@ -172,7 +142,7 @@ class ResourceManager { return getUintVal(fExeMgrStr, "MaxPct", defaultEMMaxPct); } - EXPORT int getEmPriority() const; + EXPORT int getEmPriority() const; // FOr Windows only int getEmExecQueueSize() const { return getIntVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); @@ -200,10 +170,6 @@ class ResourceManager { return getUintVal(fHashJoinStr, "MaxElems", defaultHJMaxElems); } - uint32_t getHjFifoSizeLargeSide() const - { - return getUintVal(fHashJoinStr, "FifoSizeLargeSide", defaultHJFifoSizeLargeSide); - } uint32_t getHjCPUniqueLimit() const { return getUintVal(fHashJoinStr, "CPUniqueLimit", defaultHjCPUniqueLimit); @@ -221,10 +187,6 @@ class ResourceManager { return getUintVal(fJobListStr, "FifoSize", defaultFifoSize); } - uint32_t getJlScanLbidReqLimit() const - { - return getUintVal(fJobListStr, "ScanLbidReqLimit", defaultScanLbidReqLimit); - } uint32_t getJlScanLbidReqThreshold() const { return getUintVal(fJobListStr, "ScanLbidReqThreshold", defaultScanLbidReqThreshold); @@ -250,14 +212,6 @@ class ResourceManager { return getUintVal(fJobListStr, "LogicalBlocksPerScan", defaultLogicalBlocksPerScan); } - uint32_t getJlProjectBlockReqLimit() const - { - return getUintVal(fJobListStr, "ProjectBlockReqLimit", defaultProjectBlockReqLimit); - } - uint32_t getJlProjectBlockReqThreshold() const - { - return getUintVal(fJobListStr, "ProjectBlockReqThreshold", defaultProjectBlockReqThreshold); - } uint32_t getJlNumScanReceiveThreads() const { return fJlNumScanReceiveThreads; @@ -290,49 +244,15 @@ class ResourceManager { return getUintVal(fPrimitiveServersStr, "ConnectionsPerPrimProc", defaultConnectionsPerPrimProc); } - uint32_t getPsLBID_Shift() const - { - return getUintVal(fPrimitiveServersStr, "LBID_Shift", defaultLBID_Shift); - } - std::string getScTempDiskPath() const { return startup::StartUp::tmpDir(); } - uint64_t getScTempSaveSize() const - { - return getUintVal(fSystemConfigStr, "TempSaveSize", defaultTempSaveSize); - } std::string getScWorkingDir() const { return startup::StartUp::tmpDir(); } - uint32_t getTwMaxSize() const - { - return getUintVal(fTupleWSDLStr, "MaxSize", defaultTupleDLMaxSize); - } - uint64_t getTwInitialCapacity() const - { - return getUintVal(fTupleWSDLStr, "InitialCapacity", defaultInitialCapacity); - } - int getTwMaxBuckets() const - { - return getUintVal(fTupleWSDLStr, "MaxBuckets", defaultTWMaxBuckets); - } - uint8_t getTwNumThreads() const - { - return fTwNumThreads; - } // getUintVal(fTupleWSDLStr, "NumThreads", defaultNumThreads ); } - uint64_t getZdl_MaxElementsInMem() const - { - return getUintVal(fZDLStr, "ZDL_MaxElementsInMem", defaultMaxElementsInMem); - } - uint64_t getZdl_MaxElementsPerBucket() const - { - return getUintVal(fZDLStr, "ZDL_MaxElementsPerBucket", defaultMaxElementsPerBuckert); - } - uint64_t getExtentRows() const { return getUintVal(fExtentMapStr, "ExtentRows", defaultExtentRows); @@ -347,13 +267,6 @@ class ResourceManager return getUintVal(fPrimitiveServersStr, "Count", 1); } - std::vector getHbrPredicate() const - { - std::vector columns; - fConfig->getConfig(fHashBucketReuseStr, "Predicate", columns); - return columns; - } - uint64_t getDMLMaxDeleteRows() const { return getUintVal(fDMLProcStr, "MaxDeleteRows", defaultDMLMaxDeleteRows); @@ -364,17 +277,6 @@ class ResourceManager return getUintVal(fBatchInsertStr, "RowsPerBatch", defaultRowsPerBatch); } - uint8_t getUseCpimport() const - { - int val = getIntVal(fBatchInsertStr, "UseCpimport", defaultUseCpimport); - return val; - } - - uint64_t getOrderByLimitMaxMemory() const - { - return getUintVal(fOrderByLimitStr, "MaxMemory", defaultOrderByLimitMaxMemory); - } - uint64_t getDECThrottleThreshold() const { return getUintVal(fJobListStr, "DECThrottleThreshold", defaultDECThrottleThreshold); @@ -455,32 +357,6 @@ class ResourceManager fHJUmMaxMemorySmallSideDistributor.returnResource(mem); } - EXPORT void jlFlushInterval(); - EXPORT void jlFifoSize(); - EXPORT void jlScanLbidReqLimit(); - EXPORT void jlScanLbidReqThreshold(); - EXPORT void jlProjectBlockReqLimit(); - EXPORT void jlProjectBlockReqThreshold(); - EXPORT void jlNumScanReceiveThreads(); - - EXPORT void psCount(); - EXPORT void psConnectionsPerPrimProc(); - EXPORT void psLBID_Shift(); - - EXPORT void scTempDiskPath(); - EXPORT void scTempSaveSize(); - EXPORT void scWorkingDir(); - - EXPORT void twMaxSize(); - EXPORT void twInitialCapacity(); - EXPORT void twMaxBuckets(); - EXPORT void twNumThreads(); - - EXPORT void zdl_MaxElementsInMem(); - EXPORT void zdl_MaxElementsPerBucket(); - - EXPORT void hbrPredicate(); - void setTraceFlags(uint32_t flags) { fTraceFlags = flags; @@ -577,12 +453,9 @@ class ResourceManager std::string fExeMgrStr; inline static const std::string fHashJoinStr = "HashJoin"; - inline static const std::string fHashBucketReuseStr = "HashBucketReuse"; inline static const std::string fJobListStr = "JobList"; inline static const std::string fPrimitiveServersStr = "PrimitiveServers"; /*static const*/ std::string fSystemConfigStr; - inline static const std::string fTupleWSDLStr = "TupleWSDL"; - inline static const std::string fZDLStr = "ZDL"; inline static const std::string fExtentMapStr = "ExtentMap"; /*static const*/ std::string fDMLProcStr; /*static const*/ std::string fBatchInsertStr; @@ -596,7 +469,6 @@ class ResourceManager unsigned fHjNumThreads; uint32_t fJlProcessorThreadsPerScan; uint32_t fJlNumScanReceiveThreads; - uint8_t fTwNumThreads; uint32_t fJlMaxOutstandingRequests; /* old HJ support */ diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index d8e161301..56393792e 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -9,61 +9,6 @@ 8601 unassigned - - 0.0.0.0 - 8602 - - - 127.0.0.1 - 8603 - - - 127.0.0.1 - 8606 - - - 127.0.0.1 - 8604 - - - 0.0.0.0 - 8605 - - - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8800 - - - 0.0.0.0 - 8800 - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8622 - - - 0.0.0.0 - 8622 - - - 127.0.0.1 - 8622 - 127.0.0.1 8630 @@ -85,7 +30,6 @@ 128 10K 0 - 13 512 512 @@ -96,206 +40,40 @@ y - - - + + + 127.0.0.1 8620 - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - columnstore-1 pm1 - unassigned pm1 - 1 - 3 - 12 // 2.5 minutes - 1 - /var/lib/columnstore/data1 - /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves - /var/lib/columnstore/data1/systemFiles/dbrm/tablelocks + /var/lib/columnstore/data1 + /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves + /var/lib/columnstore/data1/systemFiles/dbrm/tablelocks 15 100000 - 90 - 80 - 70 - 10 - 0.0.0.0 - 128M 10 - 10 - 120 - restartSystem - n 95 OFF /rdwrscratch - /columnstore_tmp_files /tmp/columnstore_tmp_files - dm - Director Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned um User Module - SIMPLEX 0 0.0.0.0 unassigned @@ -318,7 +96,6 @@ unassigned pm Performance Module - SIMPLEX 1 127.0.0.1 localhost @@ -340,30 +117,24 @@ 1 1 - - 0 - unassigned - 0.0.0.0 - ENABLED - 1000 - /var/lib/columnstore/data1/systemFiles/dbrm/SMTxnID + /var/lib/columnstore/data1/systemFiles/dbrm/SMTxnID + One version buffer file will be put on each DB root. --> 1GB - /var/lib/columnstore/data1/systemFiles/dbrm/oidbitmap + /var/lib/columnstore/data1/systemFiles/dbrm/oidbitmap 3000 - /var/log/mariadb/columnstore/data/bulk - /var/lib/columnstore/data1/systemFiles/bulkRollback + /var/log/mariadb/columnstore/data/bulk + /var/lib/columnstore/data1/systemFiles/bulkRollback 98 1 @@ -378,55 +149,10 @@ 8700 pm1 - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - + 1 @@ -439,40 +165,13 @@ 50 - n - y - y 2 y n internal internal - /etc/rsyslog.d/49-columnstore.conf - unassigned - autoassign - unassigned - unassigned - unassigned - gp2 - unassigned - unassigned - unassigned - gp2 - unassigned - y - y - 0 - unassigned - unassigned - n - 0 - unassigned - n - 3306 - /dev/xvd - /var/lock/subsys /etc/profile.d/columnstoreAlias.sh - + - 64 1G 25% 100 N Y - Snappy + Snappy 16K @@ -511,11 +209,6 @@ 100 - - 1M - 1M - 512 - @@ -540,17 +233,17 @@ Y - Snappy + Snappy 127.0.0.1 0 - - 30 - N - - - - + + 30 + N + + + + diff --git a/oam/etc/Columnstore.xml.singleserver b/oam/etc/Columnstore.xml.singleserver deleted file mode 100644 index 0f70acc4e..000000000 --- a/oam/etc/Columnstore.xml.singleserver +++ /dev/null @@ -1,547 +0,0 @@ - - - - - 127.0.0.1 - 8601 - pm1 - - - 0.0.0.0 - 8602 - - - 127.0.0.1 - 8603 - - - 127.0.0.1 - 8606 - - - 127.0.0.1 - 8604 - - - 0.0.0.0 - 8605 - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8800 - - - 0.0.0.0 - 8800 - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8622 - - - 0.0.0.0 - 8622 - - - 127.0.0.1 - 8622 - - - 127.0.0.1 - 8630 - - - 127.0.0.1 - 8612 - - - 127.0.0.1 - 8614 - - - 10000 - - - 1 - 8 - 128 - - 10K - 0 - 13 - 512 - 512 - - 1 - 0 - n - - - - 95 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - C - columnstore-1 - pm1 - unassigned - - - 1 - /var/lib/columnstore/data1 - /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves - /var/lib/columnstore/data1/systemFiles/dbrm/tablelocks - 20 - 100000 - 90 - 80 - 70 - /tmp - /tmp - 10 - 0.0.0.0 - 128M - 10 - /tmp/columnstore_tmp_files - 10 - 3 - 10 - 120 - restartSystem - n - 95 - OFF - - /tmp/rdwrscratch - - /tmp/columnstore_tmp_files - - - dm - Director Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned - um - User Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned - pm - Performance Module - SIMPLEX - 1 - 127.0.0.1 - localhost - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - 1 - 1 - - - 0 - unassigned - 0.0.0.0 - ENABLED - - - 1000 - /var/lib/columnstore/data1/systemFiles/dbrm/CalpontShm - /var/lib/columnstore/data1/systemFiles/dbrm/SMTxnID - - - /var/lib/columnstore/data1/systemFiles/dbrm/CalpontSessionMonitorShm - 10 - - - - 1GB - - - - - /var/lib/columnstore/data1/systemFiles/dbrm/oidbitmap - - 3000 - - - /var/lib/columnstore/data/bulk - /var/lib/columnstore/data/bulk/rollback - 98 - 1 - - - 1 - 127.0.0.1 - 8616 - - - - 127.0.0.1 - 8700 - pm1 - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - - - 50 - - 1 - 0 - 0 - 65536 - 2K - 200 - 0 - - - n - y - y - 2 - n - n - internal - internal - unassigned - unassigned - unassigned - rpm - unassigned - unassigned - us-east-1 - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - n - n - 0 - unassigned - unassigned - n - unassigned - unassigned - 3306 - /var/lock/subsys - - - - 4 - 0x0 - - - 128 - 128K - 64 - 1G - 25% - 10% - 100 - N - Y - - - 16K - 16 - 1 - - - - - 100 - - - 1M - 1M - 512 - - - - - - - - - - 127.0.0.1 - 3306 - root - - - - - - - N - - - N - - - Y - - - 127.0.0.1 - 0 - - - 30 - N - - diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 015f09d83..471535848 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -167,7 +167,6 @@ void Oam::getSystemConfig(const std::string& moduletype, ModuleTypeConfig& modul const string Section = "SystemModuleConfig"; const string MODULE_TYPE = "ModuleType"; const string MODULE_DESC = "ModuleDesc"; - const string MODULE_RUN_TYPE = "RunType"; const string MODULE_COUNT = "ModuleCount"; const string MODULE_DISABLE_STATE = "ModuleDisableState"; const string MODULE_CPU_CRITICAL = "ModuleCPUCriticalThreshold"; @@ -198,7 +197,6 @@ void Oam::getSystemConfig(const std::string& moduletype, ModuleTypeConfig& modul string ModuleCount = MODULE_COUNT + itoa(moduleTypeID); string ModuleType = MODULE_TYPE + itoa(moduleTypeID); string ModuleDesc = MODULE_DESC + itoa(moduleTypeID); - string ModuleRunType = MODULE_RUN_TYPE + itoa(moduleTypeID); string ModuleCPUCriticalThreshold = MODULE_CPU_CRITICAL + itoa(moduleTypeID); string ModuleCPUMajorThreshold = MODULE_CPU_MAJOR + itoa(moduleTypeID); string ModuleCPUMinorThreshold = MODULE_CPU_MINOR + itoa(moduleTypeID); @@ -216,7 +214,6 @@ void Oam::getSystemConfig(const std::string& moduletype, ModuleTypeConfig& modul moduletypeconfig.ModuleCount = strtol(sysConfig->getConfig(Section, ModuleCount).c_str(), 0, 0); moduletypeconfig.ModuleType = sysConfig->getConfig(Section, ModuleType); moduletypeconfig.ModuleDesc = sysConfig->getConfig(Section, ModuleDesc); - moduletypeconfig.RunType = sysConfig->getConfig(Section, ModuleRunType); moduletypeconfig.ModuleCPUCriticalThreshold = strtol(sysConfig->getConfig(Section, ModuleCPUCriticalThreshold).c_str(), 0, 0); moduletypeconfig.ModuleCPUMajorThreshold = diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index c2df5bf24..b4fbd207b 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -115,7 +115,6 @@ const std::string UnassignedName = "unassigned"; const std::string configSections[] = {"SystemConfig", "SystemModuleConfig", "SystemModuleConfig", - "SystemExtDeviceConfig", "SessionManager", "VersionBuffer", "OIDManager", @@ -235,7 +234,6 @@ struct ModuleTypeConfig_s { std::string ModuleType; //!< Module Type std::string ModuleDesc; //!< Module Description - std::string RunType; //!< Run Type uint16_t ModuleCount; //!< Module Equipage Count uint16_t ModuleCPUCriticalThreshold; //!< CPU Critical Threahold % uint16_t ModuleCPUMajorThreshold; //!< CPU Major Threahold % diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp index 0f8abad74..d7bf28a9b 100644 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -62,7 +62,6 @@ string rootPassword = ""; string debug_flag = "0"; string mysqlpw = " "; string tmpDir; -string ProfileFile; int runningThreads = 0; pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; @@ -121,9 +120,8 @@ void* childReportThread(threadInfo_t* st) cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl; - string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + " '. " + ProfileFile + ";" + - reportType + "Report.sh " + remoteModuleName + "' " + debug_flag + " - forcetty"; - + string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + ";" + reportType + "Report.sh " + + remoteModuleName + "' " + debug_flag + " - forcetty"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) @@ -333,7 +331,7 @@ int main(int argc, char* argv[]) } // get Local Module Name and Server Install Indicator - string singleServerInstall; + string singleServerInstall = "n"; oamModuleInfo_t st; @@ -348,15 +346,6 @@ int main(int argc, char* argv[]) exit(-1); } - try - { - oam.getSystemConfig("SingleServerInstall", singleServerInstall); - } - catch (...) - { - singleServerInstall = "y"; - } - if (argc == 1) { argv[1] = &helpArg[0]; @@ -592,17 +581,6 @@ int main(int argc, char* argv[]) exit(-1); } - // Get Profile file - try - { - ProfileFile = sysConfig->getConfig(InstallSection, "ProfileFile"); - } - catch (...) - { - cout << "ERROR: Problem getting ProfileFile" << endl; - exit(-1); - } - string ModuleSection = "SystemModuleConfig"; for (unsigned int i = 0; i < sysModuleTypeConfig.moduletypeconfig.size(); i++) diff --git a/oamapps/columnstoreSupport/mcsSupportUtil.cpp b/oamapps/columnstoreSupport/mcsSupportUtil.cpp index 4c3e7106e..586a342e5 100644 --- a/oamapps/columnstoreSupport/mcsSupportUtil.cpp +++ b/oamapps/columnstoreSupport/mcsSupportUtil.cpp @@ -160,7 +160,6 @@ void getModuleTypeConfig(FILE* pOutputFile) fprintf(pOutputFile, "ModuleType '%s' Configuration information\n", moduletype.c_str()); fprintf(pOutputFile, "ModuleDesc = %s\n", systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc.c_str()); - fprintf(pOutputFile, "RunType = %s\n", systemmoduletypeconfig.moduletypeconfig[i].RunType.c_str()); fprintf(pOutputFile, "ModuleCount = %i\n", moduleCount); if (moduleCount > 0) @@ -379,23 +378,16 @@ void getStorageConfig(FILE* pOutputFile) string volumeName = oam::UnassignedName; string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt); string deviceName = oam::UnassignedName; - string amazonDeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(*pt); - string amazondeviceName = oam::UnassignedName; try { oam.getSystemConfig(volumeNameID, volumeName); oam.getSystemConfig(deviceNameID, deviceName); - oam.getSystemConfig(amazonDeviceNameID, amazondeviceName); } catch (...) { continue; } - - fprintf(pOutputFile, - "Amazon EC2 Volume Name/Device Name/Amazon Device Name for DBRoot%u: %s, %s, %s", *pt, - volumeName.c_str(), deviceName.c_str(), amazondeviceName.c_str()); } } catch (exception& e) @@ -412,22 +404,16 @@ void getStorageConfig(FILE* pOutputFile) string volumeName = oam::UnassignedName; string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt1); string deviceName = oam::UnassignedName; - string amazonDeviceNameID = "PMVolumeAmazonDeviceName" + oam.itoa(*pt1); - string amazondeviceName = oam::UnassignedName; try { oam.getSystemConfig( volumeNameID, volumeName); oam.getSystemConfig( deviceNameID, deviceName); - oam.getSystemConfig( amazonDeviceNameID, amazondeviceName); } catch (...) { continue; } - - fprintf(pOutputFile,"Amazon EC2 Volume Name/Device Name/Amazon Device Name for DBRoot%u: %s, %s, - %s",*pt1,volumeName.c_str(),deviceName.c_str(),amazondeviceName.c_str()); }*/ } diff --git a/primitives/primproc/bpp.h b/primitives/primproc/bpp.h index 2190129bb..8d9161695 100644 --- a/primitives/primproc/bpp.h +++ b/primitives/primproc/bpp.h @@ -27,7 +27,6 @@ // Copyright: See COPYING file that comes with this distribution // // -#include "primitivemsg.h" #include "bytestream.h" #include "messagequeue.h" #include "serializeable.h" diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index 7688a9f9f..69fa86ccc 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -117,7 +117,6 @@ void DictStep::prep(int8_t outputType, bool makeAbsRids) primMsg->ism.Interleave = 0; primMsg->ism.Flags = 0; - // primMsg->ism.Flags = PrimitiveMsg::planFlagsToPrimFlags(traceFlags); primMsg->ism.Command = DICT_SIGNATURE; primMsg->ism.Size = bufferSize; primMsg->ism.Type = 2; diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 260a3ca96..4d761899a 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -601,7 +601,6 @@ int ServicePrimProc::Child() if (temp >= 0) maxPct = temp; - // @bug4507, configurable pm aggregation AggregationMemoryCheck // We could use this same mechanism for other growing buffers. int aggPct = 95; temp = toInt(cf->getConfig("SystemConfig", "MemoryCheckPercent")); diff --git a/primitives/primproc/umsocketselector.cpp b/primitives/primproc/umsocketselector.cpp index 7b58fa73a..5cdd5f383 100644 --- a/primitives/primproc/umsocketselector.cpp +++ b/primitives/primproc/umsocketselector.cpp @@ -119,7 +119,6 @@ void UmSocketSelector::loadUMModuleInfo() std::cout << "ModuleConfig for type: " << UM_MODTYPE << std::endl; std::cout << "ModuleDesc = " << moduleTypeConfig.ModuleDesc << std::endl; std::cout << "ModuleCount = " << moduleCount << std::endl; - std::cout << "RunType = " << moduleTypeConfig.RunType << std::endl; #endif if (moduleCount > 0) diff --git a/utils/configcpp/tdriver2.cpp b/utils/configcpp/tdriver2.cpp index e652ac045..f6d00cc49 100644 --- a/utils/configcpp/tdriver2.cpp +++ b/utils/configcpp/tdriver2.cpp @@ -65,7 +65,6 @@ class WOConfigFileTest : public CppUnit::TestFixture void test1() { WriteOnceConfig woc(cf); - CPPUNIT_ASSERT(woc.owns("PrimitiveServers", "LBID_Shift")); CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRootCount")); CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRMRoot")); @@ -73,9 +72,6 @@ class WOConfigFileTest : public CppUnit::TestFixture int vali; - vali = Config::fromText(woc.getConfig("PrimitiveServers", "LBID_Shift")); - CPPUNIT_ASSERT(vali == 13); - woc.setConfig("SystemConfig", "DBRootCount", "10"); vali = Config::fromText(woc.getConfig("SystemConfig", "DBRootCount")); CPPUNIT_ASSERT(vali == 10); diff --git a/utils/winport/fixup.cpp b/utils/winport/fixup.cpp index dfba53a50..2e909d564 100644 --- a/utils/winport/fixup.cpp +++ b/utils/winport/fixup.cpp @@ -258,22 +258,6 @@ int fixupCalpontXML() cout << "Fixing " << section << "." << parm << " = " << val << endl; } - // Fixup WES - section = "pm1_WriteEngineServer"; - parm = "IPAddr"; - val = cf->getConfig(section, parm); - - if (val.empty()) - { - val = "127.0.0.1"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - parm = "Port"; - val = "8630"; - cf->setConfig(section, parm, val); - cout << "Adding " << section << "." << parm << " = " << val << endl; - } - // Fixup TableLockSaveFile section = "SystemConfig"; parm = "TableLockSaveFile"; From 4bb3743110570d345380e42604868c6f9075915f Mon Sep 17 00:00:00 2001 From: Commander thrashdin Date: Wed, 20 Apr 2022 16:04:14 +0300 Subject: [PATCH 45/55] Added word bytes --- writeengine/server/we_ddlcommandproc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 6e8217bb1..5ad015aa7 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -487,7 +487,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err (dataType != CalpontSystemCatalog::TEXT)) { ostringstream os; - os << "char, varchar and varbinary length may not exceed 8000"; + os << "char, varchar and varbinary length may not exceed 8000 bytes"; throw std::runtime_error(os.str()); } } @@ -872,7 +872,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) (dataType != CalpontSystemCatalog::TEXT)) { ostringstream os; - os << "char, varchar and varbinary length may not exceed 8000"; + os << "char, varchar and varbinary length may not exceed 8000 bytes"; throw std::runtime_error(os.str()); } } From 53695cf0db35ec398aa2d63133ba471406d18ea0 Mon Sep 17 00:00:00 2001 From: mariadb-RomanNavrotskiy Date: Fri, 22 Apr 2022 15:07:15 +0200 Subject: [PATCH 46/55] add container image steps disable suse and centos8 builds --- .drone.jsonnet | 67 ++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 38 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 516cad409..8329b8a13 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,10 +1,9 @@ local events = ['pull_request', 'cron']; local platforms = { - develop: ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], - 'develop-6': ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], - 'develop-5': ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], - + develop: ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], + 'develop-6': ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], + 'develop-5': ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'], }; local platforms_arm = { @@ -13,10 +12,10 @@ local platforms_arm = { }; local any_branch = '**'; -local platforms_custom = ['opensuse/leap:15', 'centos:7', 'centos:8', 'rockylinux:8', 'debian:10', 'ubuntu:20.04']; -local platforms_arm_custom = ['centos:8']; +local platforms_custom = ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04']; +local platforms_arm_custom = ['rockylinux:8']; -local platforms_mtr = ['centos:7', 'centos:8', 'ubuntu:20.04']; +local platforms_mtr = ['centos:7', 'rockylinux:8', 'ubuntu:20.04']; local server_ref_map = { develop: '10.8', @@ -95,6 +94,9 @@ local Pipeline(branch, platform, event, arch='amd64') = { local regression_ref = if (std.split(branch, '-')[0] == 'develop') then branch else 'develop-6', local branchp = if (branch == '**') then '' else branch, + local container_tags = if (event == 'cron') then [branch, branch + '-' + std.strReplace(event, '_', '-') + '-${DRONE_BUILD_NUMBER}'] else [branch + '-' + std.strReplace(event, '_', '-') + '-${DRONE_BUILD_NUMBER}'], + local container_version = branch + '/' + event + '/${DRONE_BUILD_NUMBER}/' + arch, + local server_remote = if (std.split(branch, '-')[0] == 'columnstore' || branch == 'develop-6') then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server', local pipeline = self, @@ -118,6 +120,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { delete: 'true', }, }, + _volumes:: { mdb: { name: 'mdb', @@ -277,46 +280,34 @@ local Pipeline(branch, platform, event, arch='amd64') = { }, dockerfile:: { name: 'dockerfile', - image: 'docker:git', - volumes: [pipeline._volumes.docker], + image: 'alpine/git', commands: [ - 'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-community-columnstore-docker.git', - 'cd mariadb-community-columnstore-docker', - 'apk add --no-cache patch', - 'patch Dockerfile ../Dockerfile.patch', - 'cp ../result/MariaDB-common-10* ../result/MariaDB-client-10* ../result/MariaDB-server-10* ../result/MariaDB-shared-10* ../result/MariaDB-columnstore-engine-10* ./', + 'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-enterprise-columnstore-docker', + "sed -i 's|dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup|cspkg.s3.amazonaws.com/cs_repo|' mariadb-enterprise-columnstore-docker/Dockerfile", ], }, - ecr:: { - name: 'ecr', - image: 'plugins/ecr', - settings: { - registry: '866067714787.dkr.ecr.us-east-1.amazonaws.com', - repo: 'columnstore/engine', - context: 'mariadb-community-columnstore-docker', - dockerfile: 'mariadb-community-columnstore-docker/Dockerfile', - access_key: { - from_secret: 'aws_access_key_id', - }, - secret_key: { - from_secret: 'aws_secret_access_key', - }, - }, - }, - docker:: { - name: 'docker', + dockerhub:: { + name: 'dockerhub', image: 'plugins/docker', + environment: { + VERSION: container_version, + }, settings: { - repo: 'romcheck/columnstore', - context: '/drone/src/mariadb-community-columnstore-docker', - dockerfile: 'mariadb-community-columnstore-docker/Dockerfile', - username: 'romcheck', + repo: 'mariadb/enterprise-columnstore-dev', + context: 'mariadb-enterprise-columnstore-docker', + dockerfile: 'mariadb-enterprise-columnstore-docker/Dockerfile', + build_args_from_env: ['VERSION'], + tags: container_tags, + username: { + from_secret: 'dockerhub_user', + }, password: { - from_secret: 'dockerhub_token', + from_secret: 'dockerhub_password', }, }, }, + kind: 'pipeline', type: 'docker', name: std.join(' ', [branch, platform, event, arch]), @@ -427,7 +418,7 @@ local Pipeline(branch, platform, event, arch='amd64') = { ] + [pipeline.publish()] + (if (event == 'cron') || (event == 'push') then [pipeline.publish('pkg latest', 'latest')] else []) + - // (if (platform == 'centos:8' && event == 'cron') then [pipeline.dockerfile] + [pipeline.docker] + [pipeline.ecr] else []) + + (if (event != 'custom') && (platform == 'rockylinux:8') && (arch == 'amd64') && (branch != 'develop-5') then [pipeline.dockerfile] + [pipeline.dockerhub] else []) + [pipeline.smoke] + [pipeline.smokelog] + (if (std.member(platforms_mtr, platform)) then [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish('mtr')] else []) + From c25ae4f378047311251ba608aebf14dad61c9382 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 18 Apr 2022 16:16:45 +0000 Subject: [PATCH 47/55] Use external boost 1.78 --- .gitignore | 1 + CMakeLists.txt | 7 +--- cmake/boost.CMakeLists.txt.in | 64 ++++++++++++++++---------------- cmake/boost.cmake | 16 ++++++-- storage-manager/CMakeLists.txt | 18 ++++----- tools/rebuildEM/CMakeLists.txt | 2 +- utils/common/CMakeLists.txt | 2 +- utils/idbdatafile/CMakeLists.txt | 2 +- utils/joiner/joinpartition.cpp | 1 - utils/threadpool/CMakeLists.txt | 1 + 10 files changed, 60 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index d1f85955f..3ed72171a 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,4 @@ mariadb-columnstore-regression-test/ build/Testing/ tests/*\[1\]_tests.cmake tests/*\[1\]_include.cmake +.boost diff --git a/CMakeLists.txt b/CMakeLists.txt index a2fcc7954..ca8542fbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,12 +95,7 @@ SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE(columnstore_version) INCLUDE(misc) - -FIND_PACKAGE(Boost 1.53.0 COMPONENTS system filesystem thread regex date_time chrono atomic) -IF (NOT Boost_FOUND) - MESSAGE_ONCE(CS_NO_BOOST "Required Boost libraries not found!") - return() -ENDIF() +INCLUDE(boost) FIND_PACKAGE(BISON) IF (NOT BISON_FOUND) diff --git a/cmake/boost.CMakeLists.txt.in b/cmake/boost.CMakeLists.txt.in index 5df75c405..fa7c975fe 100644 --- a/cmake/boost.CMakeLists.txt.in +++ b/cmake/boost.CMakeLists.txt.in @@ -1,33 +1,33 @@ -cmake_minimum_required(VERSION @CMAKE_VERSION@) - -include(ExternalProject) - -if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(_toolset "gcc") -elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") - set(_toolset "clang") -elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") - set(_toolset "intel-linux") -endif() - -set(_b2args link=shared;threading=multi;variant=release;toolset=${_toolset};--with-system;--with-filesystem;--with-thread;--with-regex;--with-date_time) - -ExternalProject_Add(boost - PREFIX build - URL https://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.zip - URL_HASH SHA256=ae85620e810b87a03e1acf8bbf0d4ad87c0cf7040cf6a4e1d8958488ebe42e7e - DOWNLOAD_NO_PROGRESS TRUE - UPDATE_COMMAND "" - CONFIGURE_COMMAND /bootstrap.sh - --with-toolset=${_toolset} - --prefix=${CMAKE_CURRENT_SOURCE_DIR}/../boost - --with-libraries=system,filesystem,thread,regex,date_time - BUILD_COMMAND /b2 -q ${_b2args} - LOG_BUILD TRUE - BUILD_IN_SOURCE TRUE - INSTALL_COMMAND /b2 -q install ${_b2args} - LOG_INSTALL TRUE -) - -unset(_b2args) +cmake_minimum_required(VERSION @CMAKE_VERSION@) +project (BoostExternal) +include(ExternalProject) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + set(_toolset "gcc") +elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") + set(_toolset "clang") +elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") + set(_toolset "intel-linux") +endif() + +set(INSTALL_LOCATION ${CMAKE_BINARY_DIR}/boost-lib) + +set(_cxxargs "-fPIC -DBOOST_NO_AUTO_PTR -fvisibility=default") +set(_b2args cxxflags=${_cxxargs};cflags=-fPIC;threading=multi; toolset=${_toolset} --without-python;--prefix=${INSTALL_LOCATION}) + +ExternalProject_Add(boost + PREFIX build + GIT_REPOSITORY https://github.com/boostorg/boost.git + GIT_TAG boost-1.78.0 + DOWNLOAD_NO_PROGRESS TRUE + UPDATE_COMMAND "" + CONFIGURE_COMMAND /bootstrap.sh + BUILD_COMMAND /b2 -q ${_b2args} + LOG_BUILD TRUE + BUILD_IN_SOURCE TRUE + INSTALL_COMMAND /b2 -q install ${_b2args} + LOG_INSTALL TRUE +) + +unset(_b2args) unset(_toolset) \ No newline at end of file diff --git a/cmake/boost.cmake b/cmake/boost.cmake index ae129287d..25ca5c3fb 100644 --- a/cmake/boost.cmake +++ b/cmake/boost.cmake @@ -3,9 +3,11 @@ configure_file(${CMAKE_CURRENT_LIST_DIR}/boost.CMakeLists.txt.in ${CMAKE_CURRENT_BINARY_DIR}/.boost/CMakeLists.txt @ONLY) +SET(BOOST_ROOT ${CMAKE_CURRENT_BINARY_DIR}/.boost/boost-lib) +message ("-- External: Configuring Boost") execute_process( - COMMAND ${CMAKE_COMMAND} . + COMMAND ${CMAKE_COMMAND} . -G "${CMAKE_GENERATOR}" -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} @@ -17,6 +19,7 @@ if(${_exec_ret}) message(FATAL_ERROR "Error ${_exec_ret} configuring boost dependency.") endif() +message ("-- External: Building Boost") execute_process( COMMAND ${CMAKE_COMMAND} --build . WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/.boost @@ -29,6 +32,13 @@ endif() unset(_exec_ret) -set(BOOST_ROOT ${CMAKE_CURRENT_BINARY_DIR}/boost) -find_package(Boost 1.55.0 REQUIRED COMPONENTS system filesystem thread regex date_time) +message("Boost installed localy at ${BOOST_ROOT}") + + +SET(Boost_USE_STATIC_LIBS ON) +FIND_PACKAGE(Boost 1.78.0 COMPONENTS system filesystem thread regex date_time chrono atomic) +IF (NOT Boost_FOUND) + MESSAGE_ONCE(CS_NO_BOOST "Required Boost libraries not found!") + return() +ENDIF() diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index cd71417b8..695d02d03 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -3,7 +3,7 @@ project(storagemanager) include_directories(include ${CMAKE_BINARY_DIR}/include ${ENGINE_UTILS_COMMON_INCLUDE} ${S3API_DIR}) -set(storagemanager_SRCS +set(storagemanager_SRCS src/AppendTask.cpp src/ClientRequestProcessor.cpp src/ListDirectoryTask.cpp @@ -62,12 +62,12 @@ set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) add_library(storagemanager SHARED ${storagemanager_SRCS}) add_dependencies(storagemanager marias3) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) -set_property(TARGET storagemanager PROPERTY CXX_STANDARD 11) +target_link_libraries(storagemanager Boost::chrono Boost::system Boost::thread Boost::filesystem Boost::regex pthread ${S3API_DEPS}) +set_property(TARGET storagemanager PROPERTY CXX_STANDARD 20) add_executable(StorageManager src/main.cpp) target_link_libraries(StorageManager storagemanager) -set_property(TARGET StorageManager PROPERTY CXX_STANDARD 11) +set_property(TARGET StorageManager PROPERTY CXX_STANDARD 20) set(TMPDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) @@ -75,7 +75,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_executable(unit_tests src/unit_tests.cpp) target_compile_definitions(unit_tests PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) target_link_libraries(unit_tests storagemanager) -set_property(TARGET unit_tests PROPERTY CXX_STANDARD 11) +set_property(TARGET unit_tests PROPERTY CXX_STANDARD 20) add_executable(testS3Connection src/testS3Connection.cpp) target_compile_definitions(testS3Connection PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) @@ -96,10 +96,10 @@ add_custom_command( ${CMAKE_CURRENT_BINARY_DIR}/test_data # COMMAND ${CMAKE_COMMAND} -E copy # ../bin/unit_tests # what is putting our bins in ../bin? -# ${CMAKE_CURRENT_BINARY_DIR} +# ${CMAKE_CURRENT_BINARY_DIR} ) -# The includes and lib linkages required to link against cloudio ... +# The includes and lib linkages required to link against cloudio ... # pretty crazy. When lib dependencies are eventually config'd right, # change this to only include and link against cloudio. include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES}) @@ -141,12 +141,12 @@ install(TARGETS StorageManager smcat smput smls smrm testS3Connection COMPONENT columnstore-engine ) -install(FILES storagemanager.cnf +install(FILES storagemanager.cnf DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT columnstore-engine ) -install(FILES storagemanager.cnf +install(FILES storagemanager.cnf RENAME storagemanager.cnf.example DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT columnstore-engine diff --git a/tools/rebuildEM/CMakeLists.txt b/tools/rebuildEM/CMakeLists.txt index 54bd14e6b..5d27c2f00 100644 --- a/tools/rebuildEM/CMakeLists.txt +++ b/tools/rebuildEM/CMakeLists.txt @@ -2,5 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rebuildEM_SRCS main.cpp rebuildEM.cpp) add_executable(mcsRebuildEM ${rebuildEM_SRCS}) -target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem boost_system) +target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} Boost::system Boost::filesystem) install(TARGETS mcsRebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index 17d78d476..59e072ec1 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -16,7 +16,7 @@ set(common_LIB_SRCS add_library(common SHARED ${common_LIB_SRCS}) -target_link_libraries(common boost_filesystem) +target_link_libraries(common Boost::filesystem) add_dependencies(common loggingcpp) diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index c51cc60fb..a4b62074f 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -15,7 +15,7 @@ set(idbdatafile_LIB_SRCS add_library(idbdatafile SHARED ${idbdatafile_LIB_SRCS}) -target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS}) +target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} Boost::filesystem Boost::system) target_compile_definitions(idbdatafile PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) diff --git a/utils/joiner/joinpartition.cpp b/utils/joiner/joinpartition.cpp index c31932046..c1f6fff18 100644 --- a/utils/joiner/joinpartition.cpp +++ b/utils/joiner/joinpartition.cpp @@ -155,7 +155,6 @@ JoinPartition::JoinPartition(const JoinPartition& jp, bool splitMode) , nextSmallOffset(0) , nextLargeOffset(0) { - boost::posix_time::ptime t; ostringstream os; fileMode = true; diff --git a/utils/threadpool/CMakeLists.txt b/utils/threadpool/CMakeLists.txt index d20887ed6..519ba3b5c 100644 --- a/utils/threadpool/CMakeLists.txt +++ b/utils/threadpool/CMakeLists.txt @@ -9,5 +9,6 @@ set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool add_library(threadpool SHARED ${threadpool_LIB_SRCS}) add_dependencies(threadpool loggingcpp) +target_link_libraries(threadpool Boost::chrono) install(TARGETS threadpool DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) From 4c26e4f960dab8b7b81b79dc588ea5eec4b6bbe1 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 30 Mar 2022 08:57:05 +0000 Subject: [PATCH 48/55] MCOL-4912 This patch introduces Extent Map index to improve EM scaleability EM scaleability project has two parts: phase1 and phase2. This is phase1 that brings EM index to speed up(from O(n) down to the speed of boost::unordered_map) EM lookups looking for tuple to turn it into LBID, e.g. most bulk insertion meta info operations. The basis is boost::shared_managed_object where EMIndex is stored. Whilst it is not debug-friendly it allows to put a nested structs into shmem. EMIndex has 3 tiers. Top down description: vector of dbroots, map of oids to partition vectors, partition vectors that have EM indices. Separate EM methods now queries index before they do EM run. EMIndex has a separate shmem file with the fixed id MCS-shm-00060001. --- dbcon/mysql/ha_mcs_client_udfs.cpp | 64 +- tests/CMakeLists.txt | 5 + tests/brm-em-standalone.cpp | 1914 +++++++++++++++++++++++++ tools/clearShm/main.cpp | 36 +- utils/rwlock/rwlock.h | 7 + versioning/BRM/brmshmimpl.cpp | 172 ++- versioning/BRM/brmshmimpl.h | 77 +- versioning/BRM/dbrm.cpp | 23 +- versioning/BRM/dbrm.h | 9 +- versioning/BRM/extentmap.cpp | 1246 +++++++++++----- versioning/BRM/extentmap.h | 220 ++- versioning/BRM/lock_grabber.cpp | 61 +- versioning/BRM/lock_state.cpp | 47 +- versioning/BRM/mastersegmenttable.cpp | 1 + versioning/BRM/mastersegmenttable.h | 4 +- versioning/BRM/shmkeys.cpp | 3 +- versioning/BRM/shmkeys.h | 1 + versioning/BRM/slavedbrmnode.cpp | 5 + versioning/BRM/slavedbrmnode.h | 1 + versioning/BRM/slavenode.cpp | 2 + writeengine/bulk/cpimport.cpp | 22 +- writeengine/xml/we_xmlgenproc.cpp | 21 +- writeengine/xml/we_xmlgenproc.h | 3 + writeengine/xml/we_xmljob.cpp | 11 +- writeengine/xml/we_xmljob.h | 2 +- 25 files changed, 3498 insertions(+), 459 deletions(-) create mode 100644 tests/brm-em-standalone.cpp diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 7581b67df..35e8ff664 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -52,6 +52,8 @@ extern "C" const char* SetParmsPrelude = "Updated "; const char* SetParmsError = "Invalid parameter: "; const char* InvalidParmSize = "Invalid parameter size: Input value cannot be larger than "; + const char* MsgEMIndexSizeInitErrMsg = "mcs_emindex_size() takes no arguments"; + const char* MsgEMIndexFreeInitErrMsg = "mcs_emindex_free() takes no arguments"; const size_t Plen = strlen(SetParmsPrelude); const size_t Elen = strlen(SetParmsError); @@ -1156,4 +1158,64 @@ extern "C" { } +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long mcs_emindex_size(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + DBRM dbrm; + return dbrm.EMIndexShmemSize(); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcs_emindex_size_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, MsgEMIndexSizeInitErrMsg); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcs_emindex_size_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + long long mcs_emindex_free(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) + { + DBRM dbrm; + return dbrm.EMIndexShmemFree(); + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcs_emindex_free_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + if (args->arg_count != 0) + { + strcpy(message, MsgEMIndexFreeInitErrMsg); + return 1; + } + + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcs_emindex_free_deinit(UDF_INIT* initid) + { + } + } // extern "C" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b00c0482a..a8af32368 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,6 +62,11 @@ if (WITH_UNITTESTS) target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) add_test(NAME columnstore:comparators_tests, COMMAND comparators_tests) + # standalone EM routines test + # add_executable(brm_em_standalone brm-em-standalone.cpp) + # target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + # install(TARGETS brm_em_standalone DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) + endif() # Saving this as the example of the microbench diff --git a/tests/brm-em-standalone.cpp b/tests/brm-em-standalone.cpp new file mode 100644 index 000000000..92b410107 --- /dev/null +++ b/tests/brm-em-standalone.cpp @@ -0,0 +1,1914 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2021 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/***************************************************************************** + * $Id: tdriver-dbrm2.cpp 1823 2013-01-21 14:13:09Z rdempsey $ + * + ****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "brm.h" +#include "extentmap.h" +#include "IDBPolicy.h" + +//#define BRM_VERBOSE 1 + +using namespace BRM; +using namespace std; + +int threadStop; +int oid = 1; +uint64_t opCount = 0; +LBID_t lbidCounter = 0; +VER_t nextTxnID = 1; +u_int64_t vbOffset = 0; +pthread_mutex_t pthreadMutex; +const std::vector colWidthsAvailable = {1, 2, 4, 8, 16}; +const DBRootT dbroot = 1; +const uint32_t KibiBlocks = 1024; + +struct Range +{ + LBID_t start, end, nextBlock; + VER_t txnID; + Range() + { + start = end = nextBlock = 0; + txnID = 0; + } +}; + +struct EMEntries +{ + LBID_t LBIDstart; + uint32_t size; + OID_t OID; + uint32_t FBO; + uint32_t HWM; + uint32_t secondHWM; + int32_t txnID; + DBRootT dbroot; + PartitionNumberT partNum; + SegmentT segNum; + struct EMEntries* next; + EMEntries() : HWM(0), secondHWM(0), txnID(0), next(nullptr) + { } + EMEntries(const uint32_t aSize, const OID_t aOid, const uint32_t aFbo, + const LBID_t aLBIDStart, EMEntries* aNext) + : LBIDstart(aLBIDStart), size(aSize), OID(aOid), FBO(aFbo), HWM(0), + secondHWM(0), txnID(0), next(aNext) + { } + EMEntries(const uint32_t aSize, const OID_t aOid, const uint32_t aFbo, + const LBID_t aLBIDStart, EMEntries* aNext, const DBRootT aDbroot, + const PartitionNumberT aPartNum, const SegmentT aSegNum) + : LBIDstart(aLBIDStart), size(aSize), OID(aOid), FBO(aFbo), HWM(0), + secondHWM(0), txnID(0), dbroot(aDbroot), partNum(aPartNum), + segNum(aSegNum), next(aNext) + { } +}; +/* +static void* BRMRunner_2(void* arg) +{ + + vector copyList, copiedList, committedList; + vector::iterator rit; + vector writtenList; + vector::iterator lit; + + pthread_mutex_t listMutex; + int op; + uint32_t randstate; + DBRM* brm; + struct timeval tv; + VER_t txnID; + + pthread_mutex_init(&listMutex, NULL); + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + brm = new DBRM(); + + while (!threadStop) + { + op = rand_r(&randstate) % 9; + + switch (op) + { + case 0: // beginVBCopy + { + int blockCount, size, err; + Range newEntry; + VBRange_v vbRanges; + VBRange_v::iterator vit; + LBIDRange_v ranges; + LBIDRange range; + + size = rand_r(&randstate) % 10000; + + pthread_mutex_lock(&pthreadMutex); + newEntry.start = lbidCounter; + lbidCounter += size; + txnID = nextTxnID++; + pthread_mutex_unlock(&pthreadMutex); + + newEntry.nextBlock = newEntry.start; + newEntry.end = newEntry.start + size; + range.start = newEntry.start; + range.size = size; + + err = brm->beginVBCopy(txnID, dbroot, ranges, vbRanges); + CPPUNIT_ASSERT(err == 0); + + for (blockCount = 0, vit = vbRanges.begin(); vit != vbRanges.end(); vit++) + blockCount += (*vit).size; + + CPPUNIT_ASSERT(blockCount == size); + + pthread_mutex_lock(&listMutex); + copyList.push_back(newEntry); + pthread_mutex_unlock(&listMutex); + + err = brm->beginVBCopy(txnID, dbroot, ranges, vbRanges); + CPPUNIT_ASSERT(err == -1); + break; + } + + case 1: // writeVBEntry + { + int randIndex; + Range* entry; + + pthread_mutex_lock(&listMutex); + + if (copyList.size() == 0) + break; + + randIndex = rand_r(&randstate) % copyList.size(); + entry = &(copyList[randIndex]); + entry->nextBlock++; + txnID = entry->txnID; + break; + } + + default: + cerr << "not finished yet" << endl; + } + } + + return NULL; +} +*/ + +/* +static void* BRMRunner_1(void* arg) +{ + + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. + +#ifdef BRM_VERBOSE + int threadNum = reinterpret_cast(arg); +#endif + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries* head = NULL, *tmp; + struct timeval tv; + DBRM* brm; + ExtentMap em; + vector lbids; + LBID_t lbid; + uint32_t colWidth; + PartitionNumberT partNum; + SegmentT segmentNum; + execplan::CalpontSystemCatalog::ColDataType colDataType; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + brm = new DBRM(); + + + while (!threadStop) + { + auto randNumber = rand_r(&randstate); + op = randNumber % 10; + colWidth = colWidthsAvailable[randNumber % colWidthsAvailable.size()]; + partNum = randNumber % std::numeric_limits::max(); + segmentNum = randNumber % std::numeric_limits::max(); + colDataType = (execplan::CalpontSystemCatalog::ColDataType) (randNumber % (int)execplan::CalpontSystemCatalog::ColDataType::TIMESTAMP); +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: //allocate space for a new file + { + struct EMEntries* newEm; + size_t size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize, err; + uint32_t startBlockOffset; + + + pthread_mutex_lock(&pthreadMutex); + OID = oid++; + opCount++; + pthread_mutex_unlock(&pthreadMutex); + lbids.clear(); + for (size_t i = 0; i < size; ++i) + { + err = brm->createColumnExtent_DBroot(OID, colWidth, dbroot, + partNum, segmentNum, colDataType, lbid, allocdSize, startBlockOffset); + CPPUNIT_ASSERT(err == 0); + lbids.push_back(lbid); + } + + entries = size / brm->getExtentSize(); + + if ((size % brm->getExtentSize()) != 0) + entries++; + + if ((uint32_t)entries != lbids.size()) + cerr << "entries = " << entries << " lbids.size = " << lbids.size() << endl; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0 ; i < entries; i++) + { + + newEm = new EMEntries(brm->getExtentSize(), OID, brm->getExtentSize(), lbids[i], + head, dbroot, partNum, segmentNum); + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 1: //allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries* newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize, err; + uint32_t startBlockOffset; + vector lbids; + LBID_t lbid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + err = brm->createColumnExtentExactFile(oid, colWidth, dbroot, + partNum, segmentNum, colDataType, lbid, allocdSize, startBlockOffset); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm->getExtentSize(); + + if ((size % brm->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + + newEm = new EMEntries((i != entries) ? brm->getExtentSize() : size % brm->getExtentSize(), + oid, lastExtent + (i * brm->getExtentSize()), + lbids[i], head, dbroot, partNum, segmentNum); + newEm->HWM = tmpHWM; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 2: //delete an OID + { + if (listSize == 0) + break; + + struct EMEntries* tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid, err; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + err = brm->deleteOID(oid); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em.checkConsistency(); + break; + } + + case 3: //lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + DBRootT localDbroot; + PartitionNumberT localPartNum; + SegmentT localSegmentNum; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = brm->lookupLocal(target, 0, false, oid, localDbroot, localPartNum, + localSegmentNum, fbo); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em.checkConsistency(); + break; + } + + case 4: //lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = brm->lookupLocal(oid, partNum, segmentNum, offset + tmp->FBO, lbid); + + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << + " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); + em.checkConsistency(); + break; + } + + case 5: //getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, status; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + err = brm->getLocalHWM(tmp->OID, partNum, segmentNum, hwm, status); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM + << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em.checkConsistency(); + break; + } + + case 6: //setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid, err; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + brm->getExtentSize()); + err = brm->setLocalHWM(oid, tmp->partNum, tmp->segNum, hwm); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em.checkConsistency(); + break; + } + + case 7: // renew this EM object + { + delete brm; + brm = new DBRM(); +#ifdef BRM_VERBOSE + cerr << "got a new BRM instance" << endl; +#endif + em.checkConsistency(); + break; + } +#if 0 + case 8: //getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = brm->getBulkInsertVars(lbid, hwm, txnID); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 9: //setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = brm->setBulkInsertVars(tmp->LBIDstart + offset, + tmp->secondHWM, tmp->txnID); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + break; + } +#endif + default: + break; + } + } + + delete brm; + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; +} +*/ +DBRM brm_si; +/* +static void* BRMRunner_si(void* arg) +{ + + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. + +#ifdef BRM_VERBOSE + int threadNum = reinterpret_cast(arg); +#endif + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries* head = NULL, *tmp; + struct timeval tv; + ExtentMap em; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + + while (!threadStop) + { + op = rand_r(&randstate) % 10; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: //allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize, err; + + pthread_mutex_lock(&pthreadMutex); + OID = oid++; + opCount++; + pthread_mutex_unlock(&pthreadMutex); + + err = brm_si.createExtent(size, OID, lbids, allocdSize); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm_si.getExtentSize(); + + if ((size % brm_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0 ; i < entries; i++) + { + + newEm = new EMEntries(); + newEm->size = brm_si.getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * brm_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 1: //allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries* newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize, err; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + err = brm_si.createExtent(size, oid, lbids, allocdSize); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + entries = size / brm_si.getExtentSize(); + + if ((size % brm_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + + newEm = new EMEntries(); + + if (i != entries) + newEm->size = brm_si.getExtentSize(); + else + newEm->size = size % brm_si.getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * brm_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em.checkConsistency(); + break; + } + + case 2: //delete an OID + { + if (listSize == 0) + break; + + struct EMEntries* tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid, err; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + err = brm_si.deleteOID(oid); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em.checkConsistency(); + break; + } + + case 3: //lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = brm_si.lookup(target, 0, false, oid, fbo); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em.checkConsistency(); + break; + } + + case 4: //lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = brm_si.lookup(oid, offset + tmp->FBO, lbid); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << + " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == static_cast(tmp->LBIDstart + offset)); + em.checkConsistency(); + break; + } + + case 5: //getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + err = brm_si.getHWM(tmp->OID, hwm); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM + << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em.checkConsistency(); + break; + } + + case 6: //setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid, err; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + brm_si.getExtentSize()); + err = brm_si.setHWM(oid, hwm); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em.checkConsistency(); + break; + } + + case 7: //getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = brm_si.getBulkInsertVars(lbid, hwm, txnID); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 8: //setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = brm_si.setBulkInsertVars(tmp->LBIDstart + offset, + tmp->secondHWM, tmp->txnID); + pthread_mutex_lock(&pthreadMutex); + opCount++; + pthread_mutex_unlock(&pthreadMutex); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: + break; + } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; +} +*/ + +static void* EMRunner(void* arg) +{ + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. + +#ifdef BRM_VERBOSE + uint64_t threadNum = (uint64_t)arg; +#endif + int op, listSize = 0; + uint32_t randstate; + struct EMEntries* head = NULL, *tmp; + struct timeval tv; + ExtentMap* em; + LBID_t lbid; + uint32_t colWidth; + PartitionNumberT partNum; + SegmentT segmentNum; + execplan::CalpontSystemCatalog::ColDataType colDataType; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + //pthread_mutex_lock(&pthreadMutex); + em = new ExtentMap(); + //pthread_mutex_unlock(&pthreadMutex); + + while (!threadStop) + { + auto randNumber = rand_r(&randstate); + op = randNumber % 10; + + colWidth = colWidthsAvailable[randNumber % colWidthsAvailable.size()]; + partNum = randNumber % std::numeric_limits::max(); + segmentNum = randNumber % std::numeric_limits::max(); + colDataType = (execplan::CalpontSystemCatalog::ColDataType) (randNumber % (int)execplan::CalpontSystemCatalog::ColDataType::TIMESTAMP); +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: //allocate space for a new file + { + vector emEntriesVec; + struct EMEntries* newEm; + size_t numberOfExtents = randNumber % 4 + 1; + int OID; + uint32_t startBlockOffset; + + pthread_mutex_lock(&pthreadMutex); + OID = oid++; + pthread_mutex_unlock(&pthreadMutex); + + em->getExtents(OID, emEntriesVec, false, false, true); + size_t extentsNumberBefore = emEntriesVec.size(); + int allocdsize; + for (size_t i = 0; i < numberOfExtents; ++i) + { + em->createColumnExtent_DBroot(OID, colWidth, dbroot, colDataType, + partNum, segmentNum, lbid, allocdsize, startBlockOffset); + em->confirmChanges(); + + newEm = new EMEntries(allocdsize, OID, startBlockOffset, lbid, + head, dbroot, partNum, segmentNum); + head = newEm; + listSize++; + } + + emEntriesVec.clear(); + em->getExtents(OID, emEntriesVec, false, false, true); + size_t extentsNumberAfter = emEntriesVec.size(); + + CPPUNIT_ASSERT(extentsNumberBefore + numberOfExtents == extentsNumberAfter); + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + //em->checkConsistency(); + break; + } +/* + case 1: //allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries* newEm, *tmp; + size_t size = rand_r(&randstate) % 10; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + uint32_t startBlockOffset; + lbids.clear(); + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + for (size_t i = 0; i < size; ++i) + { + em->createColumnExtent_DBroot(oid, colWidth, dbroot, colDataType, + partNum, segmentNum, lbid, allocdSize, startBlockOffset); + em->confirmChanges(); + lbids.push_back(lbid); + } + + //em->createExtent(size, oid, lbids, allocdSize); + //em->confirmChanges(); + + entries = size / em->getExtentSize(); + + if ((size % em->getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + newEm = new EMEntries((i != entries) ? em->getExtentSize() : size % em->getExtentSize(), + oid, lastExtent + (i * em->getExtentSize()), + lbids[i], head, dbroot, partNum, segmentNum); + newEm->HWM = tmpHWM; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em->checkConsistency(); + break; + } +*/ + + case 2: //delete an OID + { + if (listSize == 0) + break; + + struct EMEntries* tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + em->deleteOID(oid); + em->confirmChanges(); + + vector emEntriesVec; + em->getExtents(oid, emEntriesVec, false, false, true); + CPPUNIT_ASSERT(emEntriesVec.empty()); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + //em->checkConsistency(); + break; + } + + case 3: //lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + DBRootT localDbroot; + PartitionNumberT localPartNum; + SegmentT localSegmentNum; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % (tmp->size - 1); + + target = tmp->LBIDstart + offset; + err = em->lookupLocal(target, oid, localDbroot, localPartNum, localSegmentNum, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; + cerr << "op 3 fbo " << fbo << " offset + tmp->FBO " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + //em->checkConsistency(); + break; + } + + case 4: //lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % (tmp->size - 1); + oid = tmp->OID; + + err = em->lookupLocal(oid, tmp->partNum, tmp->segNum, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << + " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); + //em->checkConsistency(); + break; + } + + case 5: //getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, status; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = em->getLocalHWM(tmp->OID, tmp->partNum, tmp->segNum, status); +#ifdef BRM_VERBOSE_I + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM + << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + //em->checkConsistency(); + break; + } + + case 6: //setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->size - 1); + bool firstNode = true; + em->setLocalHWM(oid, tmp->partNum, tmp->segNum, hwm, firstNode); + + em->confirmChanges(); + + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + //em->checkConsistency(); + break; + } + +/* + case 7: // renew this EM object + { + delete em; + em = new ExtentMap(); +#ifdef BRM_VERBOSE + cerr << "got a new EM instance" << endl; +#endif + em->checkConsistency(); + break; + } + case 8: //getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = em->getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 9: //setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = em->setBulkInsertVars(tmp->LBIDstart + offset, + tmp->secondHWM, tmp->txnID); + em->confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } +*/ + default: + break; + } + } + + delete em; + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; +} + +/* +ExtentMap em_si; +static void* EMRunner_si(void* arg) +{ + + // keep track of LBID ranges allocated here and + // randomly allocate, lookup, delete, get/set HWM, and + // destroy the EM object. + +#ifdef BRM_VERBOSE + int threadNum = reinterpret_cast(arg); +#endif + int op, listSize = 0, i; + uint32_t randstate; + struct EMEntries* head = NULL, *tmp; + struct timeval tv; + vector lbids; + +#ifdef BRM_VERBOSE + cerr << "thread number " << threadNum << " started." << endl; +#endif + + gettimeofday(&tv, NULL); + randstate = static_cast(tv.tv_usec); + + while (!threadStop) + { + op = rand_r(&randstate) % 9; +#ifdef BRM_VERBOSE + cerr << "next op is " << op << endl; +#endif + + switch (op) + { + case 0: //allocate space for a new file + { + struct EMEntries* newEm; + int size = rand_r(&randstate) % 102399 + 1; + int entries, OID, allocdSize; + + pthread_mutex_lock(&pthreadMutex); + OID = oid++; + pthread_mutex_unlock(&pthreadMutex); + + em_si.createExtent(size, OID, lbids, allocdSize); + em_si.confirmChanges(); + + entries = size / em_si.getExtentSize(); + + if ((size % em_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0 ; i < entries; i++) + { + + newEm = new EMEntries(); + newEm->size = em_si.getExtentSize(); + newEm->OID = OID; + newEm->FBO = i * em_si.getExtentSize(); + newEm->LBIDstart = lbids[i]; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created new space for OID " << newEm->OID << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 1: //allocate space for an existing file + { + if (listSize == 0) + break; + + struct EMEntries* newEm, *tmp; + int size = rand_r(&randstate) % 102399 + 1; + int fileRand = rand_r(&randstate) % listSize; + int i, lastExtent, blockEnd, oid; + int tmpHWM, entries, allocdSize; + vector lbids; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) + { + if (tmp->OID != oid) + continue; + + tmpHWM = tmp->HWM; + blockEnd = tmp->FBO + tmp->size; + + if (lastExtent < blockEnd) + lastExtent = blockEnd; + } + + em_si.createExtent(size, oid, lbids, allocdSize); + em_si.confirmChanges(); + + entries = size / em_si.getExtentSize(); + + if ((size % em_si.getExtentSize()) != 0) + entries++; + + CPPUNIT_ASSERT((uint32_t)entries == lbids.size()); + + for (i = 0; i < entries; i++) + { + + newEm = new EMEntries(); + + if (i != entries) + newEm->size = em_si.getExtentSize(); + else + newEm->size = size % em_si.getExtentSize(); + + newEm->OID = oid; + newEm->FBO = lastExtent + (i * em_si.getExtentSize()); + newEm->LBIDstart = lbids[i]; + newEm->HWM = tmpHWM; + + newEm->next = head; + head = newEm; + listSize++; + } + +#ifdef BRM_VERBOSE + cerr << "created another extent for OID " << newEm->OID << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 2: //delete an OID + { + if (listSize == 0) + break; + + struct EMEntries* tmp, *prev; + int fileRand = rand_r(&randstate) % listSize; + int i, oid; + + for (i = 0, tmp = head; i < fileRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + + em_si.deleteOID(oid); + em_si.confirmChanges(); + + for (tmp = head; tmp != NULL;) + { + if (tmp->OID == oid) + { + if (tmp == head) + { + head = head->next; + delete tmp; + tmp = head; + } + else + { + prev->next = tmp->next; + delete tmp; + tmp = prev->next; + } + + listSize--; + } + else + { + prev = tmp; + tmp = tmp->next; + } + } + +#ifdef BRM_VERBOSE + cerr << "deleted OID " << oid << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 3: //lookup by LBID + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset, oid; + struct EMEntries* tmp; + LBID_t target; + uint32_t fbo; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + + target = tmp->LBIDstart + offset; + err = em_si.lookup(target, oid, fbo); +#ifdef BRM_VERBOSE + cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; + cerr << " oid should be " << tmp->OID << " fbo should be " << offset + tmp->FBO << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(oid == tmp->OID); + CPPUNIT_ASSERT(fbo == offset + tmp->FBO); + em_si.checkConsistency(); + break; + } + + case 4: //lookup by OID, FBO + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, oid, err, offset; + struct EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + oid = tmp->OID; + + err = em_si.lookup(oid, offset + tmp->FBO, lbid); +#ifdef BRM_VERBOSE + cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << + " got lbid " << lbid << endl; + cerr << " lbid should be " << tmp->LBIDstart + offset << endl; +#endif + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); + em_si.checkConsistency(); + break; + } + + case 5: //getHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i; + struct EMEntries* tmp; + uint32_t hwm; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + hwm = em_si.getHWM(tmp->OID); +#ifdef BRM_VERBOSE + cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM + << " BRM says it's " << hwm << endl; +#endif + CPPUNIT_ASSERT(hwm == tmp->HWM); + em_si.checkConsistency(); + break; + } + + case 6: //setHWM + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, hwm, oid; + struct EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + oid = tmp->OID; + hwm = rand_r(&randstate) % (tmp->FBO + em_si.getExtentSize()); + em_si.setHWM(oid, hwm); + em_si.confirmChanges(); + + for (tmp = head; tmp != NULL; tmp = tmp->next) + if (tmp->OID == oid) + tmp->HWM = hwm; + +#ifdef BRM_VERBOSE + cerr << "setHWM of OID " << oid << " to " << hwm << endl; +#endif + em_si.checkConsistency(); + break; + } + + case 7: //getBulkInsertVars + { + if (listSize == 0) + break; + + HWM_t hwm; + VER_t txnID; + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + LBID_t lbid; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + lbid = tmp->LBIDstart + offset; + err = em_si.getBulkInsertVars(lbid, hwm, txnID); + CPPUNIT_ASSERT(err == 0); + CPPUNIT_ASSERT(hwm == tmp->secondHWM); + CPPUNIT_ASSERT(txnID == tmp->txnID); + break; + } + + case 8: //setBulkInsertVars + { + if (listSize == 0) + break; + + int entryRand = rand_r(&randstate) % listSize; + int i, err, offset; + EMEntries* tmp; + + for (i = 0, tmp = head; i < entryRand; i++) + tmp = tmp->next; + + offset = rand_r(&randstate) % tmp->size; + tmp->secondHWM = rand_r(&randstate) % MAXINT; + tmp->txnID = rand_r(&randstate) % MAXINT; + err = em_si.setBulkInsertVars(tmp->LBIDstart + offset, + tmp->secondHWM, tmp->txnID); + em_si.confirmChanges(); + CPPUNIT_ASSERT(err == 0); + break; + } + + default: + break; + } + } + + while (head != NULL) + { + tmp = head->next; + delete head; + head = tmp; + } + +#ifdef BRM_VERBOSE + cerr << "thread " << threadNum << " exiting" << endl; +#endif + return NULL; +} +*/ + + +class LongBRMTests : public CppUnit::TestFixture +{ + + CPPUNIT_TEST_SUITE(LongBRMTests); + + CPPUNIT_TEST(longEMTest_1); +// CPPUNIT_TEST(longEMTest_2); +// CPPUNIT_TEST(longBRMTest_1); +// CPPUNIT_TEST(longBRMTest_2); + + CPPUNIT_TEST_SUITE_END(); + +private: +public: + void longEMTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, multiple instance ExtentMap test. " + "This runs for 5 minutes." << endl; + + threadStop = 0; + pthread_mutex_init(&pthreadMutex, nullptr); + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, EMRunner, + reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); + + usleep(1000); + } + + sleep(300); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], nullptr); + } + } + +/* + void longEMTest_2() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, single instance ExtentMap test. " + "This runs for 5 minutes." << endl; + + threadStop = 0; + pthread_mutex_init(&pthreadMutex, NULL); + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, EMRunner_si, + reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the ExtentMap test"); + + usleep(1000); + } + + sleep(60); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + } + void longBRMTest_1() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, multiple instance DBRM test. " + "This runs for 5 minutes." << endl; + + threadStop = 0; + pthread_mutex_init(&pthreadMutex, NULL); + opCount = 0; + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, BRMRunner_1, + reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the DBRM test"); + + usleep(1000); + } + + sleep(300); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + + cerr << "opCount = " << opCount << endl; + } + void longBRMTest_2() + { + const int threadCount = 10; + int i; + pthread_t threads[threadCount]; + + cerr << endl << "Multithreaded, single instance DBRM test. " + "This runs for 5 minutes." << endl; + + threadStop = 0; + pthread_mutex_init(&pthreadMutex, NULL); + opCount = 0; + + for (i = 0; i < threadCount; i++) + { + if (pthread_create(&threads[i], NULL, BRMRunner_si, + reinterpret_cast(i + 1)) < 0) + throw logic_error("Error creating threads for the DBRM test"); + + usleep(1000); + } + + sleep(300); + threadStop = 1; + + for (i = 0; i < threadCount; i++) + { + cerr << "Waiting for thread #" << i << endl; + pthread_join(threads[i], NULL); + } + + cerr << "opCount = " << opCount << endl; + } +*/ + +}; +CPPUNIT_TEST_SUITE_REGISTRATION( LongBRMTests ); + +#include +#include + +int main( int argc, char** argv) +{ + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest( registry.makeTest() ); + idbdatafile::IDBPolicy::configIDBPolicy(); + bool wasSuccessful = runner.run( "", false ); + return (wasSuccessful ? 0 : 1); +} diff --git a/tools/clearShm/main.cpp b/tools/clearShm/main.cpp index 79e38dea5..d31f9a75e 100644 --- a/tools/clearShm/main.cpp +++ b/tools/clearShm/main.cpp @@ -60,7 +60,7 @@ void shmDoit(key_t shm_key, const string& label) bi::offset_t memSize = 0; memObj.get_size(memSize); std::lock_guard lk(coutMutex); - cout << label << ": shm_key: " << shm_key << "; key_name: " << key_name << "; size: " << memSize + cout << label << ": shm|sem_key: " << shm_key << "; key_name: " << key_name << "; size: " << memSize << endl; } catch (...) @@ -74,6 +74,11 @@ void shmDoit(key_t shm_key, const string& label) } } +void semDoit(key_t sem_key, const string& label) +{ + shmDoit(sem_key, label); +} + void shmDoitRange(key_t shm_key, const string& label) { if (shm_key == 0) @@ -87,32 +92,6 @@ void shmDoitRange(key_t shm_key, const string& label) } } -void semDoit(key_t sem_key, const string& label) -{ - string key_name = ShmKeys::keyToName(sem_key); - - if (vFlg) - { - try - { - bi::shared_memory_object memObj(bi::open_only, key_name.c_str(), bi::read_only); - bi::offset_t memSize = 0; - memObj.get_size(memSize); - std::lock_guard lk(coutMutex); - cout << label << ": sem_key: " << sem_key << "; key_name: " << key_name << "; size: " << memSize - << endl; - } - catch (...) - { - } - } - - if (!nFlg) - { - bi::shared_memory_object::remove(key_name.c_str()); - } -} - void usage() { cout << "usage: clearShm [-cvnh]" << endl; @@ -206,6 +185,8 @@ int main(int argc, char** argv) tg.add_thread(tp); tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_VSS_BASE, "VSS ")); tg.add_thread(tp); + tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE, "EXTMAP_INDX")); + tg.add_thread(tp); tg.join_all(); shmDoit(BrmKeys.MST_SYSVKEY, "MST "); @@ -226,6 +207,7 @@ int main(int argc, char** argv) semDoit(BrmKeys.KEYRANGE_EMFREELIST_BASE, "EXTMAP_FREE"); semDoit(BrmKeys.KEYRANGE_VBBM_BASE, "VBBM "); semDoit(BrmKeys.KEYRANGE_VSS_BASE, "VSS "); + semDoit(BrmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE, "EXTMAP_INDX"); semDoit(BrmKeys.MST_SYSVKEY, "MST "); if (!cFlg) diff --git a/utils/rwlock/rwlock.h b/utils/rwlock/rwlock.h index cd961ad81..a0beaf572 100644 --- a/utils/rwlock/rwlock.h +++ b/utils/rwlock/rwlock.h @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -26,6 +27,8 @@ #pragma once +#include + #include #include @@ -42,6 +45,10 @@ namespace rwlock { +const std::array RWLockNames = { + "all", "VSS", "ExtentMap", "FreeList", "VBBM", "CopyLocks", "ExtentMapIndex", +}; + /// the layout of the shmseg struct State { diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index c5504428c..44750c630 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -40,7 +40,15 @@ namespace bi = boost::interprocess; namespace BRM { -BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : fKey(key), fSize(size), fReadOnly(readOnly) +const constexpr uint32_t ShmCreateMaxRetries = 10; +const constexpr unsigned int NapTimer = 500000; + +BRMShmImplParent::BRMShmImplParent(unsigned key, off_t size, bool readOnly) + : fKey(key), fSize(size), fReadOnly(readOnly){}; + +BRMShmImplParent::~BRMShmImplParent(){}; + +BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : BRMShmImplParent(key, size, readOnly) { string keyName = ShmKeys::keyToName(fKey); @@ -238,5 +246,167 @@ void BRMShmImpl::destroy() bi::shared_memory_object::remove(oldName.c_str()); } +BRMManagedShmImpl::BRMManagedShmImpl(unsigned key, off_t size, bool readOnly) + : BRMShmImplParent(key, size, readOnly) +{ + string keyName = ShmKeys::keyToName(fKey); + off_t curSize = 0; + + for (uint32_t tries = 0; fSize == 0 && tries <= ShmCreateMaxRetries; ++tries) + { + try + { + auto* shmSegment = new boost::interprocess::managed_shared_memory(bi::open_only, keyName.c_str()); + curSize = shmSegment->get_size(); + + if (curSize == 0) + { + delete shmSegment; + throw bi::interprocess_exception("shared memory segment size is 0."); + } + else + { + fShmSegment = shmSegment; + fSize = curSize; + return; + } + } + catch (bi::interprocess_exception&) + { + if (tries == ShmCreateMaxRetries) + { + log("BRMManagedShmImpl::BRMManagedShmImpl(): re-creating shared memory segment\ + b/c of its size == 0. Re-throw."); + throw; + } + + cerr << "BRMManagedShmImpl::BRMManagedShmImpl(): re-creating shared memory segment\ + b/c of its size == 0" + << endl; + usleep(NapTimer); + } + } + + try + { + bi::permissions perms; + perms.set_unrestricted(); + fShmSegment = new bi::managed_shared_memory(bi::create_only, keyName.c_str(), fSize, + 0, // use a default address to map the segment + perms); + // fSize == 0 on any process startup but managed_shared_memory ctor throws + // so control flow doesn't get here. + idbassert(fSize > 0); + } + catch (bi::interprocess_exception& b) + { + if (b.get_error_code() != bi::already_exists_error) + { + ostringstream o; + o << "BRM caught an exception creating a shared memory segment: " << b.what(); + log(o.str()); + throw; + } + bi::managed_shared_memory* shmSegment = nullptr; + try + { + if (fReadOnly) + shmSegment = new bi::managed_shared_memory(bi::open_read_only, keyName.c_str()); + else + shmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str()); + } + catch (exception& e) + { + ostringstream o; + o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << b.what(); + log(o.str()); + throw; + } + off_t curSize = shmSegment->get_size(); + + idbassert(curSize > 0); + idbassert(curSize >= fSize); + fShmSegment = shmSegment; + fSize = curSize; + } +} + +int BRMManagedShmImpl::grow(off_t newSize) +{ + auto keyName = ShmKeys::keyToName(fKey); + + if (newSize > fSize) + { + const auto incSize = newSize - fSize; + if (fShmSegment) + { + // Call destructor to unmap the segment. + delete fShmSegment; + // Grow the segment. + bi::managed_shared_memory::grow(keyName.c_str(), incSize); + // Open only with the assumption ::grow() can be called on read-write shmem. + fShmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str()); + // Update size. + fSize = newSize; + } + } + + return 0; +} + +// Dummy method that has no references in the code. +int BRMManagedShmImpl::clear(unsigned newKey, off_t newSize) +{ + return 0; +} + +// This method calls for all related shmem pointers to be refreshed. +void BRMManagedShmImpl::setReadOnly() +{ + if (fReadOnly) + return; + const bool readOnly = true; + remap(readOnly); + fReadOnly = true; +} + +void BRMManagedShmImpl::swap(BRMManagedShmImpl& rhs) +{ + fShmSegment->swap(*rhs.fShmSegment); + std::swap(fKey, rhs.fKey); + std::swap(fSize, rhs.fSize); + std::swap(fReadOnly, rhs.fReadOnly); +} + +// The method was copied from non-managed shmem impl class +// and it has no refences in MCS 6.x code. +void BRMManagedShmImpl::destroy() +{ + string keyName = ShmKeys::keyToName(fKey); + try + { + bi::shared_memory_object::remove(keyName.c_str()); + } + catch (bi::interprocess_exception& b) + { + std::ostringstream o; + o << "BRMManagedShmImpl::destroy caught an exception removing a managed shared memory segment: " + << b.what(); + log(o.str()); + throw; + } +} + +void BRMManagedShmImpl::remap(const bool readOnly) +{ + delete fShmSegment; + fShmSegment = nullptr; + string keyName = ShmKeys::keyToName(fKey); + if (readOnly) + fShmSegment = new bi::managed_shared_memory(bi::open_read_only, keyName.c_str()); + else + fShmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str()); +} + } // namespace BRM diff --git a/versioning/BRM/brmshmimpl.h b/versioning/BRM/brmshmimpl.h index a34d40c5f..fa1f0cfee 100644 --- a/versioning/BRM/brmshmimpl.h +++ b/versioning/BRM/brmshmimpl.h @@ -30,17 +30,18 @@ //#define NDEBUG #include #include +#include #include +namespace bi = boost::interprocess; + namespace BRM { -class BRMShmImpl +class BRMShmImplParent { public: - BRMShmImpl(unsigned key, off_t size, bool readOnly = false); - ~BRMShmImpl() - { - } + BRMShmImplParent(unsigned key, off_t size, bool readOnly = false); + virtual ~BRMShmImplParent(); inline unsigned key() const { @@ -55,23 +56,63 @@ class BRMShmImpl return fReadOnly; } - void setReadOnly(); - int grow(unsigned newKey, off_t newSize); - int clear(unsigned newKey, off_t newSize); - - void swap(BRMShmImpl& rhs); - void destroy(); - - boost::interprocess::shared_memory_object fShmobj; - boost::interprocess::mapped_region fMapreg; - - private: - BRMShmImpl(const BRMShmImpl& rhs); - BRMShmImpl& operator=(const BRMShmImpl& rhs); + virtual void setReadOnly() = 0; + virtual int clear(unsigned newKey, off_t newSize) = 0; + virtual void destroy() = 0; + protected: unsigned fKey; off_t fSize; bool fReadOnly; }; +class BRMShmImpl : public BRMShmImplParent +{ + public: + BRMShmImpl(unsigned key, off_t size, bool readOnly = false); + BRMShmImpl(const BRMShmImpl& rhs) = delete; + BRMShmImpl& operator=(const BRMShmImpl& rhs) = delete; + ~BRMShmImpl() + { + } + + int clear(unsigned newKey, off_t newSize) override; + void destroy() override; + void setReadOnly() override; + + int grow(unsigned newKey, off_t newSize); + void swap(BRMShmImpl& rhs); + + bi::shared_memory_object fShmobj; + bi::mapped_region fMapreg; +}; + +class BRMManagedShmImpl : public BRMShmImplParent +{ + public: + BRMManagedShmImpl(unsigned key, off_t size, bool readOnly = false); + BRMManagedShmImpl(const BRMManagedShmImpl& rhs) = delete; + BRMManagedShmImpl& operator=(const BRMManagedShmImpl& rhs) = delete; + ~BRMManagedShmImpl() + { + delete fShmSegment; + } + + int clear(unsigned newKey, off_t newSize) override; + void destroy() override; + void setReadOnly() override; + + int grow(off_t newSize); + void remap(const bool readOnly = false); + void swap(BRMManagedShmImpl& rhs); + bi::managed_shared_memory* getManagedSegment() + { + assert(fShmSegment); + return fShmSegment; + } + + private: + bi::managed_shared_memory* fShmSegment; +}; + } // namespace BRM diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index 1cc3fb983..2ab47d8ca 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -98,7 +98,7 @@ DBRM::DBRM(const DBRM& brm) throw logic_error("DBRM: Don't use the copy constructor."); } -DBRM::~DBRM() throw() +DBRM::~DBRM() { if (msgClient != NULL) MessageQueueClientPool::releaseInstance(msgClient); @@ -461,7 +461,7 @@ int DBRM::markExtentsInvalid(const vector& lbids, } template -int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw() +int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) { #ifdef BRM_INFO @@ -489,7 +489,7 @@ int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) th } } -int DBRM::getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw() +int DBRM::getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) { try { @@ -4555,10 +4555,19 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN setExtentsMaxMin(cpInfos); } -template int DBRM::getExtentMaxMin(const LBID_t lbid, int128_t& max, int128_t& min, - int32_t& seqNum) throw(); +size_t DBRM::EMIndexShmemSize() +{ + return em->EMIndexShmemSize(); +} -template int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, - int32_t& seqNum) throw(); +size_t DBRM::EMIndexShmemFree() +{ + return em->EMIndexShmemFree(); +} + +template int DBRM::getExtentMaxMin(const LBID_t lbid, int128_t& max, int128_t& min, + int32_t& seqNum); + +template int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum); } // namespace BRM diff --git a/versioning/BRM/dbrm.h b/versioning/BRM/dbrm.h index 38f350781..f9eaf54a0 100644 --- a/versioning/BRM/dbrm.h +++ b/versioning/BRM/dbrm.h @@ -103,7 +103,7 @@ class DBRM // The param noBRMFcns suppresses init of the ExtentMap, VSS, VBBM, and CopyLocks. // It can speed up init if the caller only needs the other structures. EXPORT DBRM(bool noBRMFcns = false); - EXPORT ~DBRM() throw(); + EXPORT ~DBRM(); EXPORT static void refreshShm() { @@ -780,12 +780,12 @@ class DBRM const std::vector& colDataTypes) DBRM_THROW; template - EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw(); + EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum); EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, const int32_t seqNum) DBRM_THROW; - EXPORT int getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw(); + EXPORT int getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin); /** @brief Updates the max and min casual partitioning info for the passed extents. * @@ -984,6 +984,9 @@ class DBRM EXPORT void invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN txnid, bool allExtents, std::vector* plbidList = NULL); + size_t EMIndexShmemSize(); + size_t EMIndexShmemFree(); + private: DBRM(const DBRM& brm); DBRM& operator=(const DBRM& brm); diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 0180242af..0f8d32a88 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -1,19 +1,21 @@ + /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ /***************************************************************************** * $Id: extentmap.cpp 1936 2013-07-09 22:10:29Z dhall $ @@ -45,7 +47,6 @@ #include #include -namespace bi = boost::interprocess; #include "liboamcpp.h" #include "brmtypes.h" @@ -113,6 +114,7 @@ inline void incSeqNum(int32_t& seqNum) namespace BRM { +static const char* EmIndexObjectName = "i"; //------------------------------------------------------------------------------ // EMCasualPartition_struct methods //------------------------------------------------------------------------------ @@ -217,9 +219,10 @@ bool EMEntry::operator<(const EMEntry& e) const /*static*/ boost::mutex ExtentMapImpl::fInstanceMutex; boost::mutex ExtentMap::mutex; +boost::mutex ExtentMap::emIndexMutex; /*static*/ -ExtentMapImpl* ExtentMapImpl::fInstance = 0; +ExtentMapImpl* ExtentMapImpl::fInstance = nullptr; /*static*/ ExtentMapImpl* ExtentMapImpl::makeExtentMapImpl(unsigned key, off_t size, bool readOnly) @@ -279,19 +282,305 @@ FreeListImpl::FreeListImpl(unsigned key, off_t size, bool readOnly) : fFreeList( { } +/*static*/ +std::mutex ExtentMapIndexImpl::fInstanceMutex_; + +/*static*/ +ExtentMapIndexImpl* ExtentMapIndexImpl::fInstance_ = nullptr; + +/*static*/ +ExtentMapIndexImpl* ExtentMapIndexImpl::makeExtentMapIndexImpl(unsigned key, off_t size, bool readOnly) +{ + std::lock_guard lock(fInstanceMutex_); + + if (fInstance_) + { + if (size != fInstance_->getShmemSize()) + { + fInstance_->fBRMManagedShmMemImpl_.remap(); + } + + return fInstance_; + } + + fInstance_ = new ExtentMapIndexImpl(key, size, readOnly); + fInstance_->createExtentMapIndexIfNeeded(); + + return fInstance_; +} + +ExtentMapIndexImpl::ExtentMapIndexImpl(unsigned key, off_t size, bool readOnly) + : fBRMManagedShmMemImpl_(key, size, readOnly) +{ +} + +void ExtentMapIndexImpl::createExtentMapIndexIfNeeded() +{ + // pair + auto managedShmemSearchPair = + fBRMManagedShmMemImpl_.getManagedSegment()->find(EmIndexObjectName); + if (!managedShmemSearchPair.first || managedShmemSearchPair.second == 0) + { + ShmVoidAllocator alloc(fBRMManagedShmMemImpl_.getManagedSegment()->get_segment_manager()); + fBRMManagedShmMemImpl_.getManagedSegment()->construct(EmIndexObjectName)(alloc); + } +} + +ExtentMapIndex* ExtentMapIndexImpl::get() +{ + // pair + auto managedShmemSearchPair = + fBRMManagedShmMemImpl_.getManagedSegment()->find(EmIndexObjectName); + assert(managedShmemSearchPair.first && managedShmemSearchPair.second > 0); + return managedShmemSearchPair.first; +} + +bool ExtentMapIndexImpl::growIfNeeded(const size_t memoryNeeded) +{ + auto freeShmem = getShmemFree(); + // Worst case managed segment can't get continues buffer with len = memoryNeeded + if (freeShmem < memoryNeeded) + { + const size_t currentShmemSize = getShmemSize(); + constexpr static const size_t minAllowance = 16 * 1024 * 1024; + const size_t newShmemSize = std::max(minAllowance, memoryNeeded) + currentShmemSize; + grow(newShmemSize); + return true; + } + return false; +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert(const EMEntry& emEntry, const size_t emIdx) +{ + auto dbRoot = emEntry.dbRoot; + auto* extentMapIndexPtr = get(); + bool shmemHasGrown = false; + + while (dbRoot >= extentMapIndexPtr->size()) + { + const size_t memNeeded = (extentMapIndexPtr->capacity() + extraUnits_) * dbRootContainerUnitSize_; + shmemHasGrown = growIfNeeded(memNeeded); + // Need to refresh all refs and iterators b/c the local address range changed. + extentMapIndexPtr = get(); + assert(extentMapIndexPtr); + ShmVoidAllocator alloc(fBRMManagedShmMemImpl_.getManagedSegment()->get_segment_manager()); + OIDIndexContainerT oidIndices(alloc); + extentMapIndexPtr->push_back(std::move(oidIndices)); + } + auto& extentMapIndex = *extentMapIndexPtr; + return insert2ndLayerWrapper(extentMapIndex[dbRoot], emEntry, emIdx, shmemHasGrown); +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert2ndLayer(OIDIndexContainerT& oids, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown) +{ + OID_t oid = emEntry.fileID; + ShmVoidAllocator alloc(fBRMManagedShmMemImpl_.getManagedSegment()->get_segment_manager()); + + PartitionIndexContainerT partitionIndex(alloc); + auto iterAndResult = oids.insert({oid, std::move(partitionIndex)}); + + if (iterAndResult.second) + { + PartitionIndexContainerT& partitionsContainer = (*iterAndResult.first).second; + return insert3dLayerWrapper(partitionsContainer, emEntry, emIdx, aShmemHasGrown); + } + else + return {false, aShmemHasGrown}; +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert2ndLayerWrapper(OIDIndexContainerT& oids, + const EMEntry& emEntry, const size_t emIdx, + const bool aShmemHasGrown) +{ + OID_t oid = emEntry.fileID; + auto oidsIter = oids.find(oid); + bool shmemHasGrown = aShmemHasGrown; + if (oidsIter == oids.end()) + { + const size_t freeShmem = fBRMManagedShmMemImpl_.getManagedSegment()->get_free_memory(); + const size_t memNeeded = (oids.size() + extraUnits_) * oidContainerUnitSize_; + if (oids.load_factor() >= oids.max_load_factor() || freeShmem <= freeSpaceThreshold_) + { + // Need to refresh all refs and iterators b/c the local address range changed. + shmemHasGrown = growIfNeeded(memNeeded); + auto* extMapIndexPtr = get(); + assert(extMapIndexPtr); + auto& extMapIndex = *extMapIndexPtr; + shmemHasGrown = shmemHasGrown || aShmemHasGrown; + // The dbroot must be here b/c it was already found once in the upper insert(). + OIDIndexContainerT& refreshedOidsRef = extMapIndex[emEntry.dbRoot]; + return insert2ndLayer(refreshedOidsRef, emEntry, emIdx, shmemHasGrown); + } + return insert2ndLayer(oids, emEntry, emIdx, shmemHasGrown); + } + PartitionIndexContainerT& partitions = (*oidsIter).second; + return insert3dLayerWrapper(partitions, emEntry, emIdx, shmemHasGrown); +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert3dLayer(PartitionIndexContainerT& partitions, + const EMEntry& emEntry, const size_t emIdx, + const bool aShmemHasGrown) +{ + auto partitionNumber = emEntry.partitionNum; + ShmVoidAllocator alloc(fBRMManagedShmMemImpl_.getManagedSegment()->get_segment_manager()); + ExtentMapIndicesT emIndices(alloc); + emIndices.push_back(emIdx); + auto iterAndResult = partitions.insert({partitionNumber, std::move(emIndices)}); + return {iterAndResult.second, aShmemHasGrown}; +} + +InsertUpdateShmemKeyPair ExtentMapIndexImpl::insert3dLayerWrapper(PartitionIndexContainerT& partitions, + const EMEntry& emEntry, const size_t emIdx, + const bool aShmemHasGrown) +{ + auto partitionNumber = emEntry.partitionNum; + auto partitionsIter = partitions.find(partitionNumber); + bool shmemHasGrown = aShmemHasGrown; + if (partitionsIter == partitions.end()) + { + const size_t freeShmem = fBRMManagedShmMemImpl_.getManagedSegment()->get_free_memory(); + const size_t memNeeded = + (partitions.size() + extraUnits_) * partitionContainerUnitSize_ + emIdentUnitSize_; + if (partitions.load_factor() >= partitions.max_load_factor() || freeShmem <= freeSpaceThreshold_) + { + // Need to refresh all refs and iterators b/c the local address range changed. + shmemHasGrown = growIfNeeded(memNeeded); + auto* extMapIndexPtr = get(); + assert(extMapIndexPtr); + auto& extMapIndex = *extMapIndexPtr; + shmemHasGrown = shmemHasGrown || aShmemHasGrown; + // The dbroot must be here b/c we found it once in insert(). + OIDIndexContainerT& refreshedOidsRef = extMapIndex[emEntry.dbRoot]; + auto oidsIter = refreshedOidsRef.find(emEntry.fileID); + PartitionIndexContainerT& refreshedPartitionsRef = (*oidsIter).second; + return insert3dLayer(refreshedPartitionsRef, emEntry, emIdx, shmemHasGrown); + } + return insert3dLayer(partitions, emEntry, emIdx, shmemHasGrown); + } + + ExtentMapIndicesT& emIndices = (*partitionsIter).second; + emIndices.push_back(emIdx); + return {true, shmemHasGrown}; +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::find(const DBRootT dbroot, const OID_t oid, + const PartitionNumberT partitionNumber) +{ + ExtentMapIndex& emIndex = *get(); + if (dbroot >= emIndex.size()) + return {}; + return search2ndLayer(emIndex[dbroot], oid, partitionNumber); +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::find(const DBRootT dbroot, const OID_t oid) +{ + ExtentMapIndex& emIndex = *get(); + if (dbroot >= emIndex.size()) + return {}; + return search2ndLayer(emIndex[dbroot], oid); +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::search2ndLayer(OIDIndexContainerT& oids, const OID_t oid, + const PartitionNumberT partitionNumber) +{ + auto oidsIter = oids.find(oid); + if (oidsIter == oids.end()) + return {}; + + PartitionIndexContainerT& partitions = (*oidsIter).second; + return search3dLayer(partitions, partitionNumber); +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::search2ndLayer(OIDIndexContainerT& oids, const OID_t oid) +{ + auto oidsIter = oids.find(oid); + if (oidsIter == oids.end()) + return {}; + + ExtentMapIndexFindResult result; + PartitionIndexContainerT& partitions = (*oidsIter).second; + for (auto& partKeyValue : partitions) + { + ExtentMapIndicesT& emIdentifiers = partKeyValue.second; + for (auto& emIdent : emIdentifiers) + result.push_back(emIdent); + } + + return result; +} + +ExtentMapIndexFindResult ExtentMapIndexImpl::search3dLayer(PartitionIndexContainerT& partitions, + const PartitionNumberT partitionNumber) +{ + auto partitionsIter = partitions.find(partitionNumber); + if (partitionsIter == partitions.end()) + return {}; + + ExtentMapIndexFindResult result; + ExtentMapIndicesT& emIndicesVec = (*partitionsIter).second; + for (auto& emIndex : emIndicesVec) + result.push_back(emIndex); + return result; +} + +void ExtentMapIndexImpl::deleteDbRoot(const DBRootT dbroot) +{ + auto& extMapIndex = *get(); + extMapIndex[dbroot].clear(); +} + +void ExtentMapIndexImpl::deleteOID(const DBRootT dbroot, const OID_t oid) +{ + auto& extMapIndex = *get(); + auto oidsIter = extMapIndex[dbroot].find(oid); + // Nothing to delete. Might be a sign of a problem. + if (oidsIter == extMapIndex[dbroot].end()) + return; + extMapIndex[dbroot].erase(oidsIter); +} + +void ExtentMapIndexImpl::deleteEMEntry(const EMEntry& emEntry, const ExtentMapIdxT emIdent) +{ + // find partition + auto& extMapIndex = *get(); + auto oidsIter = extMapIndex[emEntry.dbRoot].find(emEntry.fileID); + if (oidsIter == extMapIndex[emEntry.dbRoot].end()) + return; + PartitionIndexContainerT& partitions = (*oidsIter).second; + auto partitionsIter = partitions.find(emEntry.partitionNum); + if (partitionsIter == partitions.end()) + return; + ExtentMapIndicesT& emIdentifiers = (*partitionsIter).second; + // pop the identifier + if (emIdentifiers.size() > 1) + { + auto emIdentifiersTargetIter = std::find(emIdentifiers.begin(), emIdentifiers.end(), emIdent); + std::swap(*emIdentifiersTargetIter, emIdentifiers.back()); + emIdentifiers.pop_back(); + } + else // only 1 ident in this partition + { + partitions.erase(partitionsIter); + } +} + ExtentMap::ExtentMap() { - fExtentMap = NULL; - fFreeList = NULL; + fExtentMap = nullptr; + fFreeList = nullptr; + fPExtMapImpl = nullptr; fCurrentEMShmkey = -1; fCurrentFLShmkey = -1; - fEMShminfo = NULL; - fFLShminfo = NULL; + fEMShminfo = nullptr; + fFLShminfo = nullptr; + fEMIndexShminfo = nullptr; r_only = false; flLocked = false; emLocked = false; - fPExtMapImpl = 0; - fPFreeListImpl = 0; + emIndexLocked = false; + fPFreeListImpl = nullptr; + fPExtMapIndexImpl_ = nullptr; #ifdef BRM_INFO fDebug = ("Y" == config::Config::makeConfig()->getConfig("DBRM", "Debug")); @@ -416,6 +705,7 @@ int ExtentMap::markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatal #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); return _markInvalid(lbid, colDataType); } @@ -448,6 +738,7 @@ int ExtentMap::markInvalid(const vector& lbids, #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); // XXXPAT: what's the proper return code when one and only one fails? for (i = 0; i < size; ++i) @@ -514,6 +805,7 @@ int ExtentMap::setMaxMin(const LBID_t lbid, const int64_t max, const int64_t min #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < entries; i++) @@ -571,8 +863,10 @@ int ExtentMap::setMaxMin(const LBID_t lbid, const int64_t max, const int64_t min if (emLocked) releaseEMEntryTable(WRITE); + if (emIndexLocked) + releaseEMIndex(WRITE); + throw logic_error("ExtentMap::setMaxMin(): lbid isn't allocated"); - // return -1; } // @bug 1970. Added updateExtentsMaxMin function. @@ -625,7 +919,10 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo #endif if (useLock) + { grabEMEntryTable(WRITE); + grabEMIndex(WRITE); + } entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -759,7 +1056,7 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo // @bug 1970. Added mergeExtentsMaxMin to merge CP info for list of extents. // @note - The key passed in the map must the starting LBID in the extent. // Used by cpimport to update extentmap casual partition min/max. -// NULL or empty values should not be passed in as min/max values. +// nullptr or empty values should not be passed in as min/max values. // seqNum in the input struct is not currently used. // // Note that DML calls markInvalid() to flag an extent as CP_UPDATING and incre- @@ -823,7 +1120,10 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) #endif if (useLock) + { grabEMEntryTable(WRITE); + grabEMIndex(WRITE); + } int entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -1127,6 +1427,7 @@ int ExtentMap::getMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) #endif grabEMEntryTable(READ); + grabEMIndex(READ); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < entries; i++) @@ -1149,12 +1450,14 @@ int ExtentMap::getMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) } seqNum = fExtentMap[i].partition.cprange.sequenceNum; isValid = fExtentMap[i].partition.cprange.isValid; + releaseEMIndex(READ); releaseEMEntryTable(READ); return isValid; } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); throw logic_error("ExtentMap::getMaxMin(): that lbid isn't allocated"); // return -1; @@ -1323,7 +1626,6 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) void* fExtentMapPtr = static_cast(fExtentMap); memset(fExtentMapPtr, 0, fEMShminfo->allocdSize); fEMShminfo->currentSize = 0; - // init the free list memset(fFreeList, 0, fFLShminfo->allocdSize); fFreeList[0].size = (1 << 26); // 2^36 LBIDs @@ -1344,15 +1646,15 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) } growEMShmseg(nrows); + growEMIndexShmseg(ExtentMapIndexImpl::estimateEMIndexSize(emNumElements)); } + size_t progress = 0, writeSize = emNumElements * sizeof(EMEntry); int err; char* writePos; - size_t progress, writeSize; if (!upgradeV4ToV5) { - progress = 0; writeSize = emNumElements * sizeof(EMEntry); writePos = (char*)fExtentMap; @@ -1414,6 +1716,14 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) //@bug 1911 - verify status value is valid if (fExtentMap[i].status < EXTENTSTATUSMIN || fExtentMap[i].status > EXTENTSTATUSMAX) fExtentMap[i].status = EXTENTAVAILABLE; + + auto resShmemHasGrownPair = fPExtMapIndexImpl_->insert(fExtentMap[i], i); + + if (resShmemHasGrownPair.second) + fEMIndexShminfo->allocdSize = fPExtMapIndexImpl_->getShmemSize(); + + if (!resShmemHasGrownPair.first) + logAndSetEMIndexReadOnly("loadVersion4or5"); } fEMShminfo->currentSize = emNumElements * sizeof(EMEntry); @@ -1454,6 +1764,7 @@ void ExtentMap::load(const string& filename, bool fixFL) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); try { @@ -1461,6 +1772,7 @@ void ExtentMap::load(const string& filename, bool fixFL) } catch (...) { + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); throw; } @@ -1473,6 +1785,7 @@ void ExtentMap::load(const string& filename, bool fixFL) { log_errno("ExtentMap::load(): open"); releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); throw ios_base::failure("ExtentMap::load(): open failed. Check the error log."); } @@ -1485,13 +1798,14 @@ void ExtentMap::load(const string& filename, bool fixFL) catch (...) { releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); throw; } releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); - // checkConsistency(); } // This is a quick workaround, to be able to initialize initial system tables @@ -1535,11 +1849,13 @@ void ExtentMap::loadFromBinaryBlob(const char* blob) catch (...) { releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); throw; } releaseFreeList(WRITE); + releaseEMIndex(WRITE); releaseEMEntryTable(WRITE); } @@ -1586,6 +1902,7 @@ void ExtentMap::save(const string& filename) int allocdSize, loadSize[3], i; grabEMEntryTable(READ); + grabEMIndex(READ); try { @@ -1593,6 +1910,7 @@ void ExtentMap::save(const string& filename) } catch (...) { + releaseEMIndex(READ); releaseEMEntryTable(READ); throw; } @@ -1601,6 +1919,7 @@ void ExtentMap::save(const string& filename) { log("ExtentMap::save(): got request to save an empty BRM"); releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw runtime_error("ExtentMap::save(): got request to save an empty BRM"); } @@ -1613,6 +1932,7 @@ void ExtentMap::save(const string& filename) { log_errno("ExtentMap::save(): open"); releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); } @@ -1634,6 +1954,7 @@ void ExtentMap::save(const string& filename) catch (...) { releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw; } @@ -1659,6 +1980,7 @@ void ExtentMap::save(const string& filename) if (err < 0) { releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } @@ -1678,6 +2000,7 @@ void ExtentMap::save(const string& filename) if (err < 0) { releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } @@ -1685,9 +2008,6 @@ void ExtentMap::save(const string& filename) } } - // allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - // const int inlineLbidRangeSize = sizeof(InlineLBIDRange); - progress = 0; writeSize = fFLShminfo->allocdSize; char* writePos = (char*)fFreeList; @@ -1697,6 +2017,7 @@ void ExtentMap::save(const string& filename) if (err < 0) { releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } @@ -1705,6 +2026,7 @@ void ExtentMap::save(const string& filename) } releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -1714,7 +2036,9 @@ void ExtentMap::grabEMEntryTable(OPS op) boost::mutex::scoped_lock lk(mutex); if (op == READ) + { fEMShminfo = fMST.getTable_read(MasterSegmentTable::EMTable); + } else { fEMShminfo = fMST.getTable_write(MasterSegmentTable::EMTable); @@ -1723,9 +2047,9 @@ void ExtentMap::grabEMEntryTable(OPS op) if (!fPExtMapImpl || fPExtMapImpl->key() != (unsigned)fEMShminfo->tableShmkey) { - if (fExtentMap != NULL) + if (fExtentMap != nullptr) { - fExtentMap = NULL; + fExtentMap = nullptr; } if (fEMShminfo->allocdSize == 0) @@ -1736,17 +2060,22 @@ void ExtentMap::grabEMEntryTable(OPS op) emLocked = true; if (fEMShminfo->allocdSize == 0) + { growEMShmseg(); + } emLocked = false; // has to be done holding the write lock fMST.getTable_downgrade(MasterSegmentTable::EMTable); } else + { growEMShmseg(); + } } else { fPExtMapImpl = ExtentMapImpl::makeExtentMapImpl(fEMShminfo->tableShmkey, 0); + ASSERT(fPExtMapImpl); if (r_only) @@ -1754,7 +2083,7 @@ void ExtentMap::grabEMEntryTable(OPS op) fExtentMap = fPExtMapImpl->get(); - if (fExtentMap == NULL) + if (fExtentMap == nullptr) { log_errno("ExtentMap::grabEMEntryTable(): shmat"); throw runtime_error("ExtentMap::grabEMEntryTable(): shmat failed. Check the error log."); @@ -1762,7 +2091,9 @@ void ExtentMap::grabEMEntryTable(OPS op) } } else + { fExtentMap = fPExtMapImpl->get(); + } } /* always returns holding the FL lock */ @@ -1783,9 +2114,9 @@ void ExtentMap::grabFreeList(OPS op) if (!fPFreeListImpl || fPFreeListImpl->key() != (unsigned)fFLShminfo->tableShmkey) { - if (fFreeList != NULL) + if (fFreeList != nullptr) { - fFreeList = NULL; + fFreeList = nullptr; } if (fFLShminfo->allocdSize == 0) @@ -1815,7 +2146,7 @@ void ExtentMap::grabFreeList(OPS op) fFreeList = fPFreeListImpl->get(); - if (fFreeList == NULL) + if (fFreeList == nullptr) { log_errno("ExtentMap::grabFreeList(): shmat"); throw runtime_error("ExtentMap::grabFreeList(): shmat failed. Check the error log."); @@ -1834,10 +2165,66 @@ void ExtentMap::grabFreeList(OPS op) } } +void ExtentMap::grabEMIndex(OPS op) +{ + boost::mutex::scoped_lock lk(emIndexMutex); + + if (op == READ) + { + fEMIndexShminfo = fMST.getTable_read(MasterSegmentTable::EMIndex); + } + else + { + fEMIndexShminfo = fMST.getTable_write(MasterSegmentTable::EMIndex); + emIndexLocked = true; + } + + if (!fPExtMapIndexImpl_) + { + if (fEMIndexShminfo->allocdSize == 0) + { + if (op == READ) + { + fMST.getTable_upgrade(MasterSegmentTable::EMIndex); + emIndexLocked = true; + + // Checking race conditions + if (fEMIndexShminfo->allocdSize == 0) + growEMIndexShmseg(); + + emIndexLocked = false; + fMST.getTable_downgrade(MasterSegmentTable::EMIndex); + } + else + { + growEMIndexShmseg(); + } + } + else + { + // Sending down current Managed Shmem size. If EMIndexImpl instance size doesn't match + // fEMIndexShminfo->allocdSize makeExtentMapIndexImpl will remap managed shmem segment. + fPExtMapIndexImpl_ = + ExtentMapIndexImpl::makeExtentMapIndexImpl(getInitialEMIndexShmkey(), fEMIndexShminfo->allocdSize); + + if (r_only) + fPExtMapIndexImpl_->makeReadOnly(); + } + } + else if (fPExtMapIndexImpl_->getShmemImplSize() != (unsigned)fEMIndexShminfo->allocdSize) + { + fPExtMapIndexImpl_->refreshShm(); + fPExtMapIndexImpl_ = + ExtentMapIndexImpl::makeExtentMapIndexImpl(getInitialEMIndexShmkey(), fEMIndexShminfo->allocdSize); + } +} + void ExtentMap::releaseEMEntryTable(OPS op) { if (op == READ) + { fMST.releaseTable_read(MasterSegmentTable::EMTable); + } else { /* @@ -1863,32 +2250,50 @@ void ExtentMap::releaseFreeList(OPS op) } } +void ExtentMap::releaseEMIndex(OPS op) +{ + if (op == READ) + { + fMST.releaseTable_read(MasterSegmentTable::EMIndex); + } + else + { + emIndexLocked = false; + fMST.releaseTable_write(MasterSegmentTable::EMIndex); + } +} + key_t ExtentMap::chooseEMShmkey() { - int fixedKeys = 1; - key_t ret; - - if (fEMShminfo->tableShmkey + 1 == (key_t)(fShmKeys.KEYRANGE_EXTENTMAP_BASE + fShmKeys.KEYRANGE_SIZE - 1) || - (unsigned)fEMShminfo->tableShmkey < fShmKeys.KEYRANGE_EXTENTMAP_BASE) - ret = fShmKeys.KEYRANGE_EXTENTMAP_BASE + fixedKeys; - else - ret = fEMShminfo->tableShmkey + 1; - - return ret; + return chooseShmkey(fEMShminfo, fShmKeys.KEYRANGE_EXTENTMAP_BASE); } key_t ExtentMap::chooseFLShmkey() { - int fixedKeys = 1, ret; + return chooseShmkey(fFLShminfo, fShmKeys.KEYRANGE_EMFREELIST_BASE); +} - if (fFLShminfo->tableShmkey + 1 == - (key_t)(fShmKeys.KEYRANGE_EMFREELIST_BASE + fShmKeys.KEYRANGE_SIZE - 1) || - (unsigned)fFLShminfo->tableShmkey < fShmKeys.KEYRANGE_EMFREELIST_BASE) - ret = fShmKeys.KEYRANGE_EMFREELIST_BASE + fixedKeys; - else - ret = fFLShminfo->tableShmkey + 1; +// The key values is fixed b/c MCS doesn't need to increase a segment id number +key_t ExtentMap::chooseEMIndexShmkey() +{ + return chooseShmkey(fEMIndexShminfo, fShmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE); +} - return ret; +key_t ExtentMap::getInitialEMIndexShmkey() const +{ + return fShmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE + 1; +} + +key_t ExtentMap::chooseShmkey(const MSTEntry* masterTableEntry, const uint32_t keyRangeBase) const +{ + int fixedKeys = 1; + + if (masterTableEntry->tableShmkey + 1 == (key_t)(keyRangeBase + fShmKeys.KEYRANGE_SIZE - 1) || + (unsigned)masterTableEntry->tableShmkey < keyRangeBase) + { + return keyRangeBase + fixedKeys; + } + return masterTableEntry->tableShmkey + 1; } /* Must be called holding the EM write lock @@ -1927,6 +2332,31 @@ void ExtentMap::growEMShmseg(size_t nrows) fExtentMap = fPExtMapImpl->get(); } +void ExtentMap::growEMIndexShmseg(const size_t suggestedSize) +{ + static const constexpr int InitEMIndexSize_ = 16 * 1024 * 1024; + size_t allocSize = std::max(InitEMIndexSize_, fEMIndexShminfo->allocdSize); + key_t newshmkey = chooseEMIndexShmkey(); + key_t fixedManagedSegmentKey = getInitialEMIndexShmkey(); + + allocSize = std::max(allocSize, suggestedSize); + if (!fPExtMapIndexImpl_) + { + fPExtMapIndexImpl_ = + ExtentMapIndexImpl::makeExtentMapIndexImpl(fixedManagedSegmentKey, allocSize, r_only); + } + else + { + fPExtMapIndexImpl_->growIfNeeded(allocSize); + } + + if (r_only) + fPExtMapIndexImpl_->makeReadOnly(); + + fEMIndexShminfo->tableShmkey = newshmkey; + fEMIndexShminfo->allocdSize = allocSize; +} + /* Must be called holding the FL lock Returns with the new shmseg mapped */ void ExtentMap::growFLShmseg() @@ -1986,7 +2416,6 @@ int ExtentMap::lookup(LBID_t lbid, LBID_t& firstLbid, LBID_t& lastLbid) #ifdef BRM_DEBUG - // printEM(); if (lbid < 0) { log("ExtentMap::lookup(): lbid must be >= 0", logging::LOG_TYPE_DEBUG); @@ -1997,6 +2426,7 @@ int ExtentMap::lookup(LBID_t lbid, LBID_t& firstLbid, LBID_t& lastLbid) #endif grabEMEntryTable(READ); + grabEMIndex(READ); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < entries; i++) @@ -2009,12 +2439,13 @@ int ExtentMap::lookup(LBID_t lbid, LBID_t& firstLbid, LBID_t& lastLbid) { firstLbid = fExtentMap[i].range.start; lastLbid = lastBlock; + releaseEMIndex(READ); releaseEMEntryTable(READ); return 0; } } } - + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; } @@ -2063,6 +2494,7 @@ int ExtentMap::lookupLocal(LBID_t lbid, int& OID, uint16_t& dbRoot, uint32_t& pa } grabEMEntryTable(READ); + grabEMIndex(READ); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -2083,12 +2515,13 @@ int ExtentMap::lookupLocal(LBID_t lbid, int& OID, uint16_t& dbRoot, uint32_t& pa offset = lbid - fExtentMap[i].range.start; fileBlockOffset = fExtentMap[i].blockOffset + offset; + releaseEMIndex(READ); releaseEMEntryTable(READ); return 0; } } } - + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; } @@ -2110,7 +2543,7 @@ int ExtentMap::lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, } #endif - int entries, i, offset; + int offset; if (OID < 0) { @@ -2119,25 +2552,31 @@ int ExtentMap::lookupLocal(int OID, uint32_t partitionNum, uint16_t segmentNum, } grabEMEntryTable(READ); + grabEMIndex(READ); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + DBRootVec dbRootVec(getAllDbRoots()); - for (i = 0; i < entries; i++) + for (auto dbRoot : dbRootVec) { - // TODO: Blockoffset logic. - if (fExtentMap[i].range.size != 0 && fExtentMap[i].fileID == OID && - fExtentMap[i].partitionNum == partitionNum && fExtentMap[i].segmentNum == segmentNum && - fExtentMap[i].blockOffset <= fileBlockOffset && - fileBlockOffset <= - (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + for (auto i : emIdents) { - offset = fileBlockOffset - fExtentMap[i].blockOffset; - LBID = fExtentMap[i].range.start + offset; - releaseEMEntryTable(READ); - return 0; + // TODO: Blockoffset logic. + if (fExtentMap[i].range.size != 0 && fExtentMap[i].segmentNum == segmentNum && + fExtentMap[i].blockOffset <= fileBlockOffset && + fileBlockOffset <= + (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) + { + offset = fileBlockOffset - fExtentMap[i].blockOffset; + LBID = fExtentMap[i].range.start + offset; + releaseEMIndex(READ); + releaseEMEntryTable(READ); + return 0; + } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; } @@ -2168,6 +2607,7 @@ int ExtentMap::lookupLocal_DBroot(int OID, uint16_t dbroot, uint32_t partitionNu } grabEMEntryTable(READ); + grabEMIndex(READ); entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -2182,11 +2622,13 @@ int ExtentMap::lookupLocal_DBroot(int OID, uint16_t dbroot, uint32_t partitionNu { offset = fileBlockOffset - fExtentMap[i].blockOffset; LBID = fExtentMap[i].range.start + offset; + releaseEMIndex(READ); releaseEMEntryTable(READ); return 0; } } + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; } @@ -2214,7 +2656,6 @@ int ExtentMap::lookupLocalStartLbid(int OID, uint32_t partitionNum, uint16_t seg } #endif - int entries, i; if (OID < 0) { @@ -2225,22 +2666,29 @@ int ExtentMap::lookupLocalStartLbid(int OID, uint32_t partitionNum, uint16_t seg } grabEMEntryTable(READ); - entries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + grabEMIndex(READ); - for (i = 0; i < entries; i++) + DBRootVec dbRootVec(getAllDbRoots()); + + for (auto dbRoot : dbRootVec) { - if (fExtentMap[i].range.size != 0 && fExtentMap[i].fileID == OID && - fExtentMap[i].partitionNum == partitionNum && fExtentMap[i].segmentNum == segmentNum && - fExtentMap[i].blockOffset <= fileBlockOffset && - fileBlockOffset <= - (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + for (auto i : emIdents) { - LBID = fExtentMap[i].range.start; - releaseEMEntryTable(READ); - return 0; + if (fExtentMap[i].range.size != 0 && fExtentMap[i].segmentNum == segmentNum && + fExtentMap[i].blockOffset <= fileBlockOffset && + fileBlockOffset <= + (fExtentMap[i].blockOffset + (static_cast(fExtentMap[i].range.size) * 1024) - 1)) + { + LBID = fExtentMap[i].range.start; + releaseEMIndex(READ); + releaseEMEntryTable(READ); + return 0; + } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); return -1; @@ -2270,6 +2718,7 @@ void ExtentMap::createStripeColumnExtents(const vectorcurrentSize == fEMShminfo->allocdSize) + { growEMShmseg(); + } // size is the number of multiples of 1024 blocks. // ex: size=1 --> 1024 blocks @@ -2414,11 +2866,11 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& startBlockOffset) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; - uint32_t highestPartNum = 0; - uint16_t highestSegNum = 0; + EmptyEMEntry emptyEMEntry = -1; + LastExtentIndexT lastExtentIndex = -1; + HighestOffset highestOffset = 0; + PartitionNumberT highestPartNum = 0; + SegmentT highestSegNum = 0; const unsigned FILES_PER_COL_PART = getFilesPerColumnPartition(); const unsigned EXTENT_ROWS = getExtentRows(); const unsigned EXTENTS_PER_SEGFILE = getExtentsPerSegmentFile(); @@ -2426,9 +2878,9 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co // Variables that track list of segfiles in target (HWM) DBRoot & partition. // Map segment number to the highest fbo extent in each file - typedef tr1::unordered_map TargetDbRootSegsMap; - typedef TargetDbRootSegsMap::iterator TargetDbRootSegsMapIter; - typedef TargetDbRootSegsMap::const_iterator TargetDbRootSegsMapConstIter; + using TargetDbRootSegsMap = tr1::unordered_map; + using TargetDbRootSegsMapIter = TargetDbRootSegsMap::iterator; + using TargetDbRootSegsMapConstIter = TargetDbRootSegsMap::const_iterator; TargetDbRootSegsMap targetDbRootSegs; uint32_t highEmptySegNum = 0; // high seg num for user specified partition; @@ -2441,50 +2893,74 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co // 2. if DBRoot is empty, track highest seg num in user specified partition // 3. Find first unused extent map entry //-------------------------------------------------------------------------- - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - LBID_t startLBID = getLBIDsFromFreeList(size); + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID); // Find the first empty Entry; and find last extent for this OID and dbRoot - for (int i = 0; i < emEntries; i++) + + for (auto i : emIdents) { if (fExtentMap[i].range.size != 0) { - if (fExtentMap[i].fileID == OID) + // 1. Find HWM extent in relevant DBRoot + if ((fExtentMap[i].partitionNum > highestPartNum) || + ((fExtentMap[i].partitionNum == highestPartNum) && (fExtentMap[i].blockOffset > highestOffset)) || + ((fExtentMap[i].partitionNum == highestPartNum) && (fExtentMap[i].blockOffset == highestOffset) && + (fExtentMap[i].segmentNum >= highestSegNum))) { - // 1. Find HWM extent in relevant DBRoot - if (fExtentMap[i].dbRoot == dbRoot) - { - if ((fExtentMap[i].partitionNum > highestPartNum) || - ((fExtentMap[i].partitionNum == highestPartNum) && - (fExtentMap[i].blockOffset > highestOffset)) || - ((fExtentMap[i].partitionNum == highestPartNum) && - (fExtentMap[i].blockOffset == highestOffset) && (fExtentMap[i].segmentNum >= highestSegNum))) - { - lastExtentIndex = i; - highestPartNum = fExtentMap[i].partitionNum; - highestSegNum = fExtentMap[i].segmentNum; - highestOffset = fExtentMap[i].blockOffset; - } - } - - // 2. for empty DBRoot track hi seg# in user specified part# - if ((lastExtentIndex == -1) && (fExtentMap[i].partitionNum == partitionNum)) - { - if ((fExtentMap[i].segmentNum > highEmptySegNum) || (!bHighEmptySegNumSet)) - { - highEmptySegNum = fExtentMap[i].segmentNum; - bHighEmptySegNumSet = true; - } - } - } // found extentmap entry for specified OID - } // found valid extentmap entry + lastExtentIndex = i; + highestPartNum = fExtentMap[i].partitionNum; + highestSegNum = fExtentMap[i].segmentNum; + highestOffset = fExtentMap[i].blockOffset; + } + } // found valid extentmap entry // 3. Find first available extent map entry that can be reused else if (emptyEMEntry < 0) emptyEMEntry = i; } // Loop through extent map entries + DBRootVec dbRootVec(getAllDbRoots()); + // 2. for empty DBRoot track hi seg# in user specified part# + if (lastExtentIndex == -1) + { + // loop over all extents that doesn't belong to the target dbroot + for (auto dbRootFromList : dbRootVec) + { + if (dbRootFromList == dbRoot) + continue; + + auto emIdentsLocal = fPExtMapIndexImpl_->find(dbRootFromList, OID, partitionNum); + for (auto i : emIdentsLocal) + { + if ((fExtentMap[i].range.size != 0) && + ((fExtentMap[i].segmentNum > highEmptySegNum) || (!bHighEmptySegNumSet))) + { + highEmptySegNum = fExtentMap[i].segmentNum; + bHighEmptySegNumSet = true; + } + + // Search for the first empty Entry + if (fExtentMap[i].range.size == 0) + { + emptyEMEntry = i; + break; + } + } + } + } + + size_t emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + // Search for the first empty Entry + for (size_t i = 0; emptyEMEntry < 0 && i < emEntries; ++i) + { + if (fExtentMap[i].range.size == 0) + { + emptyEMEntry = i; + break; + } + } + if (emptyEMEntry == -1) { ostringstream oss; @@ -2516,64 +2992,73 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co int partHighSeg = -1; // hi seg num for last partition int partHighSegNext = -1; // hi seg num for next partition + // Target dbroot has extents for the OID if (lastExtentIndex >= 0) { - uint32_t targetDbRootPart = fExtentMap[lastExtentIndex].partitionNum; - uint32_t targetDbRootPartNext = targetDbRootPart + 1; + PartitionNumberT targetDbRootPart = fExtentMap[lastExtentIndex].partitionNum; + PartitionNumberT targetDbRootPartNext = targetDbRootPart + 1; partHighSeg = fExtentMap[lastExtentIndex].segmentNum; targetDbRootSegs.insert(TargetDbRootSegsMap::value_type(fExtentMap[lastExtentIndex].segmentNum, fExtentMap[lastExtentIndex].blockOffset)); - for (int i = 0; i < emEntries; i++) + for (auto dbRootFromList : dbRootVec) { - if (fExtentMap[i].range.size != 0) + if (dbRootFromList == dbRoot) { - if (fExtentMap[i].fileID == OID) + auto emIdents = fPExtMapIndexImpl_->find(dbRootFromList, OID, targetDbRootPart); + for (auto i : emIdents) { - // 4. Track hi seg for hwm+1 partition - if (fExtentMap[i].partitionNum == targetDbRootPartNext) - { - if (fExtentMap[i].segmentNum > partHighSegNext) - { - partHighSegNext = fExtentMap[i].segmentNum; - } - } - // 5. Track hi seg for hwm partition - else if (fExtentMap[i].partitionNum == targetDbRootPart) + if (fExtentMap[i].segmentNum > partHighSeg) { - if (fExtentMap[i].segmentNum > partHighSeg) + partHighSeg = fExtentMap[i].segmentNum; + } + + // 6. Save list of seg files in target DBRoot/Partition, + // along with the highest fbo for each seg file + if (fExtentMap[i].status == EXTENTOUTOFSERVICE) + bSegsOutOfService = true; + + TargetDbRootSegsMapIter iter = targetDbRootSegs.find(fExtentMap[i].segmentNum); + + if (iter == targetDbRootSegs.end()) + { + targetDbRootSegs.insert( + TargetDbRootSegsMap::value_type(fExtentMap[i].segmentNum, fExtentMap[i].blockOffset)); + } + else + { + if (fExtentMap[i].blockOffset > iter->second) { - partHighSeg = fExtentMap[i].segmentNum; - } - - // 6. Save list of seg files in target DBRoot/Partition, - // along with the highest fbo for each seg file - if (fExtentMap[i].dbRoot == dbRoot) - { - if (fExtentMap[i].status == EXTENTOUTOFSERVICE) - bSegsOutOfService = true; - - TargetDbRootSegsMapIter iter = targetDbRootSegs.find(fExtentMap[i].segmentNum); - - if (iter == targetDbRootSegs.end()) - { - targetDbRootSegs.insert( - TargetDbRootSegsMap::value_type(fExtentMap[i].segmentNum, fExtentMap[i].blockOffset)); - } - else - { - if (fExtentMap[i].blockOffset > iter->second) - { - iter->second = fExtentMap[i].blockOffset; - } - } + iter->second = fExtentMap[i].blockOffset; } } - } // found extentmap entry for specified OID - } // found valid extentmap entry - } // loop through extent map entries - } // (lastExtentIndex >= 0) + } // loop over em idents + } // current dbroot == target dbroot + else + { + // 4. Track hi seg for hwm+1 partition + auto emIdentsNext = fPExtMapIndexImpl_->find(dbRootFromList, OID, targetDbRootPartNext); + for (auto i : emIdentsNext) + { + if (fExtentMap[i].segmentNum > partHighSegNext) + { + partHighSegNext = fExtentMap[i].segmentNum; + } + } + + // 5. Track hi seg for hwm partition + auto emIdents = fPExtMapIndexImpl_->find(dbRootFromList, OID, targetDbRootPart); + for (auto i : emIdents) + { + if (fExtentMap[i].segmentNum > partHighSeg) + { + partHighSeg = fExtentMap[i].segmentNum; + } + } + } // current dbroot != target dbroot + } // loop over dbroots + } // (lastExtentIndex >= 0) //-------------------------------------------------------------------------- // Third Step: Select partition and segment number for new extent @@ -2845,6 +3330,13 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); fEMShminfo->currentSize += sizeof(struct EMEntry); + auto resShmemHasGrownPair = fPExtMapIndexImpl_->insert(fExtentMap[emptyEMEntry], emptyEMEntry); + + if (resShmemHasGrownPair.second) + fEMIndexShminfo->allocdSize = fPExtMapIndexImpl_->getShmemSize(); + + if (!resShmemHasGrownPair.first) + logAndSetEMIndexReadOnly("_createColumnExtent_DBroot"); return startLBID; } @@ -2901,10 +3393,13 @@ void ExtentMap::createColumnExtentExactFile(int OID, uint32_t colWidth, uint16_t // extentRows should be multiple of blocksize (8192). const unsigned EXTENT_SIZE = (getExtentRows() * colWidth) / BLOCK_SIZE; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); if (fEMShminfo->currentSize == fEMShminfo->allocdSize) + { growEMShmseg(); + } // size is the number of multiples of 1024 blocks. // ex: size=1 --> 1024 blocks @@ -2918,6 +3413,59 @@ void ExtentMap::createColumnExtentExactFile(int OID, uint32_t colWidth, uint16_t allocdsize = EXTENT_SIZE; } +LastIndEmptyIndEmptyInd ExtentMap::_createExtentCommonSearch(const OID_t OID, const DBRootT dbRoot, + const PartitionNumberT partitionNum, + const SegmentT segmentNum) +{ + EmptyEMEntry emptyEMEntry = -1; + LastExtentIndexT lastExtentIndex = -1; + HighestOffset highestOffset = 0; + + size_t emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + // DRRTUY we might need to use cache preload here. + // Search of the last extent idx and the highest offset + for (auto i : emIdents) + { + if (fExtentMap[i].range.size != 0) + { + if ((fExtentMap[i].segmentNum == segmentNum) && (fExtentMap[i].blockOffset >= highestOffset)) + { + lastExtentIndex = i; + highestOffset = fExtentMap[i].blockOffset; + } + } + // Search for the first empty Entry + else if (emptyEMEntry < 0) + emptyEMEntry = i; + } + + // Search for the first empty Entry + // DRRTUY We might need to support empty EM ids vector + for (size_t i = 0; emptyEMEntry < 0 && i < emEntries; ++i) + { + if (fExtentMap[i].range.size == 0) + { + emptyEMEntry = i; + break; + } + } + return {lastExtentIndex, emptyEMEntry}; +} + +void ExtentMap::logAndSetEMIndexReadOnly(const std::string& funcName) +{ + fPExtMapIndexImpl_->makeReadOnly(); + ostringstream os; + os << "ExtentMap::" << funcName << ": " + << "Can not update EM Index. EM Index shmem segment is set to" + << " readonly. Please restart Columnstore."; + log(os.str(), logging::LOG_TYPE_CRITICAL); + + throw logic_error(os.str()); +} + //------------------------------------------------------------------------------ // Creates an extent for the exact segment file specified by the requested // OID, DBRoot, partition, and segment. This is the internal implementation @@ -2941,32 +3489,9 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, uint32_t execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t& startBlockOffset) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; - - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - LBID_t startLBID = getLBIDsFromFreeList(size); - - // Find the first empty Entry; and find the last extent for this - // combination of OID, partition, and segment. - for (int i = 0; i < emEntries; i++) - { - if (fExtentMap[i].range.size != 0) - { - if (fExtentMap[i].fileID == OID) - { - if ((fExtentMap[i].dbRoot == dbRoot) && (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum) && (fExtentMap[i].blockOffset >= highestOffset)) - { - lastExtentIndex = i; - highestOffset = fExtentMap[i].blockOffset; - } - } - } - else if (emptyEMEntry < 0) - emptyEMEntry = i; - } // Loop through extent map entries + auto lastIndEmptyIndEmptyInd = _createExtentCommonSearch(OID, dbRoot, partitionNum, segmentNum); + LastExtentIndexT lastExtentIndex = lastIndEmptyIndEmptyInd.first; + EmptyEMEntry emptyEMEntry = lastIndEmptyIndEmptyInd.second; if (emptyEMEntry == -1) { @@ -2981,6 +3506,7 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, uint32_t makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); EMEntry* e = &fExtentMap[emptyEMEntry]; + LBID_t startLBID = getLBIDsFromFreeList(size); e->range.start = startLBID; e->range.size = size; e->fileID = OID; @@ -3058,6 +3584,13 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, uint32_t makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); fEMShminfo->currentSize += sizeof(struct EMEntry); + auto resShmemHasGrownPair = fPExtMapIndexImpl_->insert(fExtentMap[emptyEMEntry], emptyEMEntry); + + if (resShmemHasGrownPair.second) + fEMIndexShminfo->allocdSize = fPExtMapIndexImpl_->getShmemSize(); + + if (!resShmemHasGrownPair.first) + logAndSetEMIndexReadOnly("_createColumnExtentExactFile"); return startLBID; } @@ -3108,10 +3641,13 @@ void ExtentMap::createDictStoreExtent(int OID, uint16_t dbRoot, uint32_t partiti const unsigned EXTENT_SIZE = (getExtentRows() * DICT_COL_WIDTH) / BLOCK_SIZE; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); if (fEMShminfo->currentSize == fEMShminfo->allocdSize) + { growEMShmseg(); + } // size is the number of multiples of 1024 blocks. // ex: size=1 --> 1024 blocks @@ -3141,30 +3677,9 @@ void ExtentMap::createDictStoreExtent(int OID, uint16_t dbRoot, uint32_t partiti LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum) { - int emptyEMEntry = -1; - int lastExtentIndex = -1; - uint32_t highestOffset = 0; - - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - LBID_t startLBID = getLBIDsFromFreeList(size); - - // Find the first empty Entry; and find the last extent for this - // combination of OID, partition, and segment. - for (int i = 0; i < emEntries; i++) - { - if (fExtentMap[i].range.size != 0) - { - if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].partitionNum == partitionNum) && - (fExtentMap[i].segmentNum == segmentNum) && (fExtentMap[i].blockOffset >= highestOffset)) - { - lastExtentIndex = i; - highestOffset = fExtentMap[i].blockOffset; - } - } - else if (emptyEMEntry < 0) - emptyEMEntry = i; - } // Loop through extent map entries + auto lastIndEmptyIndEmptyInd = _createExtentCommonSearch(OID, dbRoot, partitionNum, segmentNum); + LastExtentIndexT lastExtentIndex = lastIndEmptyIndEmptyInd.first; + EmptyEMEntry emptyEMEntry = lastIndEmptyIndEmptyInd.second; if (emptyEMEntry == -1) { @@ -3179,6 +3694,7 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, uint16_t dbRoot makeUndoRecord(&fExtentMap[emptyEMEntry], sizeof(EMEntry)); EMEntry* e = &fExtentMap[emptyEMEntry]; + LBID_t startLBID = getLBIDsFromFreeList(size); e->range.start = startLBID; e->range.size = size; e->fileID = OID; @@ -3215,6 +3731,13 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, uint16_t dbRoot makeUndoRecord(fEMShminfo, sizeof(MSTEntry)); fEMShminfo->currentSize += sizeof(struct EMEntry); + auto resShmemHasGrownPair = fPExtMapIndexImpl_->insert(fExtentMap[emptyEMEntry], emptyEMEntry); + + if (resShmemHasGrownPair.second) + fEMIndexShminfo->allocdSize = fPExtMapIndexImpl_->getShmemSize(); + + if (!resShmemHasGrownPair.first) + logAndSetEMIndexReadOnly("_createDictStoreExtent"); return startLBID; } @@ -3378,6 +3901,7 @@ void ExtentMap::rollbackColumnExtents_DBroot(int oid, bool bDeleteAll, uint16_t uint32_t fboLoPreviousStripe = 0; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -3576,6 +4100,7 @@ void ExtentMap::rollbackDictStoreExtents_DBroot(int oid, uint16_t dbRoot, uint32 tr1::unordered_map >::const_iterator segToHwmMapIter; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -3702,6 +4227,7 @@ void ExtentMap::deleteEmptyColExtents(const ExtentsInfoMap_t& extentsInfo) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); uint32_t fboLo = 0; @@ -3837,6 +4363,7 @@ void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); ExtentsInfoMap_t::const_iterator it; @@ -3967,8 +4494,15 @@ void ExtentMap::deleteOID(int OID) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); + // Clean up the index and tell deleteExtent to skip the clean-up. + DBRootVec dbRootVec(getAllDbRoots()); + for (auto dbRoot : dbRootVec) + fPExtMapIndexImpl_->deleteOID(dbRoot, OID); + const bool clearEMIndex = false; + int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (int emIndex = 0; emIndex < emEntries; emIndex++) @@ -3976,8 +4510,7 @@ void ExtentMap::deleteOID(int OID) if (fExtentMap[emIndex].range.size > 0 && fExtentMap[emIndex].fileID == OID) { OIDExists = true; - - deleteExtent(emIndex); + deleteExtent(emIndex, clearEMIndex); } } @@ -4005,10 +4538,20 @@ void ExtentMap::deleteOIDs(const OidsMap_t& OIDs) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); + OidsMap_t::const_iterator it; int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); + const bool clearEMIndex = false; + DBRootVec dbRootVec(getAllDbRoots()); + for (auto dbRoot : dbRootVec) + { + for (auto& oidOidPair : OIDs) + fPExtMapIndexImpl_->deleteOID(dbRoot, oidOidPair.first); + } + for (int emIndex = 0; emIndex < emEntries; emIndex++) { if (fExtentMap[emIndex].range.size > 0) @@ -4016,7 +4559,7 @@ void ExtentMap::deleteOIDs(const OidsMap_t& OIDs) it = OIDs.find(fExtentMap[emIndex].fileID); if (it != OIDs.end()) - deleteExtent(emIndex); + deleteExtent(emIndex, clearEMIndex); } } } @@ -4025,7 +4568,7 @@ void ExtentMap::deleteOIDs(const OidsMap_t& OIDs) // Delete the specified extent from the extentmap and return to the free list. // emIndex - the index (from the extent map) of the extent to be deleted //------------------------------------------------------------------------------ -void ExtentMap::deleteExtent(int emIndex) +void ExtentMap::deleteExtent(const int emIndex, const bool clearEMIndex) { int flIndex, freeFLIndex, flEntries, preceedingExtent, succeedingExtent; LBID_t flBlockEnd, emBlockEnd; @@ -4148,6 +4691,8 @@ void ExtentMap::deleteExtent(int emIndex) // invalidate the entry in the Extent Map makeUndoRecord(&fExtentMap[emIndex], sizeof(EMEntry)); fExtentMap[emIndex].range.size = 0; + if (clearEMIndex) + fPExtMapIndexImpl_->deleteEMEntry(fExtentMap[emIndex], emIndex); makeUndoRecord(&fEMShminfo, sizeof(MSTEntry)); fEMShminfo->currentSize -= sizeof(struct EMEntry); } @@ -4197,6 +4742,7 @@ HWM_t ExtentMap::getLastHWM_DBroot(int OID, uint16_t dbRoot, uint32_t& partition } grabEMEntryTable(READ); + grabEMIndex(READ); // Searching the array in reverse order should be faster since the last // extent is usually at the bottom. We still have to search the entire @@ -4231,6 +4777,7 @@ HWM_t ExtentMap::getLastHWM_DBroot(int OID, uint16_t dbRoot, uint32_t& partition bFound = true; } + releaseEMIndex(READ); releaseEMEntryTable(READ); return hwm; @@ -4266,14 +4813,14 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v& // Determine List of DBRoots for specified PM, and construct map of // EmDbRootHWMInfo objects. tr1::unordered_map emDbRootMap; - vector dbRootList; - getPmDbRoots(pmNumber, dbRootList); + vector dbRootVec; + getPmDbRoots(pmNumber, dbRootVec); - if (dbRootList.size() > 0) + if (dbRootVec.size() > 0) { - for (unsigned int iroot = 0; iroot < dbRootList.size(); iroot++) + for (unsigned int iroot = 0; iroot < dbRootVec.size(); iroot++) { - uint16_t rootID = dbRootList[iroot]; + uint16_t rootID = dbRootVec[iroot]; EmDbRootHWMInfo emDbRootInfo(rootID); emDbRootMap[rootID] = emDbRootInfo; } @@ -4289,47 +4836,52 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v& } grabEMEntryTable(READ); + grabEMIndex(READ); tr1::unordered_map::iterator emIter; // Searching the array in reverse order should be faster since the last // extent is usually at the bottom. We still have to search the entire // array (just in case), but the number of operations per loop iteration // will be less. - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (int i = emEntries - 1; i >= 0; i--) + for (auto dbRoot : dbRootVec) { - if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID)) + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID); + for (auto i : emIdents) { - // Include this extent in the search, only if the extent's - // DBRoot falls in the list of DBRoots for this PM. - emIter = emDbRootMap.find(fExtentMap[i].dbRoot); - - if (emIter == emDbRootMap.end()) - continue; - - EmDbRootHWMInfo& emDbRoot = emIter->second; - - if ((fExtentMap[i].status != EXTENTOUTOFSERVICE) && (fExtentMap[i].HWM != 0)) - emDbRoot.totalBlocks += (fExtentMap[i].HWM + 1); - - if ((fExtentMap[i].partitionNum > emDbRoot.partitionNum) || - ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && - (fExtentMap[i].blockOffset > emDbRoot.fbo)) || - ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && - (fExtentMap[i].blockOffset == emDbRoot.fbo) && (fExtentMap[i].segmentNum >= emDbRoot.segmentNum))) + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID)) { - emDbRoot.fbo = fExtentMap[i].blockOffset; - emDbRoot.partitionNum = fExtentMap[i].partitionNum; - emDbRoot.segmentNum = fExtentMap[i].segmentNum; - emDbRoot.localHWM = fExtentMap[i].HWM; - emDbRoot.startLbid = fExtentMap[i].range.start; - emDbRoot.status = fExtentMap[i].status; - emDbRoot.hwmExtentIndex = i; + // Include this extent in the search, only if the extent's + // DBRoot falls in the list of DBRoots for this PM. + emIter = emDbRootMap.find(fExtentMap[i].dbRoot); + + if (emIter == emDbRootMap.end()) + continue; + + EmDbRootHWMInfo& emDbRoot = emIter->second; + + if ((fExtentMap[i].status != EXTENTOUTOFSERVICE) && (fExtentMap[i].HWM != 0)) + emDbRoot.totalBlocks += (fExtentMap[i].HWM + 1); + + if ((fExtentMap[i].partitionNum > emDbRoot.partitionNum) || + ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && + (fExtentMap[i].blockOffset > emDbRoot.fbo)) || + ((fExtentMap[i].partitionNum == emDbRoot.partitionNum) && + (fExtentMap[i].blockOffset == emDbRoot.fbo) && + (fExtentMap[i].segmentNum >= emDbRoot.segmentNum))) + { + emDbRoot.fbo = fExtentMap[i].blockOffset; + emDbRoot.partitionNum = fExtentMap[i].partitionNum; + emDbRoot.segmentNum = fExtentMap[i].segmentNum; + emDbRoot.localHWM = fExtentMap[i].HWM; + emDbRoot.startLbid = fExtentMap[i].range.start; + emDbRoot.status = fExtentMap[i].status; + emDbRoot.hwmExtentIndex = i; + } } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); for (tr1::unordered_map::iterator iter = emDbRootMap.begin(); @@ -4409,6 +4961,7 @@ void ExtentMap::getExtentState(int OID, uint32_t partitionNum, uint16_t segmentN } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -4423,6 +4976,7 @@ void ExtentMap::getExtentState(int OID, uint32_t partitionNum, uint16_t segmentN } } + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -4455,7 +5009,6 @@ HWM_t ExtentMap::getLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum #endif - int i, emEntries; HWM_t ret = 0; bool OIDPartSegExists = false; @@ -4468,26 +5021,31 @@ HWM_t ExtentMap::getLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum } grabEMEntryTable(READ); + grabEMIndex(READ); - emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (i = 0; i < emEntries; i++) + DBRootVec dbRootVec(getAllDbRoots()); + for (auto dbRoot : dbRootVec) { - if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID) && - (fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].segmentNum == segmentNum)) + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + for (auto i : emIdents) { - OIDPartSegExists = true; - status = fExtentMap[i].status; - - if (fExtentMap[i].HWM != 0) + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].segmentNum == segmentNum)) { - ret = fExtentMap[i].HWM; - releaseEMEntryTable(READ); - return ret; + OIDPartSegExists = true; + status = fExtentMap[i].status; + + if (fExtentMap[i].HWM != 0) + { + ret = fExtentMap[i].HWM; + releaseEMIndex(READ); + releaseEMEntryTable(READ); + return ret; + } } } } + releaseEMIndex(READ); releaseEMEntryTable(READ); if (OIDPartSegExists) @@ -4534,31 +5092,37 @@ void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum, #endif - int lastExtentIndex = -1; + LastExtentIndexT lastExtentIndex = -1; int oldHWMExtentIndex = -1; - uint32_t highestOffset = 0; + HighestOffset highestOffset = 0; if (uselock) - grabEMEntryTable(WRITE); - - int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); - - for (int i = 0; i < emEntries; i++) { - if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].fileID == OID) && - (fExtentMap[i].partitionNum == partitionNum) && (fExtentMap[i].segmentNum == segmentNum)) - { - // Find current HWM extent - if (fExtentMap[i].blockOffset >= highestOffset) - { - highestOffset = fExtentMap[i].blockOffset; - lastExtentIndex = i; - } + grabEMEntryTable(WRITE); + grabEMIndex(WRITE); + } - // Find previous HWM extent - if (fExtentMap[i].HWM != 0) + DBRootVec dbRootVec(getAllDbRoots()); + + for (auto dbRoot : dbRootVec) + { + auto emIdents = fPExtMapIndexImpl_->find(dbRoot, OID, partitionNum); + for (auto i : emIdents) + { + if ((fExtentMap[i].range.size != 0) && (fExtentMap[i].segmentNum == segmentNum)) { - oldHWMExtentIndex = i; + // Find current HWM extent + if (fExtentMap[i].blockOffset >= highestOffset) + { + highestOffset = fExtentMap[i].blockOffset; + lastExtentIndex = i; + } + + // Find previous HWM extent + if (fExtentMap[i].HWM != 0) + { + oldHWMExtentIndex = i; + } } } } @@ -4633,6 +5197,7 @@ void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, uint16_t segmentNum, void ExtentMap::bulkSetHWM(const vector& v, bool firstNode) { grabEMEntryTable(WRITE); + grabEMIndex(WRITE); for (uint32_t i = 0; i < v.size(); i++) setLocalHWM(v[i].oid, v[i].partNum, v[i].segNum, v[i].hwm, firstNode, false); @@ -4667,6 +5232,7 @@ void ExtentMap::bulkUpdateDBRoot(const vector& args) sArgs.insert(args[i]); grabEMEntryTable(WRITE); + grabEMIndex(WRITE); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -4706,10 +5272,10 @@ void ExtentMap::getExtents(int OID, vector& entries, bool sorted } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); // Pre-expand entries to stop lots of small allocs entries.reserve(emEntries); - if (incOutOfService) { for (i = 0; i < emEntries; i++) @@ -4724,6 +5290,7 @@ void ExtentMap::getExtents(int OID, vector& entries, bool sorted entries.push_back(fExtentMap[i]); } + releaseEMIndex(READ); releaseEMEntryTable(READ); if (sorted) @@ -4781,12 +5348,14 @@ void ExtentMap::getExtents_dbroot(int OID, vector& entries, cons } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < emEntries; i++) if ((fExtentMap[i].fileID == OID) && (fExtentMap[i].range.size != 0) && (fExtentMap[i].dbRoot == dbroot)) entries.push_back(fExtentMap[i]); + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -4808,6 +5377,7 @@ void ExtentMap::getExtentCount_dbroot(int OID, uint16_t dbroot, bool incOutOfSer } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); numExtents = 0; @@ -4831,6 +5401,7 @@ void ExtentMap::getExtentCount_dbroot(int OID, uint16_t dbroot, bool incOutOfSer } } + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -4856,6 +5427,7 @@ void ExtentMap::getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) bool bFound = false; grabEMEntryTable(READ); + grabEMIndex(READ); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (int i = 0; i < emEntries; i++) @@ -4868,6 +5440,7 @@ void ExtentMap::getSysCatDBRoot(OID_t oid, uint16_t& dbRoot) } } + releaseEMIndex(READ); releaseEMEntryTable(READ); if (!bFound) @@ -4918,6 +5491,7 @@ void ExtentMap::deletePartition(const set& oids, const set foundPartitions; int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); @@ -5019,6 +5593,7 @@ void ExtentMap::markPartitionForDeletion(const set& oids, const setallocdSize / sizeof(struct EMEntry); set foundPartitions; vector extents; @@ -5126,6 +5701,7 @@ void ExtentMap::markAllPartitionForDeletion(const set& oids) set::const_iterator it; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (int i = 0; i < emEntries; i++) @@ -5179,6 +5755,7 @@ void ExtentMap::restorePartition(const set& oids, const set::const_iterator it; grabEMEntryTable(WRITE); + grabEMIndex(WRITE); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); vector extents; @@ -5271,6 +5848,7 @@ void ExtentMap::getOutOfServicePartitions(OID_t oid, set& part } grabEMEntryTable(READ); + grabEMIndex(READ); int emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (int i = 0; i < emEntries; i++) @@ -5284,6 +5862,7 @@ void ExtentMap::getOutOfServicePartitions(OID_t oid, set& part } } + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -5305,11 +5884,13 @@ void ExtentMap::deleteDBRoot(uint16_t dbroot) #endif grabEMEntryTable(WRITE); + grabEMIndex(WRITE); grabFreeList(WRITE); for (unsigned i = 0; i < fEMShminfo->allocdSize / sizeof(struct EMEntry); i++) if (fExtentMap[i].range.size != 0 && fExtentMap[i].dbRoot == dbroot) deleteExtent(i); + fPExtMapIndexImpl_->deleteDbRoot(dbroot); } //------------------------------------------------------------------------------ @@ -5332,6 +5913,7 @@ bool ExtentMap::isDBRootEmpty(uint16_t dbroot) bool bEmpty = true; int i, emEntries; grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); if (fEMShminfo->currentSize == 0) @@ -5348,6 +5930,7 @@ bool ExtentMap::isDBRootEmpty(uint16_t dbroot) } } + releaseEMIndex(READ); releaseEMEntryTable(READ); return bEmpty; @@ -5407,6 +5990,7 @@ void ExtentMap::lookup(OID_t OID, LBIDRange_v& ranges) } grabEMEntryTable(READ); + grabEMIndex(READ); emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (i = 0; i < emEntries; i++) @@ -5418,6 +6002,7 @@ void ExtentMap::lookup(OID_t OID, LBIDRange_v& ranges) ranges.push_back(tmp); } + releaseEMIndex(READ); releaseEMEntryTable(READ); } @@ -5449,6 +6034,7 @@ int ExtentMap::checkConsistency() uint32_t usedEntries; grabEMEntryTable(READ); + grabEMIndex(READ); try { @@ -5456,6 +6042,7 @@ int ExtentMap::checkConsistency() } catch (...) { + releaseEMIndex(READ); releaseEMEntryTable(READ); throw; } @@ -5669,6 +6256,7 @@ int ExtentMap::checkConsistency() cout << "test 5a passed\n"; releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); return 0; } @@ -5707,6 +6295,9 @@ void ExtentMap::finishChanges() if (flLocked) releaseFreeList(WRITE); + if (emIndexLocked) + releaseEMIndex(WRITE); + if (emLocked) releaseEMEntryTable(WRITE); } @@ -5721,6 +6312,11 @@ const bool* ExtentMap::getEMLockStatus() return &emLocked; } +const bool* ExtentMap::getEMIndexLockStatus() +{ + return &emIndexLocked; +} + //------------------------------------------------------------------------------ // Reload Config cache if config file time stamp has changed //------------------------------------------------------------------------------ @@ -5868,19 +6464,36 @@ unsigned ExtentMap::getDbRootCount() // Get list of DBRoots that map to the specified PM. DBRoot list is cached // internally in fPmDbRootMap after getting from Columnstore.xml via OAM. //------------------------------------------------------------------------------ -void ExtentMap::getPmDbRoots(int pm, vector& dbRootList) +void ExtentMap::getPmDbRoots(int pm, vector& dbRootVec) { oam::OamCache* oamcache = oam::OamCache::makeOamCache(); oam::OamCache::PMDbrootsMap_t pmDbroots = oamcache->getPMToDbrootsMap(); - dbRootList.clear(); - dbRootList = (*pmDbroots)[pm]; + dbRootVec.clear(); + dbRootVec = (*pmDbroots)[pm]; +} + +DBRootVec ExtentMap::getAllDbRoots() +{ + DBRootVec dbRootResultVec; + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + // NB The routine uses int for dbroot id that contradicts with the type used here, namely uint16_t + oam::OamCache::PMDbrootsMap_t pmDbroots = oamcache->getPMToDbrootsMap(); + auto& pmDbrootsRef = *pmDbroots; + + for (auto& pmDBRootPair : pmDbrootsRef) + { + for (auto dbRootId : pmDBRootPair.second) + dbRootResultVec.push_back(dbRootId); + } + return dbRootResultVec; } vector ExtentMap::getFreeListEntries() { vector v; grabEMEntryTable(READ); + grabEMIndex(READ); grabFreeList(READ); int allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); @@ -5889,6 +6502,7 @@ vector ExtentMap::getFreeListEntries() v.push_back(fFreeList[i]); releaseFreeList(READ); + releaseEMIndex(READ); releaseEMEntryTable(READ); return v; } @@ -5896,6 +6510,7 @@ vector ExtentMap::getFreeListEntries() void ExtentMap::dumpTo(ostream& os) { grabEMEntryTable(READ); + grabEMIndex(READ); unsigned emEntries = fEMShminfo->allocdSize / sizeof(struct EMEntry); for (unsigned i = 0; i < emEntries; i++) @@ -5911,54 +6526,25 @@ void ExtentMap::dumpTo(ostream& os) } } + releaseEMIndex(READ); releaseEMEntryTable(READ); } -/*int ExtentMap::physicalPartitionNum(const set& oids, - const set& partitionNums, - vector& partitionInfos) +size_t ExtentMap::EMIndexShmemSize() { -#ifdef BRM_INFO - if (fDebug) - { - TRACER_WRITENOW("physicalPartitionNum"); - ostringstream oss; - set::const_iterator partIt; - oss << "partitionNums: " - for (partIt=partitionNums.begin(); it!=partitionNums.end(); ++it) - oss << (*it) << " "; - oss << endl; - TRACER_WRITEDIRECT(oss.str()); - } -#endif - - set::const_iterator it; - grabEMEntryTable(READ); - - int emEntries = fEMShminfo->allocdSize/sizeof(struct EMEntry); - PartitionInfo partInfo; - vector extents; - set foundPartitions; - for (int i = 0; i < emEntries; i++) - { - if ((fExtentMap[i].range.size != 0 ) && - partitionNums.find(logicalPartitionNum(fExtentMap[i])) != partitionNums.end()) - { - it = oids.find( fExtentMap[i].fileID ); - if (it != oids.end()) - { - partInfo.oid = fExtentMap[i].fileID; - partInfo.lp.dbroot = fExtentMap[i].dbRoot; - partInfo.lp.pp = fExtentMap[i].partitionNum; - partInfo.lp.seg = fExtentMap[i].segmentNum; - partitionInfos.push_back(partInfo); - } - } - } - releaseEMEntryTable(READ); - return 0; + grabEMIndex(READ); + size_t EMIndexShmemSize = fPExtMapIndexImpl_->getShmemSize(); + releaseEMIndex(READ); + return EMIndexShmemSize; +} + +size_t ExtentMap::EMIndexShmemFree() +{ + grabEMIndex(READ); + size_t EMIndexShmemFree = fPExtMapIndexImpl_->getShmemFree(); + releaseEMIndex(READ); + return EMIndexShmemFree; } -*/ template int ExtentMap::getMaxMin(const LBID_t lbidRange, int128_t& max, int128_t& min, int32_t& seqNum); @@ -5966,4 +6552,4 @@ template int ExtentMap::getMaxMin(const LBID_t lbidRange, int128_t& ma template int ExtentMap::getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); -} // namespace BRM +} // namespace BRM \ No newline at end of file diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index de8cf4df0..805a37c35 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -29,15 +30,18 @@ #include #include #include -#ifdef _MSC_VER #include -#else #include -#endif +#include + //#define NDEBUG #include #include #include +#include +#include +#include +#include //boost::hash #include "shmkeys.h" #include "brmtypes.h" @@ -63,6 +67,8 @@ #define EXPORT #endif +namespace bi = boost::interprocess; + namespace oam { typedef std::vector DBRootConfigList; @@ -75,6 +81,15 @@ class IDBDataFile; namespace BRM { +using PartitionNumberT = uint32_t; +using DBRootT = uint16_t; +using SegmentT = uint16_t; +using LastExtentIndexT = int; +using EmptyEMEntry = int; +using HighestOffset = uint32_t; +using LastIndEmptyIndEmptyInd = std::pair; +using DBRootVec = std::vector; + // assumed column width when calculating dictionary store extent size #define DICT_COL_WIDTH 8 @@ -98,8 +113,6 @@ const char CP_INVALID = 0; const char CP_UPDATING = 1; const char CP_VALID = 2; -// The _v4 structs are defined below for upgrading extent map -// from v4 to v5; see ExtentMap::loadVersion4or5 for details. struct EMCasualPartition_struct_v4 { RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. @@ -113,16 +126,15 @@ struct EMPartition_struct_v4 { EMCasualPartition_struct_v4 cprange; }; - struct EMEntry_v4 { InlineLBIDRange range; int fileID; uint32_t blockOffset; HWM_t HWM; - uint32_t partitionNum; // starts at 0 - uint16_t segmentNum; // starts at 0 - uint16_t dbRoot; // starts at 1 to match Columnstore.xml + PartitionNumberT partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + DBRootT dbRoot; // starts at 1 to match Columnstore.xml uint16_t colWid; int16_t status; // extent avail for query or not, or out of service EMPartition_struct_v4 partition; @@ -151,7 +163,7 @@ struct EMCasualPartition_struct EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em); }; -typedef EMCasualPartition_struct EMCasualPartition_t; +using EMCasualPartition_t = EMCasualPartition_struct; struct EMPartition_struct { @@ -165,9 +177,9 @@ struct EMEntry int fileID; uint32_t blockOffset; HWM_t HWM; - uint32_t partitionNum; // starts at 0 - uint16_t segmentNum; // starts at 0 - uint16_t dbRoot; // starts at 1 to match Columnstore.xml + PartitionNumberT partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + DBRootT dbRoot; // starts at 1 to match Columnstore.xml uint16_t colWid; int16_t status; // extent avail for query or not, or out of service EMPartition_t partition; @@ -319,6 +331,146 @@ class FreeListImpl static FreeListImpl* fInstance; }; +using ShmSegmentManagerT = bi::managed_shared_memory::segment_manager; +using ShmVoidAllocator = bi::allocator; + +using ExtentMapIdxT = size_t; +using ExtentMapIdxTAlloc = bi::allocator; +using PartitionNumberTAlloc = bi::allocator; +using ExtentMapIndicesT = std::vector; + +using PartitionIndexContainerKeyT = PartitionNumberT; +using PartitionIndexContainerValT = std::pair; +using PartitionIndexContainerValTAlloc = bi::allocator; +// Can't use std::unordered_map presumably b/c the map's pointer type doesn't use offset_type as boost::u_map +// does +using PartitionIndexContainerT = + boost::unordered_map, std::equal_to, + PartitionIndexContainerValTAlloc>; + +using OIDIndexContainerKeyT = OID_t; +using OIDIndexContainerValT = std::pair; +using OIDIndexContainerValTAlloc = bi::allocator; +using OIDIndexContainerT = + boost::unordered_map, + std::equal_to, OIDIndexContainerValTAlloc>; + +using DBRootIndexTAlloc = bi::allocator; +using DBRootIndexContainerT = std::vector; +using ExtentMapIndex = DBRootIndexContainerT; +using ExtentMapIndexFindResult = std::vector; +using InsertUpdateShmemKeyPair = std::pair; + +class ExtentMapIndexImpl +{ + public: + ~ExtentMapIndexImpl(){}; + + static ExtentMapIndexImpl* makeExtentMapIndexImpl(unsigned key, off_t size, bool readOnly = false); + static void refreshShm() + { + if (fInstance_) + { + delete fInstance_; + fInstance_ = nullptr; + } + } + + // The multipliers and constants here are pure theoretical + // tested using customer's data. + static size_t estimateEMIndexSize(uint32_t numberOfExtents) + { + // These are just educated guess values to calculate initial + // managed shmem size. + constexpr const size_t tablesNumber_ = 100ULL; + constexpr const size_t columnsNumber_ = 200ULL; + constexpr const size_t dbRootsNumber_ = 3ULL; + constexpr const size_t filesInPartition_ = 4ULL; + constexpr const size_t extentsInPartition_ = filesInPartition_ * 2; + return numberOfExtents * emIdentUnitSize_ + + numberOfExtents / extentsInPartition_ * partitionContainerUnitSize_ + + dbRootsNumber_ * tablesNumber_ * columnsNumber_; + } + + bool growIfNeeded(const size_t memoryNeeded); + + inline void grow(off_t size) + { + int rc = fBRMManagedShmMemImpl_.grow(size); + idbassert(rc == 0); + } + // After this call one needs to refresh any refs or ptrs sourced + // from this shmem. + inline void makeReadOnly() + { + fBRMManagedShmMemImpl_.setReadOnly(); + } + + inline void swapout(BRMManagedShmImpl& rhs) + { + fBRMManagedShmMemImpl_.swap(rhs); + } + + inline unsigned key() const + { + return fBRMManagedShmMemImpl_.key(); + } + + unsigned getShmemSize() + { + return fBRMManagedShmMemImpl_.getManagedSegment()->get_size(); + } + + size_t getShmemFree() + { + return fBRMManagedShmMemImpl_.getManagedSegment()->get_free_memory(); + } + + unsigned getShmemImplSize() + { + return fBRMManagedShmMemImpl_.size(); + } + + void createExtentMapIndexIfNeeded(); + ExtentMapIndex* get(); + InsertUpdateShmemKeyPair insert(const EMEntry& emEntry, const size_t emIdx); + InsertUpdateShmemKeyPair insert2ndLayerWrapper(OIDIndexContainerT& oids, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown); + InsertUpdateShmemKeyPair insert2ndLayer(OIDIndexContainerT& oids, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown); + InsertUpdateShmemKeyPair insert3dLayerWrapper(PartitionIndexContainerT& partitions, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown); + InsertUpdateShmemKeyPair insert3dLayer(PartitionIndexContainerT& partitions, const EMEntry& emEntry, + const size_t emIdx, const bool aShmemHasGrown); + ExtentMapIndexFindResult find(const DBRootT dbroot, const OID_t oid, + const PartitionNumberT partitionNumber); + ExtentMapIndexFindResult find(const DBRootT dbroot, const OID_t oid); + ExtentMapIndexFindResult search2ndLayer(OIDIndexContainerT& oids, const OID_t oid, + const PartitionNumberT partitionNumber); + ExtentMapIndexFindResult search2ndLayer(OIDIndexContainerT& oids, const OID_t oid); + ExtentMapIndexFindResult search3dLayer(PartitionIndexContainerT& partitions, + const PartitionNumberT partitionNumber); + void deleteDbRoot(const DBRootT dbroot); + void deleteOID(const DBRootT dbroot, const OID_t oid); + void deleteEMEntry(const EMEntry& emEntry, const ExtentMapIdxT emIdent); + + private: + BRMManagedShmImpl fBRMManagedShmMemImpl_; + ExtentMapIndexImpl(unsigned key, off_t size, bool readOnly = false); + ExtentMapIndexImpl(const ExtentMapIndexImpl& rhs); + ExtentMapIndexImpl& operator=(const ExtentMapIndexImpl& rhs); + + static std::mutex fInstanceMutex_; + static ExtentMapIndexImpl* fInstance_; + static const constexpr uint32_t dbRootContainerUnitSize_ = 64ULL; + static const constexpr uint32_t oidContainerUnitSize_ = 352ULL; // 2 * map overhead + static const constexpr uint32_t partitionContainerUnitSize_ = 368ULL; // single map overhead + static const constexpr uint32_t emIdentUnitSize_ = sizeof(uint64_t); + static const constexpr uint32_t extraUnits_ = 2; + static const constexpr size_t freeSpaceThreshold_ = 256 * 1024; +}; + /** @brief This class encapsulates the extent map functionality of the system * * This class encapsulates the extent map functionality of the system. It @@ -345,7 +497,7 @@ class ExtentMap : public Undoable */ EXPORT void load(const std::string& filename, bool fixFL = false); - /** @brief Loads the ExtentMap entries from a binayr blob. + /** @brief Loads the ExtentMap entries from a binary blob. * * Loads the ExtentMap entries from a file. This will * clear out any existing entries. The intention is that before @@ -886,6 +1038,9 @@ class ExtentMap : public Undoable EXPORT void dumpTo(std::ostream& os); EXPORT const bool* getEMLockStatus(); EXPORT const bool* getEMFLLockStatus(); + EXPORT const bool* getEMIndexLockStatus(); + size_t EMIndexShmemSize(); + size_t EMIndexShmemFree(); #ifdef BRM_DEBUG EXPORT void printEM() const; @@ -895,11 +1050,11 @@ class ExtentMap : public Undoable #endif private: - static const size_t EM_INCREMENT_ROWS = 100; - static const size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry); - static const size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry); - static const size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange); - static const size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange); + static const constexpr size_t EM_INCREMENT_ROWS = 100; + static const constexpr size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry); + static const constexpr size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry); + static const constexpr size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange); + static const constexpr size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange); ExtentMap(const ExtentMap& em); ExtentMap& operator=(const ExtentMap& em); @@ -910,6 +1065,7 @@ class ExtentMap : public Undoable key_t fCurrentFLShmkey; MSTEntry* fEMShminfo; MSTEntry* fFLShminfo; + MSTEntry* fEMIndexShminfo; const MasterSegmentTable fMST; bool r_only; typedef std::tr1::unordered_map PmDbRootMap_t; @@ -917,8 +1073,9 @@ class ExtentMap : public Undoable time_t fCacheTime; // timestamp associated with config cache int numUndoRecords; - bool flLocked, emLocked; - static boost::mutex mutex; // @bug5355 - made mutex static + bool flLocked, emLocked, emIndexLocked; + static boost::mutex mutex; // @bug5355 - made mutex static + static boost::mutex emIndexMutex; boost::mutex fConfigCacheMutex; // protect access to Config Cache enum OPS @@ -930,6 +1087,12 @@ class ExtentMap : public Undoable OPS EMLock, FLLock; + LastIndEmptyIndEmptyInd _createExtentCommonSearch(const OID_t OID, const DBRootT dbRoot, + const PartitionNumberT partitionNum, + const SegmentT segmentNum); + + void logAndSetEMIndexReadOnly(const std::string& funcName); + LBID_t _createColumnExtent_DBroot(uint32_t size, int OID, uint32_t colWidth, uint16_t dbRoot, execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& startBlockOffset); @@ -941,24 +1104,32 @@ class ExtentMap : public Undoable uint16_t segmentNum); template bool isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const; - void deleteExtent(int emIndex); + void deleteExtent(const int emIndex, const bool clearEMIndex = true); LBID_t getLBIDsFromFreeList(uint32_t size); void reserveLBIDRange(LBID_t start, uint8_t size); // used by load() to allocate pre-existing LBIDs - key_t chooseEMShmkey(); // see the code for how keys are segmented - key_t chooseFLShmkey(); // see the code for how keys are segmented + key_t chooseEMShmkey(); + key_t chooseFLShmkey(); + key_t chooseEMIndexShmkey(); + key_t getInitialEMIndexShmkey() const; + // see the code for how keys are segmented + key_t chooseShmkey(const MSTEntry* masterTableEntry, const uint32_t keyRangeBase) const; void grabEMEntryTable(OPS op); void grabFreeList(OPS op); + void grabEMIndex(OPS op); void releaseEMEntryTable(OPS op); void releaseFreeList(OPS op); + void releaseEMIndex(OPS op); void growEMShmseg(size_t nrows = 0); void growFLShmseg(); + void growEMIndexShmseg(const size_t suggestedSize = 0); void finishChanges(); EXPORT unsigned getFilesPerColumnPartition(); unsigned getExtentsPerSegmentFile(); unsigned getDbRootCount(); void getPmDbRoots(int pm, std::vector& dbRootList); + DBRootVec getAllDbRoots(); void checkReloadConfig(); ShmKeys fShmKeys; @@ -979,6 +1150,7 @@ class ExtentMap : public Undoable ExtentMapImpl* fPExtMapImpl; FreeListImpl* fPFreeListImpl; + ExtentMapIndexImpl* fPExtMapIndexImpl_; }; inline std::ostream& operator<<(std::ostream& os, ExtentMap& rhs) diff --git a/versioning/BRM/lock_grabber.cpp b/versioning/BRM/lock_grabber.cpp index 6932b7708..7c6884bb8 100644 --- a/versioning/BRM/lock_grabber.cpp +++ b/versioning/BRM/lock_grabber.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -21,6 +22,7 @@ * third, lock or unlock it */ +#include #include #include #include @@ -32,10 +34,15 @@ char* name; void usage() { - cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n" - << " which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n" - << " which_side_to_use: r|w (read or write)\n" - << " lock_or_unlock: l|u (lock or unlock)\n"; + std::cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock" << std::endl; + size_t lockId = 0; + for (auto& lockName : RWLockNames) + { + std::cout << " " << lockId++ << "=" << lockName << " "; + } + std::cout << std::endl + << " which_side_to_use: r|w (read or write)" << std::endl + << " lock_or_unlock: l|u (lock or unlock)" << std::endl; exit(1); } @@ -54,10 +61,21 @@ int main(int argc, char** argv) if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1) usage(); - which_lock = atoi(argv[1]); - - if (which_lock < 1 || which_lock > 5) + try + { + which_lock = std::stoi(argv[1]); + } + catch (std::exception const& e) + { + std::cerr << "Cannot convert the lock id: " << e.what() << std::endl; usage(); + } + + if (which_lock >= RWLockNames.size()) + usage(); + + size_t minLockId = (which_lock > 0) ? which_lock : 1; + size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1; if (argv[2][0] == 'r') which_side = 0; @@ -73,17 +91,28 @@ int main(int argc, char** argv) else usage(); - rwlock = new RWLock(0x10000 * which_lock); + for (size_t i = minLockId; i <= maxLockId; ++i) + { + rwlock = new RWLock(0x10000 * which_lock); - if (which_side == 0) - if (lock_unlock == 0) - rwlock->read_lock(); + if (which_side == 0) + { + if (lock_unlock == 0) + rwlock->read_lock(); + else + rwlock->read_unlock(); + } + else if (lock_unlock == 0) + { + rwlock->write_lock(); + } else - rwlock->read_unlock(); - else if (lock_unlock == 0) - rwlock->write_lock(); - else - rwlock->write_unlock(); + { + rwlock->write_unlock(); + } + + delete rwlock; + } return 0; } diff --git a/versioning/BRM/lock_state.cpp b/versioning/BRM/lock_state.cpp index 812b59c08..4b5cef63e 100644 --- a/versioning/BRM/lock_state.cpp +++ b/versioning/BRM/lock_state.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2022 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -22,6 +23,7 @@ */ #include +#include #include #include @@ -32,14 +34,18 @@ char* name; void usage() { - cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n" - << " which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n"; + std::cout << "Usage " << name << " which_lock_to_use:" << std::endl; + size_t lockId = 0; + for (auto& lockName : RWLockNames) + { + std::cout << " " << lockId++ << "=" << lockName << std::endl; + } exit(1); } int main(int argc, char** argv) { - uint32_t which_lock; // 1-5 + uint32_t which_lock; // 0-6 RWLock* rwlock; LockState state; @@ -51,18 +57,35 @@ int main(int argc, char** argv) if (strlen(argv[1]) != 1) usage(); - which_lock = atoi(argv[1]); + try + { + which_lock = std::stoi(argv[1]); + } + catch (std::exception const& e) + { + std::cerr << "Cannot convert the lock id: " << e.what() << std::endl; + usage(); + } - if (which_lock < 1 || which_lock > 5) + if (which_lock >= RWLockNames.size()) usage(); - rwlock = new RWLock(0x10000 * which_lock); - state = rwlock->getLockState(); - cout << "readers = " << state.reading << endl - << "writers = " << state.writing << endl - << "readers waiting = " << state.readerswaiting << endl - << "writers waiting = " << state.writerswaiting << endl - << "mutex locked = " << (int)state.mutexLocked << endl; + size_t minLockId = (which_lock > 0) ? which_lock : 1; + size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1; + + for (size_t i = minLockId; i <= maxLockId; ++i) + { + rwlock = new RWLock(0x10000 * i); + state = rwlock->getLockState(); + + cout << RWLockNames[i] << " RWLock" << std::endl + << " readers = " << state.reading << std::endl + << " writers = " << state.writing << std::endl + << " readers waiting = " << state.readerswaiting << std::endl + << " writers waiting = " << state.writerswaiting << std::endl + << " mutex locked = " << (int)state.mutexLocked << std::endl; + delete rwlock; + } return 0; } diff --git a/versioning/BRM/mastersegmenttable.cpp b/versioning/BRM/mastersegmenttable.cpp index c88dc13ba..31b2da6f7 100644 --- a/versioning/BRM/mastersegmenttable.cpp +++ b/versioning/BRM/mastersegmenttable.cpp @@ -138,6 +138,7 @@ MasterSegmentTable::MasterSegmentTable() RWLockKeys[2] = fShmKeys.KEYRANGE_VBBM_BASE; RWLockKeys[3] = fShmKeys.KEYRANGE_VSS_BASE; RWLockKeys[4] = fShmKeys.KEYRANGE_CL_BASE; + RWLockKeys[5] = fShmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE; try { diff --git a/versioning/BRM/mastersegmenttable.h b/versioning/BRM/mastersegmenttable.h index 25580490f..be7b96639 100644 --- a/versioning/BRM/mastersegmenttable.h +++ b/versioning/BRM/mastersegmenttable.h @@ -109,8 +109,10 @@ class MasterSegmentTable static const int VSSSegment = 3; /// specifies the copy lock segment static const int CLSegment = 4; + /// specifies the EM Index segment + static const int EMIndex = 5; /// the number of tables currently defined - static const int nTables = 5; + static const int nTables = 6; /** @brief This function gets the specified table. * diff --git a/versioning/BRM/shmkeys.cpp b/versioning/BRM/shmkeys.cpp index f7642ff9f..943d81d1b 100644 --- a/versioning/BRM/shmkeys.cpp +++ b/versioning/BRM/shmkeys.cpp @@ -50,6 +50,7 @@ ShmKeys::ShmKeys() KEYRANGE_EMFREELIST_BASE = 0x30000 | (BRM_UID << 20); KEYRANGE_VBBM_BASE = 0x40000 | (BRM_UID << 20); KEYRANGE_CL_BASE = 0x50000 | (BRM_UID << 20); + KEYRANGE_EXTENTMAP_INDEX_BASE = 0x60000 | (BRM_UID << 20); MST_SYSVKEY = 0xff000000 | BRM_UID; PROCESSSTATUS_SYSVKEY = 0xfd000000 | BRM_UID; SYSTEMSTATUS_SYSVKEY = 0xfc000000 | BRM_UID; @@ -62,7 +63,7 @@ ShmKeys::ShmKeys() string ShmKeys::keyToName(unsigned key) { ostringstream oss; - oss << "InfiniDB-shm-"; + oss << "MCS-shm-"; oss << setw(8) << setfill('0') << hex << key; return oss.str(); } diff --git a/versioning/BRM/shmkeys.h b/versioning/BRM/shmkeys.h index 6b4ef9999..ce7f00d1a 100644 --- a/versioning/BRM/shmkeys.h +++ b/versioning/BRM/shmkeys.h @@ -55,6 +55,7 @@ struct ShmKeys uint32_t KEYRANGE_EMFREELIST_BASE; uint32_t KEYRANGE_VBBM_BASE; uint32_t KEYRANGE_VSS_BASE; + uint32_t KEYRANGE_EXTENTMAP_INDEX_BASE; /****** Fixed location assignments *******/ uint32_t MST_SYSVKEY; diff --git a/versioning/BRM/slavedbrmnode.cpp b/versioning/BRM/slavedbrmnode.cpp index ae7bb1d11..2331169a9 100644 --- a/versioning/BRM/slavedbrmnode.cpp +++ b/versioning/BRM/slavedbrmnode.cpp @@ -1492,6 +1492,11 @@ const bool* SlaveDBRMNode::getEMLockStatus() return em.getEMLockStatus(); } +const bool* SlaveDBRMNode::getEMIndexLockStatus() +{ + return em.getEMIndexLockStatus(); +} + const bool* SlaveDBRMNode::getVBBMLockStatus() { return &locked[0]; diff --git a/versioning/BRM/slavedbrmnode.h b/versioning/BRM/slavedbrmnode.h index f614c8ce5..e53eb4936 100644 --- a/versioning/BRM/slavedbrmnode.h +++ b/versioning/BRM/slavedbrmnode.h @@ -461,6 +461,7 @@ class SlaveDBRMNode EXPORT const bool* getEMFLLockStatus(); EXPORT const bool* getEMLockStatus(); + EXPORT const bool* getEMIndexLockStatus(); EXPORT const bool* getVBBMLockStatus(); EXPORT const bool* getVSSLockStatus(); diff --git a/versioning/BRM/slavenode.cpp b/versioning/BRM/slavenode.cpp index 8651dc823..667feaa08 100644 --- a/versioning/BRM/slavenode.cpp +++ b/versioning/BRM/slavenode.cpp @@ -147,6 +147,8 @@ int ServiceWorkerNode::Child() monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE)); monitorThreads.create_thread(RWLockMonitor(&die, slave.getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE)); monitorThreads.create_thread(RWLockMonitor(&die, slave.getVSSLockStatus(), keys.KEYRANGE_VSS_BASE)); + monitorThreads.create_thread( + RWLockMonitor(&die, slave.getEMIndexLockStatus(), keys.KEYRANGE_EXTENTMAP_INDEX_BASE)); try { diff --git a/writeengine/bulk/cpimport.cpp b/writeengine/bulk/cpimport.cpp index 08e8871be..54aeaa2c5 100644 --- a/writeengine/bulk/cpimport.cpp +++ b/writeengine/bulk/cpimport.cpp @@ -916,16 +916,24 @@ void getTableOID(const std::string& xmlGenSchema, const std::string& xmlGenTable void constructTempXmlFile(const std::string& tempJobDir, const std::string& sJobIdStr, const std::string& xmlGenSchema, const std::string& xmlGenTable, const std::string& alternateImportDir, const std::string& S3Bucket, - boost::filesystem::path& sFileName) + const std::string& tableOIDStr, boost::filesystem::path& sFileName) { // Construct the job description file name std::string xmlErrMsg; int rc = 0; - std::string tableOIDStr; - getTableOID(xmlGenSchema, xmlGenTable, tableOIDStr); + std::string localTableOIDStr; + if (tableOIDStr.empty()) + { + getTableOID(xmlGenSchema, xmlGenTable, localTableOIDStr); + } + else + { + localTableOIDStr = tableOIDStr; + } + rc = XMLJob::genJobXMLFileName(std::string(), tempJobDir, sJobIdStr, true, // using temp job xml file - xmlGenSchema, xmlGenTable, sFileName, xmlErrMsg, tableOIDStr); + xmlGenSchema, xmlGenTable, sFileName, xmlErrMsg, localTableOIDStr); if (rc != NO_ERROR) { @@ -945,7 +953,7 @@ void constructTempXmlFile(const std::string& tempJobDir, const std::string& sJob { genProc.startXMLFile(); execplan::CalpontSystemCatalog::TableName tbl(xmlGenSchema, xmlGenTable); - genProc.makeTableData(tbl); + genProc.makeTableData(tbl, localTableOIDStr); if (!genProc.makeColumnData(tbl)) { @@ -1222,9 +1230,9 @@ int main(int argc, char** argv) if (!xmlGenSchema.empty()) // create temporary job file name { // If JobID is not provided, then default to the table OID + std::string tableOIDStr{""}; if (sJobIdStr.empty()) { - std::string tableOIDStr; getTableOID(xmlGenSchema, xmlGenTable, tableOIDStr); if (!(BulkLoad::disableConsoleOutput())) @@ -1239,7 +1247,7 @@ int main(int argc, char** argv) bUseTempJobFile = true; constructTempXmlFile(curJob.getTempJobDir(), sJobIdStr, xmlGenSchema, xmlGenTable, - curJob.getAlternateImportDir(), curJob.getS3Bucket(), sFileName); + curJob.getAlternateImportDir(), curJob.getS3Bucket(), tableOIDStr, sFileName); } else // create user's persistent job file name { diff --git a/writeengine/xml/we_xmlgenproc.cpp b/writeengine/xml/we_xmlgenproc.cpp index 3fca53e50..afd6d2b6c 100644 --- a/writeengine/xml/we_xmlgenproc.cpp +++ b/writeengine/xml/we_xmlgenproc.cpp @@ -168,7 +168,21 @@ void XMLGenProc::startXMLFile() // makeTableData // Create XML tag for a table. //------------------------------------------------------------------------------ +// This method is used by colxml only and it can be relatively slower doing tableRID() +// first call. All subsequent calls will re-use data from CalpontSystemCatalog cache. void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table) +{ + boost::shared_ptr cat = + CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); + cat->identity(CalpontSystemCatalog::EC); + std::ostringstream oss; + // tableRID method might take a lot with a significant EM. + oss << cat->tableRID(table).objnum; + + makeTableData(table, oss.str()); +} + +void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table, const std::string& tableOIDStr) { static unsigned kount; @@ -180,11 +194,8 @@ void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table) { try { - boost::shared_ptr cat = - CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID); - cat->identity(CalpontSystemCatalog::EC); - xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_TBL_OID], "%d", - cat->tableRID(table).objnum); + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_TBL_OID], "%s", + tableOIDStr.c_str()); } catch (std::exception& ex) { diff --git a/writeengine/xml/we_xmlgenproc.h b/writeengine/xml/we_xmlgenproc.h index 2b880329e..2da7fc925 100644 --- a/writeengine/xml/we_xmlgenproc.h +++ b/writeengine/xml/we_xmlgenproc.h @@ -75,6 +75,9 @@ class XMLGenProc * * @param table Name of table for which the table tag is to be generated. */ + EXPORT void makeTableData(const execplan::CalpontSystemCatalog::TableName& table, + const std::string& tableOIDStr); + EXPORT void makeTableData(const execplan::CalpontSystemCatalog::TableName& table); /** @brief Creates column tags for the specified table. diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index f4f5662d7..2a95203df 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -381,11 +381,11 @@ void XMLJob::setJobData(xmlNode* pNode, const xmlTag tag, bool bExpectContent, X if (tagType == TYPE_INT) bSuccess = getNodeContent(pNode, &intVal, TYPE_INT); else // longlong - if (tagType == TYPE_LONGLONG) - bSuccess = getNodeContent(pNode, &llVal, TYPE_LONGLONG); - else // char + if (tagType == TYPE_LONGLONG) + bSuccess = getNodeContent(pNode, &llVal, TYPE_LONGLONG); + else // char if (tagType == TYPE_CHAR) - bSuccess = getNodeContentStr(pNode, bufString); + bSuccess = getNodeContentStr(pNode, bufString); if (!bSuccess) return; @@ -1194,7 +1194,8 @@ void XMLJob::validateAllColumnsHaveTags(const execplan::CalpontSystemCatalog::RI /* static */ int XMLJob::genJobXMLFileName(const string& sXMLJobDir, const string& jobDir, const string& jobId, bool bTempFile, const string& schemaName, const string& tableName, - boost::filesystem::path& xmlFilePath, string& errMsg, std::string& tableOIDStr) + boost::filesystem::path& xmlFilePath, string& errMsg, + const std::string& tableOIDStr) { // get full file directory path for XML job description file if (sXMLJobDir.empty()) diff --git a/writeengine/xml/we_xmljob.h b/writeengine/xml/we_xmljob.h index ba33cd4a7..0910bea0b 100644 --- a/writeengine/xml/we_xmljob.h +++ b/writeengine/xml/we_xmljob.h @@ -74,7 +74,7 @@ class XMLJob : public XMLOp EXPORT static int genJobXMLFileName(const std::string& sXMLJobDir, const std::string& jobDir, const std::string& jobId, bool bTempFile, const std::string& schemaName, const std::string& tableName, boost::filesystem::path& xmlDirPath, - std::string& errMsg, std::string& tableOIDStr); + std::string& errMsg, const std::string& tableOIDStr); /** * @brief Get job structure From 6aadc6cc17326e03d02767a556e56c796cab2973 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Fri, 29 Apr 2022 01:20:56 +0000 Subject: [PATCH 49/55] boost::interprocess::vector instead of std::vector --- versioning/BRM/extentmap.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index 805a37c35..8452d822e 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -36,12 +36,13 @@ //#define NDEBUG #include -#include -#include -#include -#include -#include #include //boost::hash +#include +#include +#include +#include +#include +#include #include "shmkeys.h" #include "brmtypes.h" @@ -337,7 +338,7 @@ using ShmVoidAllocator = bi::allocator; using ExtentMapIdxT = size_t; using ExtentMapIdxTAlloc = bi::allocator; using PartitionNumberTAlloc = bi::allocator; -using ExtentMapIndicesT = std::vector; +using ExtentMapIndicesT = bi::vector; using PartitionIndexContainerKeyT = PartitionNumberT; using PartitionIndexContainerValT = std::pair; @@ -357,9 +358,9 @@ using OIDIndexContainerT = std::equal_to, OIDIndexContainerValTAlloc>; using DBRootIndexTAlloc = bi::allocator; -using DBRootIndexContainerT = std::vector; +using DBRootIndexContainerT = bi::vector; using ExtentMapIndex = DBRootIndexContainerT; -using ExtentMapIndexFindResult = std::vector; +using ExtentMapIndexFindResult = bi::vector; using InsertUpdateShmemKeyPair = std::pair; class ExtentMapIndexImpl From d21834d103838787c888c67400b6e9e458973d08 Mon Sep 17 00:00:00 2001 From: mariadb-RomanNavrotskiy Date: Wed, 4 May 2022 13:36:13 +0200 Subject: [PATCH 50/55] ci: upd dockerfile repo --- .drone.jsonnet | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 8329b8a13..0f0e7b0a4 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -282,8 +282,8 @@ local Pipeline(branch, platform, event, arch='amd64') = { name: 'dockerfile', image: 'alpine/git', commands: [ - 'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-enterprise-columnstore-docker', - "sed -i 's|dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup|cspkg.s3.amazonaws.com/cs_repo|' mariadb-enterprise-columnstore-docker/Dockerfile", + 'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-skysql-columnstore-docker docker', + "sed -i 's|dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup|cspkg.s3.amazonaws.com/cs_repo|' docker/Dockerfile", ], }, dockerhub:: { @@ -294,8 +294,8 @@ local Pipeline(branch, platform, event, arch='amd64') = { }, settings: { repo: 'mariadb/enterprise-columnstore-dev', - context: 'mariadb-enterprise-columnstore-docker', - dockerfile: 'mariadb-enterprise-columnstore-docker/Dockerfile', + context: 'docker', + dockerfile: 'docker/Dockerfile', build_args_from_env: ['VERSION'], tags: container_tags, username: { From 0489682f9d411488dcf49850f24ae5705b0c8de8 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Thu, 5 May 2022 16:44:23 +0000 Subject: [PATCH 51/55] Use boost tarball instead of clone --- cmake/boost.CMakeLists.txt.in | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cmake/boost.CMakeLists.txt.in b/cmake/boost.CMakeLists.txt.in index fa7c975fe..899c31152 100644 --- a/cmake/boost.CMakeLists.txt.in +++ b/cmake/boost.CMakeLists.txt.in @@ -15,19 +15,18 @@ set(INSTALL_LOCATION ${CMAKE_BINARY_DIR}/boost-lib) set(_cxxargs "-fPIC -DBOOST_NO_AUTO_PTR -fvisibility=default") set(_b2args cxxflags=${_cxxargs};cflags=-fPIC;threading=multi; toolset=${_toolset} --without-python;--prefix=${INSTALL_LOCATION}) -ExternalProject_Add(boost - PREFIX build - GIT_REPOSITORY https://github.com/boostorg/boost.git - GIT_TAG boost-1.78.0 - DOWNLOAD_NO_PROGRESS TRUE +ExternalProject_Add(external_boost + PREFIX boost + URL https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 + URL_HASH SHA256=475d589d51a7f8b3ba2ba4eda022b170e562ca3b760ee922c146b6c65856ef39 + CONFIGURE_COMMAND ./bootstrap.sh UPDATE_COMMAND "" - CONFIGURE_COMMAND /bootstrap.sh - BUILD_COMMAND /b2 -q ${_b2args} + BUILD_COMMAND ./b2 -q ${_b2args} LOG_BUILD TRUE BUILD_IN_SOURCE TRUE - INSTALL_COMMAND /b2 -q install ${_b2args} + INSTALL_COMMAND ./b2 -q install ${_b2args} LOG_INSTALL TRUE ) unset(_b2args) -unset(_toolset) \ No newline at end of file +unset(_toolset) From e147184b8df8cdf9cdf826d2b5d6aab80c94443f Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 10 May 2022 12:32:17 -0500 Subject: [PATCH 52/55] MCOL-5065: return values of getSystemReady/getSystemQueryReady should be > 0 (#2354) --- dbcon/mysql/ha_mcs_client_udfs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 35e8ff664..bdaec9ed4 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -381,7 +381,7 @@ extern "C" try { - if (dbrm.getSystemReady() && dbrm.getSystemQueryReady()) + if (dbrm.getSystemReady() > 0 && dbrm.getSystemQueryReady() > 0) { return 1; } From 8bf4e0eaa5298e235d3606987736e5d4e6383fa8 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 14 Apr 2022 11:53:09 +0000 Subject: [PATCH 53/55] MCOL-5050 EM Index dbroot container out-of-bound access checks doing EM deletes --- versioning/BRM/extentmap.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 0f8d32a88..fab238286 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -527,12 +527,19 @@ ExtentMapIndexFindResult ExtentMapIndexImpl::search3dLayer(PartitionIndexContain void ExtentMapIndexImpl::deleteDbRoot(const DBRootT dbroot) { auto& extMapIndex = *get(); + if (dbroot >= extMapIndex.size()) + return; // nothing to delete extMapIndex[dbroot].clear(); } void ExtentMapIndexImpl::deleteOID(const DBRootT dbroot, const OID_t oid) { auto& extMapIndex = *get(); + // nothing to delete + if (dbroot >= extMapIndex.size()) + return; + if (extMapIndex[dbroot].empty()) + return; auto oidsIter = extMapIndex[dbroot].find(oid); // Nothing to delete. Might be a sign of a problem. if (oidsIter == extMapIndex[dbroot].end()) @@ -544,6 +551,10 @@ void ExtentMapIndexImpl::deleteEMEntry(const EMEntry& emEntry, const ExtentMapId { // find partition auto& extMapIndex = *get(); + if (emEntry.dbRoot >= extMapIndex.size()) + return; + if (extMapIndex[emEntry.dbRoot].empty()) + return; auto oidsIter = extMapIndex[emEntry.dbRoot].find(emEntry.fileID); if (oidsIter == extMapIndex[emEntry.dbRoot].end()) return; @@ -1646,7 +1657,6 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) } growEMShmseg(nrows); - growEMIndexShmseg(ExtentMapIndexImpl::estimateEMIndexSize(emNumElements)); } size_t progress = 0, writeSize = emNumElements * sizeof(EMEntry); From 94ca6fdd345c82a73da7dc2845f4ba2a9651152f Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Wed, 18 May 2022 13:36:52 +0300 Subject: [PATCH 54/55] [MCOL-5001] Explicitly initialize `startupRaceFlag`. Explicitly initialize `startupRaceFlag_` to avoid PrimProc freezing on some platforms. --- primitives/primproc/primproc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 4d761899a..5f9bfc463 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -124,7 +124,7 @@ class ServicePrimProc : public Service, public Opt private: // Since C++20 flag's init value is false. - std::atomic_flag startupRaceFlag_; + std::atomic_flag startupRaceFlag_{false}; }; namespace primitiveprocessor From 0dab97da7dbab84b2868fc9557540186d00efc31 Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Thu, 19 May 2022 15:48:23 +0300 Subject: [PATCH 55/55] Actual fix for percentile window function We fix 'partial out-of-bounds reference' warning uncovered when building with gcc11. --- utils/windowfunction/wf_percentile.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 24d6b9f28..ef2c12548 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -335,7 +335,10 @@ void WF_percentile::operator()(int64_t b, int64_t e, int64_t c) vd = (crn - rn) * fv + (rn - frn) * cv; } - v = *(reinterpret_cast(&vd)); + double tempvd[2]; + tempvd[0] = vd; + tempvd[1] = 0; + v = *(reinterpret_cast(&tempvd[0])); // old code that referred to 'vd' var triggered partial out-of-bounds warning. p = &v; } else // (fFunctionId == WF__PERCENTILE_DISC)