From e3cd20538887c037faea1f93cdda32763f7fc7fb Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 11 Jun 2019 10:37:04 +0100 Subject: [PATCH 1/3] MCOL-1968 Fix UTF char/varchar min/max handling If the first byte of a char/varchar was > 0x80 then it will break the min/max values for an extent during cpimport. This patch makes the min/max compare unsigned and only switches to signed when storing. In addition send all the LDI / INSERT...SELECT data to cpimport, not truncated. Let cpimport figure out the truncation point. --- dbcon/mysql/ha_calpont_dml.cpp | 18 ++++++++++-------- writeengine/bulk/we_bulkloadbuffer.cpp | 11 +++++++---- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index c2dc36e47..a9404c223 100755 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -860,7 +860,13 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ } case CalpontSystemCatalog::VARCHAR: { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + size_t length; + if (ci.utf8) + length = (ci.columnTypes[colpos].colWidth * 3); + else + length = ci.columnTypes[colpos].colWidth; + + if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) { fprintf(ci.filePtr, "%c", ci.delimiter); if (!ci.utf8) @@ -901,6 +907,7 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ dataLength = *(uint16_t*) buf; buf = buf + 2 ; } + length = dataLength; escape.assign((char*)buf, dataLength); boost::replace_all(escape, "\\", "\\\\"); fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); @@ -917,8 +924,7 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ dataLength = *(uint16_t*) buf; buf = buf + 2 ; } - if ( dataLength > ci.columnTypes[colpos].colWidth) - dataLength = ci.columnTypes[colpos].colWidth; + length = dataLength; escape.assign((char*)buf, dataLength); boost::replace_all(escape, "\\", "\\\\"); @@ -926,11 +932,7 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); } } - //buf += ci.columnTypes[colpos].colWidth; - if (ci.utf8) - buf += (ci.columnTypes[colpos].colWidth * 3); - else - buf += ci.columnTypes[colpos].colWidth; + buf += length; break; } diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index b1902e98f..7326f987b 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -524,13 +524,16 @@ void BulkLoadBuffer::convert(char *field, int fieldLength, } // Swap byte order before comparing character string - int64_t binChar = static_cast( uint64ToStr( - *(reinterpret_cast(charTmpBuf)) ) ); + // Compare must be unsigned + uint64_t compChar = uint64ToStr( *(reinterpret_cast(charTmpBuf)) ); + int64_t binChar = static_cast( compChar ); // Update min/max range - if (binChar < bufStats.minBufferVal) + uint64_t minVal = static_cast( bufStats.minBufferVal ); + uint64_t maxVal = static_cast( bufStats.maxBufferVal ); + if (compChar < minVal) bufStats.minBufferVal = binChar; - if (binChar > bufStats.maxBufferVal) + if (compChar > maxVal) bufStats.maxBufferVal = binChar; pVal = charTmpBuf; From 7d22a5945c68456029e10c44ff402b96d8b25836 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 14 Jun 2019 15:52:30 +0100 Subject: [PATCH 2/3] MCOL-1989 Fix view in view subquery outer join A view calling a view as part of a subquery outer join was not getting the view name for the derived table columns. Which caused ColumnStore to think it was joining outside of the view and triggered a missing column error. This fix adds the view name from the subquery if one cannot be obtained from the field object. --- dbcon/execplan/calpontselectexecutionplan.h | 5 +++++ dbcon/mysql/ha_calpont_execplan.cpp | 9 ++++++++- dbcon/mysql/ha_from_sub.cpp | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dbcon/execplan/calpontselectexecutionplan.h b/dbcon/execplan/calpontselectexecutionplan.h index f77d5c0f7..cacb9d3f5 100644 --- a/dbcon/execplan/calpontselectexecutionplan.h +++ b/dbcon/execplan/calpontselectexecutionplan.h @@ -354,6 +354,10 @@ public: void derivedTbAlias(const std::string derivedTbAlias) { fDerivedTbAlias = derivedTbAlias; } const std::string derivedTbAlias() const { return fDerivedTbAlias; } + void derivedTbView(const std::string derivedTbView) { fDerivedTbView = derivedTbView; } + const std::string derivedTbView() const { return fDerivedTbView; } + + void limitStart(const uint64_t limitStart) { fLimitStart = limitStart; } const uint64_t limitStart() const { return fLimitStart; } @@ -565,6 +569,7 @@ private: // for subselect uint64_t fSubType; std::string fDerivedTbAlias; + std::string fDerivedTbView; // for limit uint64_t fLimitStart; diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 2383f7ff7..d0e96c412 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -1833,7 +1833,14 @@ SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) sc->colPosition(j); string tableAlias(csep->derivedTbAlias()); sc->tableAlias(lower(tableAlias)); - sc->viewName(lower(viewName)); + if (!viewName.empty()) + { + sc->viewName(viewName); + } + else + { + sc->viewName(csep->derivedTbView()); + } sc->resultType(cols[j]->resultType()); sc->hasAggregate(cols[j]->hasAggregate()); if (col) diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 0dd9eeb0e..a12221fc5 100755 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -317,6 +317,7 @@ SCSEP FromSubQuery::transform() gwi.subQuery = this; gwi.viewName = fGwip.viewName; csep->derivedTbAlias(fAlias); // always lower case + csep->derivedTbView(fGwip.viewName.alias); if (getSelectPlan(gwi, *fFromSub, csep) != 0) { From b05c4e8df4e1f3a2bdbbd14896bce658b074d0fb Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 18 Jun 2019 16:13:50 +0100 Subject: [PATCH 3/3] MCOL-2243 Only set length and dec when dec is zero Length and dec for columns only needs to be set when it is the result of a function. Otherwise it could lead to odd side-effects in MariaDB. --- dbcon/mysql/ha_calpont_impl.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index a6cbea6a9..eb3b85384 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -579,7 +579,12 @@ int fetchNextRow(uchar *buf, cal_table_info& ti, cal_connection_info* ci) // bug 3485, reserve enough space for the longest float value // -3.402823466E+38 to -1.175494351E-38, 0, and // 1.175494351E-38 to 3.402823466E+38. - (*f)->field_length = 40; + if (!f2->dec) + { + (*f)->field_length = 40; + f2->dec = row.getScale(s); + } + //float float_val = *(float*)(&value); //f2->store(float_val); if (f2->decimals() < (uint32_t)row.getScale(s)) @@ -602,7 +607,12 @@ int fetchNextRow(uchar *buf, cal_table_info& ti, cal_connection_info* ci) // bug 3483, reserve enough space for the longest double value // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and // 2.2250738585072014E-308 to 1.7976931348623157E+308. - (*f)->field_length = 310; + if (!f2->dec) + { + (*f)->field_length = 310; + f2->dec = row.getScale(s); + } + //double double_val = *(double*)(&value); //f2->store(double_val); if (f2->decimals() < (uint32_t)row.getScale(s))