From 7b5845a4aa8f91b7bd5bd3708d7b3a49de2c4365 Mon Sep 17 00:00:00 2001
From: Roman Nozdrin <rnozdrin@mariadb.com>
Date: Fri, 17 Dec 2021 12:40:37 +0000
Subject: [PATCH] MCOL-4871 Bar's patch to do proper extent elimination for
 short CHAR

---
 dbcon/mysql/is_columnstore_extents.cpp |  4 +--
 primitives/linux-port/column.cpp       | 45 ++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp
index 38e3ac262..6589c6b37 100644
--- a/dbcon/mysql/is_columnstore_extents.cpp
+++ b/dbcon/mysql/is_columnstore_extents.cpp
@@ -90,7 +90,7 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th
                 else
                 {
                     table->field[4]->set_notnull();
-                    table->field[4]->store(iter->partition.cprange.loVal);
+                    table->field[4]->store((longlong) iter->partition.cprange.loVal, false);
                 }
 
                 if (iter->partition.cprange.hiVal <= (std::numeric_limits<int64_t>::min() + 1))
@@ -100,7 +100,7 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th
                 else
                 {
                     table->field[5]->set_notnull();
-                    table->field[5]->store(iter->partition.cprange.hiVal);
+                    table->field[5]->store((longlong) iter->partition.cprange.hiVal, false);
                 }
             }
             else
diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp
index 5d07b0f81..64e4da1a9 100644
--- a/primitives/linux-port/column.cpp
+++ b/primitives/linux-port/column.cpp
@@ -879,6 +879,47 @@ inline void updateMinMax(T& Min, T& Max, const T curValue, NewColRequestHeader*
         Max = curValue;
 }
 
+// The next templates group sets initial Min/Max values in filterColumnData.
+template<ENUM_KIND KIND, typename T,
+         typename std::enable_if<KIND == KIND_TEXT, T>::type* = nullptr>
+T getInitialMin(NewColRequestHeader* in)
+{
+    const CHARSET_INFO &cs = in->colType.getCharset();
+    T Min = 0;
+    cs.max_str((uchar*) &Min, sizeof(Min), sizeof(Min));
+    return Min;
+}
+
+template<ENUM_KIND KIND, typename T,
+         typename std::enable_if<KIND != KIND_TEXT, T>::type* = nullptr>
+T getInitialMin(NewColRequestHeader* in)
+{
+    return datatypes::numeric_limits<T>::max();
+}
+
+template<ENUM_KIND KIND, typename T,
+         typename std::enable_if<KIND != KIND_TEXT && KIND != KIND_UNSIGNED, T>::type* = nullptr>
+T getInitialMax(NewColRequestHeader* in)
+{
+    return datatypes::numeric_limits<T>::min();
+}
+
+template<ENUM_KIND KIND, typename T,
+         typename std::enable_if<KIND == KIND_UNSIGNED, T>::type* = nullptr>
+T getInitialMax(NewColRequestHeader* in)
+{
+    return 0;
+}
+
+template<ENUM_KIND KIND, typename T,
+         typename std::enable_if<KIND == KIND_TEXT, T>::type* = nullptr>
+T getInitialMax(NewColRequestHeader* in)
+{
+    const CHARSET_INFO &cs = in->colType.getCharset();
+    T Max = 0;
+    cs.min_str((uchar*) &Max, sizeof(Max), sizeof(Max));
+    return Max;
+}
 
 /*****************************************************************************
  *** READ COLUMN VALUES ******************************************************
@@ -1667,8 +1708,8 @@ void filterColumnData(
     // ###########################
     // Boolean indicating whether to capture the min and max values
     bool validMinMax = isMinMaxValid(in);
-    T Min = datatypes::numeric_limits<T>::max();
-    T Max = (KIND == KIND_UNSIGNED) ? 0 : datatypes::numeric_limits<T>::min();
+    T Min = getInitialMin<KIND,T>(in);
+    T Max = getInitialMax<KIND,T>(in);
 
     // Vectorized scanning/filtering for all numerics except float/double types.
     // If the total number of input values can't fill a vector the vector path