Don't ignore null or empty in calculation

primitives/linux-port/column.cpp

@@ -22,6 +22,7 @@
 #include <cassert>
 #include <cmath>
 #include <functional>
+#include <bit>
 #include <type_traits>
 #ifndef _MSC_VER
 #include <pthread.h>
@@ -1295,13 +1296,19 @@ inline SIMD_WRAPPER_TYPE simdSwapedOrderDataLoad(const ColRequestHeaderDataType
 }
 
 template <typename VT, typename SimdType>
-void vectorizedUpdateMinMax(const bool validMinMax, const MT nonNullOrEmptyMask, VT& simdProcessor,
+void vectorizedUpdateMinMax(const bool validMinMax, const MT nonNullOrEmptyMask, VT simdProcessor,
                             SimdType& dataVec, SimdType& simdMin, SimdType& simdMax)
 {
-  if (validMinMax && nonNullOrEmptyMask)
+  if (validMinMax)
   {
-    simdMin = simdProcessor.min(simdMin, dataVec);
+    simdMin = simdProcessor.blend(
-    simdMax = simdProcessor.max(simdMax, dataVec);
+        simdMin, dataVec,
+        simdProcessor.bwAnd(simdProcessor.cmpGt2(simdMin, dataVec),
+                            bitCast<SimdType>(simd::bitMaskToByteMask16(nonNullOrEmptyMask))));
+    simdMax = simdProcessor.blend(
+        simdMax, dataVec,
+        simdProcessor.bwAnd(simdProcessor.cmpGt2(dataVec, simdMax),
+                            bitCast<SimdType>(simd::bitMaskToByteMask16(nonNullOrEmptyMask))));
   }
 }
 
@@ -1322,7 +1329,7 @@ void scalarUpdateMinMax(const bool validMinMax, const MT nonNullOrEmptyMask, VT&
 }
 
 template<typename T, typename VT, typename SimdType>
-void extractMinMax(VT& simdProcessor, SimdType& simdMin, SimdType& simdMax, T& min, T& max)
+void extractMinMax(VT& simdProcessor, SimdType simdMin, SimdType simdMax, T& min, T& max)
 {
   constexpr const uint16_t size = VT::vecByteSize / sizeof(T);
   T* simdMinVec = reinterpret_cast<T*>(&simdMin);
@@ -1330,6 +1337,13 @@ void extractMinMax(VT& simdProcessor, SimdType& simdMin, SimdType& simdMax, T& m
   max = *std::max_element(simdMaxVec, simdMaxVec + size);
   min = *std::min_element(simdMinVec, simdMinVec + size);
 }
+
+template <typename T, typename VT, typename SimdType>
+void getInitialSimdMinMax(VT& simdProcessor, SimdType& simdMin, SimdType& simdMax, T min, T max)
+{
+  simdMin = simdProcessor.loadValue(min);
+  simdMax = simdProcessor.loadValue(max);
+}
 // This routine filters input block in a vectorized manner.
 // It supports all output types, all input types.
 // It doesn't support KIND==TEXT so upper layers filters this KIND out beforehand.
@@ -1465,9 +1479,12 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T*
       }
     }
   }
-  [[maybe_unused]] SimdType simdMin = simdDataLoad<VT, SimdWrapperType, HAS_INPUT_RIDS, T>(simdProcessor, srcArray,
+  [[maybe_unused]] SimdType simdMin;
-      origSrcArray, ridArray, 0).v;;
+  [[maybe_unused]] SimdType simdMax;
-  [[maybe_unused]] SimdType simdMax = simdMin;
+  if constexpr (KIND != KIND_TEXT)
+  {
+    getInitialSimdMinMax(simdProcessor, simdMin, simdMax, min, max);
+  }
   // main loop
   // writeMask tells which values must get into the result. Includes values that matches filters. Can have
   // NULLs. nonEmptyMask tells which vector coords are not EMPTY magics. nonNullMask tells which vector coords

tests/col1block.h

@@ -500,5 +500,5 @@ unsigned char __col1block_cdf[] = {
     0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0,
     0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff};
 unsigned int __col1block_cdf_len = 8192;
-constexpr int __col1block_cdf_umin = -128;
+constexpr int __col1block_cdf_umin = -126;
 constexpr int __col1block_cdf_umax = 127;

tests/simd_processors.cpp

@@ -37,19 +37,22 @@
 using namespace std;
 
 template <typename T>
-class SimdProcessorTypedTest : public testing::Test {
+class SimdProcessorTypedTest : public testing::Test
-public:
+{
+ public:
   using IntegralType = T;
-  #if TESTS_USING_SSE
+#if TESTS_USING_SSE
-    using SimdType = std::conditional_t<std::is_same<T, float>::value,
+  using SimdType =
-                                        simd::vi128f_wr,
+      std::conditional_t<std::is_same<T, float>::value, simd::vi128f_wr,
-                                        std::conditional_t<std::is_same<T, double>::value,
+                         std::conditional_t<std::is_same<T, double>::value, simd::vi128d_wr, simd::vi128_wr>>;
-                                                           simd::vi128d_wr,
+  using Proc = typename simd::SimdFilterProcessor<SimdType, T>;
-                                                           simd::vi128_wr>>;
+#else
-    using Proc = typename simd::SimdFilterProcessor<SimdType, T>;
+  using SimdType =
-    #else
+      std::conditional_t<std::is_same<T, float>::value, simd::vi128f_wr,
-    using Proc = typename simd::SimdFilterProcessor<typename simd::TypeToVecWrapperType<T>::WrapperType, T>;
+                         std::conditional_t<std::is_same<T, double>::value, simd::vi128d_wr,
-  #endif
+                                            typename simd::TypeToVecWrapperType<T>::WrapperType>>;
+  using Proc = typename simd::SimdFilterProcessor<SimdType, T>;
+#endif
   void SetUp() override
   {
   }

utils/common/simd_sse.h

@@ -250,17 +250,30 @@ class SimdFilterProcessor<
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return x;
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return x;
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    return x;
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return reinterpret_cast<SimdType>(std::min(reinterpret_cast<int128_t>(x), reinterpret_cast<int128_t>(y)));
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return reinterpret_cast<SimdType>(std::max(reinterpret_cast<int128_t>(x), reinterpret_cast<int128_t>(y)));
   }
-
-
 };
 
 template <typename VT, typename T>
@@ -376,15 +389,30 @@ class SimdFilterProcessor<
     _mm_storeu_pd(reinterpret_cast<T*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return _mm_min_pd(x, y);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return _mm_max_pd(x, y);
   }
+
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return _mm_blendv_pd(x, y, mask);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    return _mm_cmpgt_pd(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return _mm_and_pd(x, y);
+  }
 };
 
 template <typename VT, typename T>
@@ -500,15 +528,30 @@ class SimdFilterProcessor<
     _mm_storeu_ps(reinterpret_cast<T*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return _mm_min_ps(x, y);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return _mm_max_ps(x, y);
   }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    return _mm_cmpgt_ps(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return _mm_blendv_ps(x, y, mask);
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return _mm_and_ps(x, y);
+  }
 };
 
 template <typename VT, typename CHECK_T>
@@ -617,14 +660,29 @@ class SimdFilterProcessor<VT, CHECK_T,
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
   {
-    return _mm_blendv_epi8(x, y, _mm_cmpgt_epi64(x,y));
+    return _mm_blendv_epi8(x, y, mask);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
   {
-    return _mm_blendv_epi8(x, y, _mm_cmpgt_epi64(y,x));
+    return _mm_and_si128(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    return _mm_cmpgt_epi64(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
+  {
+    return blend(x, y, cmpGt2(x,y));
+  }
+
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
+  {
+    return blend(x, y, cmpGt2(y,x));
   }
 };
 
@@ -737,14 +795,32 @@ class SimdFilterProcessor<VT, CHECK_T,
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
   {
-    return _mm_blendv_epi8(x, y, _mm_cmpgt_epi64(x,y));
+    return _mm_blendv_epi8(x, y, mask);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
   {
-    return _mm_blendv_epi8(x, y, _mm_cmpgt_epi64(y,x));
+    return _mm_and_si128(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    SimdType signVec = constant4i<0,(int32_t)0x80000000,0,(int32_t)0x80000000>();
+    SimdType xFlip = _mm_xor_si128(x, signVec);
+    SimdType yFlip = _mm_xor_si128(y, signVec);
+    return _mm_cmpgt_epi64(xFlip, yFlip);
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
+  {
+    return blend(x, y, cmpGt2(x,y));
+  }
+
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
+  {
+    return blend(x, y, cmpGt2(y,x));
   }
 };
 
@@ -854,12 +930,27 @@ class SimdFilterProcessor<VT, CHECK_T,
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return _mm_blendv_epi8(x, y, mask);
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return _mm_and_si128(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    return _mm_cmpgt_epi32(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return _mm_min_epi32(x, y);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return _mm_max_epi32(x, y);
   }
@@ -974,12 +1065,30 @@ class SimdFilterProcessor<VT, CHECK_T,
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return _mm_blendv_epi8(x, y, mask);
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return _mm_and_si128(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    SimdType signVec = constant4i<(int32_t)0x80000000,(int32_t)0x80000000,(int32_t)0x80000000,(int32_t)0x80000000>();
+    SimdType xFlip = _mm_xor_si128(x, signVec);
+    SimdType yFlip = _mm_xor_si128(y, signVec);
+    return _mm_cmpgt_epi32(xFlip, yFlip);
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return _mm_min_epu32(x, y);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return _mm_max_epu32(x, y);
   }
@@ -1090,12 +1199,27 @@ class SimdFilterProcessor<
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return _mm_blendv_epi8(x, y, mask);
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return _mm_and_si128(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    return _mm_cmpgt_epi16(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return _mm_min_epi16(x, y);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return _mm_max_epi16(x, y);
   }
@@ -1207,12 +1331,30 @@ class SimdFilterProcessor<
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return _mm_blendv_epi8(x, y, mask);
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return _mm_and_si128(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y)
+  {
+    SimdType ones =
+        constant4i<(int32_t)0xFFFFFFFF, (int32_t)0xFFFFFFFF, (int32_t)0xFFFFFFFF, (int32_t)0xFFFFFFFF>();
+    SimdType maxOfTwo = _mm_max_epu16(x, y);
+    return _mm_xor_si128(_mm_cmpeq_epi16(y, maxOfTwo), ones);
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return _mm_min_epu16(x, y);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return _mm_max_epu16(x, y);
   }
@@ -1330,12 +1472,27 @@ class SimdFilterProcessor<
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return _mm_blendv_epi8(x, y, mask);
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return _mm_and_si128(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y) const
+  {
+    return _mm_cmpgt_epi8(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return _mm_min_epi8(x, y);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return _mm_max_epi8(x, y);
   }
@@ -1454,12 +1611,30 @@ class SimdFilterProcessor<
     _mm_storeu_si128(reinterpret_cast<SimdType*>(dst), x);
   }
 
-  MCS_FORCE_INLINE SimdType min(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType blend(SimdType x, SimdType y, SimdType mask) const
+  {
+    return _mm_blendv_epi8(x, y, mask);
+  }
+
+  MCS_FORCE_INLINE SimdType bwAnd(SimdType x, SimdType y) const
+  {
+    return _mm_and_si128(x, y);
+  }
+
+  MCS_FORCE_INLINE SimdType cmpGt2(SimdType x, SimdType y)
+  {
+    SimdType ones =
+        constant4i<(int32_t)0xFFFFFFFF, (int32_t)0xFFFFFFFF, (int32_t)0xFFFFFFFF, (int32_t)0xFFFFFFFF>();
+    SimdType maxOfTwo = _mm_max_epu8(x, y);
+    return _mm_xor_si128(_mm_cmpeq_epi8(y, maxOfTwo), ones);
+  }
+
+  MCS_FORCE_INLINE SimdType min(SimdType x, SimdType y) const
   {
     return _mm_min_epu8(x, y);
   }
 
-  MCS_FORCE_INLINE SimdType max(SimdType& x, SimdType& y)
+  MCS_FORCE_INLINE SimdType max(SimdType x, SimdType y) const
   {
     return _mm_max_epu8(x, y);
   }