From b46f4b42b3ea1dd4ab2923bb73378b48e9fff065 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 22 Feb 2022 10:43:04 +0000 Subject: [PATCH] MCOL-4809 Vectorized comparison operations unit tests This commit replaces system googletest with 0.11.1 version compiled from sources to enable typed tests feature --- .drone.jsonnet | 26 ++++++++-------- CMakeLists.txt | 8 ----- cmake/FindGTest.cmake | 37 ---------------------- tests/CMakeLists.txt | 27 ++++++++++++++-- tests/simd_processors.cpp | 65 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 61 deletions(-) delete mode 100644 cmake/FindGTest.cmake create mode 100644 tests/simd_processors.cpp diff --git a/.drone.jsonnet b/.drone.jsonnet index 18c228432..b2d9d3f18 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -41,13 +41,13 @@ local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clan local yum_vault_mirror = "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-*; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-*; dnf update -y; "; local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo '; -local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; -local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; -local ubuntu18_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; -local debian10_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; -local opensuse_build_deps = 'zypper install -y liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + gcc_update_alternatives; -local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; -local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 && ' + gcc_update_alternatives; +local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y git pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable '; +local centos8_build_deps = yum_vault_mirror + ' yum install -y gcc-toolset-10 libarchive dnf-plugins-core cmake git lz4-devel && . /opt/rh/gcc-toolset-10/enable && yum config-manager --set-enabled powertools '; +local ubuntu18_04_deps = 'apt update && apt install -y gnupg git wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; +local debian10_deps = 'apt update && apt install -y gnupg git wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives; +local opensuse_build_deps = 'zypper install -y git liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + gcc_update_alternatives; +local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; +local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 git && ' + gcc_update_alternatives; local platformMap(platform) = @@ -76,12 +76,12 @@ local testRun(platform) = local testPreparation(platform) = local platform_map = { - 'opensuse/leap:15': 'zypper install -y gtest boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake', - 'centos:7': 'yum -y install epel-release && yum install -y gtest-devel cppunit-devel cmake3 boost-devel snappy-devel', - 'centos:8': yum_vault_mirror + ' yum install -y dnf-plugins-core libarchive && yum config-manager --set-enabled powertools && yum install -y lz4 gtest-devel cppunit-devel cmake3 boost-devel snappy-devel', - 'debian:10': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev libsnappy-dev googletest cmake', - 'ubuntu:18.04': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev googletest libsnappy-dev cmake g++ && cd /usr/src/googletest; cmake . && cmake --build . --target install; cd -', - 'ubuntu:20.04': 'apt update && apt install --yes libboost-all-dev libgtest-dev libcppunit-dev googletest libsnappy-dev cmake', + 'opensuse/leap:15': 'zypper install -y git boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake', + 'centos:7': 'yum -y install epel-release && yum install -y git cppunit-devel cmake3 boost-devel snappy-devel', + 'centos:8': yum_vault_mirror + ' yum install -y dnf-plugins-core libarchive && yum config-manager --set-enabled powertools && yum install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', + 'debian:10': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', + 'ubuntu:18.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake g++', + 'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', }; platform_map[platform]; diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b6fbf260..067f1c7e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,14 +149,6 @@ if (NOT CURL_FOUND) return() endif() -IF (WITH_UNITTESTS OR WITH_GTEST) - INCLUDE (FindGTest) - IF (NOT GTEST_FOUND) - MESSAGE(FATAL_ERROR "GSuite libs not found but are requested. Please install them or build.") - ENDIF() - SET (GTEST_LIBRARIES ${GTEST_LIBRARY} ${GTESTMAIN_LIBRARY} ${PTHREAD_LIBRARY}) -ENDIF() - FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) MESSAGE_ONCE(CS_NO_AWK "awk not found!") diff --git a/cmake/FindGTest.cmake b/cmake/FindGTest.cmake deleted file mode 100644 index 5860ca415..000000000 --- a/cmake/FindGTest.cmake +++ /dev/null @@ -1,37 +0,0 @@ -find_path(GTEST_ROOT_DIR - NAMES include/gtest/gtest.h -) - -find_library(GTEST_LIBRARY - NAMES gtest - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_library(GTESTMAIN_LIBRARY - NAMES gtest_main - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_library(PTHREAD_LIBRARY - NAMES pthread - HINTS ${GTEST_ROOT_DIR}/lib -) - -find_path(GTEST_INCLUDE_DIR - NAMES gtest.h - HINTS ${GTEST_ROOT_DIR}/include/gtest -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GTest DEFAULT_MSG - GTEST_LIBRARY - GTESTMAIN_LIBRARY - PTHREAD_LIBRARY - GTEST_INCLUDE_DIR -) - -mark_as_advanced( - GTEST_ROOT_DIR - GTEST_LIBRARIES - GTEST_INCLUDE_DIR -) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index abbba30e4..b00c0482a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,35 +1,56 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}) +include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) if (WITH_UNITTESTS) - enable_testing() + set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external) + ExternalProject_Add(googletest + GIT_REPOSITORY https://github.com/google/googletest + GIT_TAG release-1.11.0 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DBUILD_SHARED_LIBS=ON + ) + + include_directories(${EXTERNAL_INSTALL_LOCATION}/include) + # lib64 for RPM-based distros + link_directories(${EXTERNAL_INSTALL_LOCATION}/lib ${EXTERNAL_INSTALL_LOCATION}/lib64) + set(GTEST_LIBRARIES gtest gtest_main pthread) include(GoogleTest) - find_package(GTest REQUIRED) #GoogleTest tests add_executable(rowgroup_tests rowgroup-tests.cpp) + add_dependencies(rowgroup_tests googletest) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(rowgroup_tests TEST_PREFIX columnstore:) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) + add_dependencies(mcs_decimal_tests googletest) target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(mcs_decimal_tests TEST_PREFIX columnstore:) add_executable(dataconvert_tests dataconvert-tests.cpp) + add_dependencies(dataconvert_tests googletest) target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) gtest_discover_tests(dataconvert_tests TEST_PREFIX columnstore:) add_executable(rebuild_em_tests rebuild-em-tests.cpp) + add_dependencies(rebuild_em_tests googletest) target_link_libraries(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) gtest_discover_tests(rebuild_em_tests TEST_PREFIX columnstore:) add_executable(compression_tests compression-tests.cpp) + add_dependencies(compression_tests googletest) target_link_libraries(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) gtest_discover_tests(compression_tests TEST_PREFIX columnstore:) add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp) + target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare) + add_dependencies(column_scan_filter_tests googletest) target_link_libraries(column_scan_filter_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) gtest_discover_tests(column_scan_filter_tests TEST_PREFIX columnstore:) + add_executable(simd_processors simd_processors.cpp) + add_dependencies(simd_processors googletest) + target_link_libraries(simd_processors ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) + gtest_discover_tests(simd_processors TEST_PREFIX columnstore:) + # CPPUNIT TESTS add_executable(we_shared_components_tests shared_components_tests.cpp) add_dependencies(we_shared_components_tests loggingcpp) diff --git a/tests/simd_processors.cpp b/tests/simd_processors.cpp new file mode 100644 index 000000000..519d02845 --- /dev/null +++ b/tests/simd_processors.cpp @@ -0,0 +1,65 @@ +/* Copyright (C) 2022 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#if defined(__x86_64__) +#include +#include + +#include "simd_sse.h" +#include "datatypes/mcs_datatype.h" +#include "datatypes/mcs_int128.h" + +using namespace std; + +template +class SimdProcessorTypedTest : public testing::Test { + using IntegralType = T; + public: + + void SetUp() override + { + } +}; + +using SimdProcessor128TypedTestTypes = ::testing::Types; +TYPED_TEST_SUITE(SimdProcessorTypedTest, SimdProcessor128TypedTestTypes); + +TYPED_TEST(SimdProcessorTypedTest, SimdFilterProcessor_simd128) +{ + using Proc = typename simd::SimdFilterProcessor; + using SimdType = typename Proc::SimdType; + constexpr static simd::MT allTrue = 0xFFFF; + constexpr static simd::MT allFalse = 0x0; + Proc proc; + SimdType lhs = proc.loadValue((TypeParam)-2); + SimdType rhs = proc.loadValue((TypeParam)-3); + EXPECT_GT((uint64_t)-2LL, (uint64_t)-3LL); + EXPECT_EQ(proc.cmpGe(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpGt(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpGe(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpGt(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpLe(rhs, lhs), allTrue); + EXPECT_EQ(proc.cmpLt(rhs, lhs), allTrue); + EXPECT_EQ(proc.cmpLe(lhs, rhs), allFalse); + EXPECT_EQ(proc.cmpLt(lhs, rhs), allFalse); + EXPECT_EQ(proc.cmpEq(rhs, lhs), allFalse); + EXPECT_EQ(proc.cmpNe(rhs, lhs), allTrue); + lhs = proc.loadValue((TypeParam)-3); + EXPECT_EQ(proc.cmpEq(lhs, rhs), allTrue); + EXPECT_EQ(proc.cmpNe(rhs, lhs), allFalse); +} +#endif \ No newline at end of file