From 3800dfda3890f0c5ebcb743c179bf36d61e1f499 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com> Date: Mon, 9 Jun 2025 20:14:22 +0400 Subject: [PATCH] chore(ci): backport CI from stable to develop-23.02 (#3586) * core(ci): try to backport CI from stable to develop-23.02 * chore(build): no Werror * chore(build): buildfiles from stable * restore old boost * no unused param warning * Some final build fix --- .drone.jsonnet | 1103 ++++++++++++------------ CMakeLists.txt | 503 ++--------- build/bootstrap_mcs.sh | 815 ++++++++++------- build/build_cmapi.sh | 90 ++ build/createrepo.sh | 69 ++ build/debian_install_file_compare.py | 81 ++ build/prepare_test_stage.sh | 125 +++ build/report_test_stage.sh | 96 +++ build/run_smoke.sh | 40 + build/update-clang-version.sh | 85 ++ build/utils.sh | 623 +++++++------ cmake-format.py | 241 ++++++ cmake/ColumnstoreLibrary.cmake | 105 +++ cmake/boost.1.88.named_proxy.hpp.patch | 10 + cmake/columnstore_version.cmake | 6 +- cmake/compiler_flags.cmake | 120 +++ cmake/configureEngine.cmake | 717 +++++++-------- cmake/dirs.cmake | 20 + cmake/includes.cmake | 108 +++ cmake/libs.cmake | 44 + cmake/packages.cmake | 64 ++ cmake/superbuild.md | 46 +- cmake/thrift.cmake | 51 +- core_dumps/columnstore_schema.sql | 55 ++ core_dumps/docker-awaiter.sh | 38 + core_dumps/save_stack.sh | 24 + core_dumps/upgrade_data.sh | 45 + core_dumps/upgrade_setup_deb.sh | 62 ++ core_dumps/upgrade_setup_rpm.sh | 42 + core_dumps/upgrade_verify.sh | 20 + tests/CMakeLists.txt | 48 +- tests/counting_allocator.cpp | 2 +- utils/funcexp/CMakeLists.txt | 20 +- 33 files changed, 3560 insertions(+), 1958 deletions(-) create mode 100755 build/build_cmapi.sh create mode 100755 build/createrepo.sh create mode 100644 build/debian_install_file_compare.py create mode 100755 build/prepare_test_stage.sh create mode 100755 build/report_test_stage.sh create mode 100755 build/run_smoke.sh create mode 100644 build/update-clang-version.sh create mode 100644 cmake-format.py create mode 100644 cmake/ColumnstoreLibrary.cmake create mode 100644 cmake/boost.1.88.named_proxy.hpp.patch create mode 100644 cmake/compiler_flags.cmake create mode 100644 cmake/dirs.cmake create mode 100644 cmake/includes.cmake create mode 100644 cmake/libs.cmake create mode 100644 cmake/packages.cmake create mode 100644 core_dumps/columnstore_schema.sql create mode 100755 core_dumps/docker-awaiter.sh create mode 100644 core_dumps/save_stack.sh create mode 100755 core_dumps/upgrade_data.sh create mode 100755 core_dumps/upgrade_setup_deb.sh create mode 100755 core_dumps/upgrade_setup_rpm.sh create mode 100755 core_dumps/upgrade_verify.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index cbfdd4ff6..1db4e0f7f 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,326 +1,397 @@ -local events = ['pull_request', 'cron']; +local events = ["pull_request", "cron"]; local servers = { - develop: ['10.6-enterprise'], - 'develop-23.02': ['10.6-enterprise'], + develop: ["10.6-enterprise"], + "develop-23.02": ["10.6-enterprise"], }; local platforms = { - develop: ['rockylinux:8', 'rockylinux:9', 'debian:11', 'ubuntu:20.04', 'ubuntu:22.04'], - 'develop-23.02': ['rockylinux:8', 'rockylinux:9', 'debian:11', 'ubuntu:20.04', 'ubuntu:22.04'], + develop: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], + "develop-23.02": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], }; local platforms_arm = { - develop: ['rockylinux:8', 'rockylinux:9', 'debian:11', 'ubuntu:20.04', 'ubuntu:22.04'], - 'develop-23.02': ['rockylinux:8', 'rockylinux:9', 'debian:11', 'ubuntu:20.04', 'ubuntu:22.04'], + develop: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], + "develop-23.02": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], }; -local any_branch = '**'; +local rewrite_ubuntu_mirror = @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list || true; " + + @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list.d/ubuntu.sources || true; " + + "cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list | grep archive || true; "; + +local customEnvCommandsMap = { + // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'], + "clang-20": [ + rewrite_ubuntu_mirror, + "apt-get clean && apt-get update", + "apt-get install -y wget curl lsb-release software-properties-common gnupg", + "wget https://apt.llvm.org/llvm.sh", + "bash llvm.sh 20", + "export CC=/usr/bin/clang", + "export CXX=/usr/bin/clang++", + ], +}; + +local customEnvCommands(envkey, builddir) = + local updateAlternatives = { + "clang-20": ["bash /mdb/" + builddir + + "/storage/columnstore/columnstore/build/update-clang-version.sh 20 100"], + }; + (if (std.objectHas(customEnvCommandsMap, envkey)) + then customEnvCommandsMap[envkey] + updateAlternatives[envkey] else []); + + +local customBootstrapParamsForExisitingPipelines(envkey) = + local customBootstrapMap = { + "ubuntu:24.04": "--custom-cmake-flags '-DCOLUMNSTORE_ASAN_FOR_UNITTESTS=YES'", + }; + (if (std.objectHas(customBootstrapMap, envkey)) + then customBootstrapMap[envkey] else ""); + +local customBootstrapParamsForAdditionalPipelinesMap = { + ASAN: "--asan", + TSAN: "--tsan", + UBSAN: "--ubsan", +}; + + +local any_branch = "**"; local platforms_custom = platforms.develop; local platforms_arm_custom = platforms_arm.develop; local platforms_mtr = platforms.develop; -local builddir = 'verylongdirnameforverystrangecpackbehavior'; +local builddir = "verylongdirnameforverystrangecpackbehavior"; -local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CONFIG=mysql_release ' + - '-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ' + - '-DPLUGIN_COLUMNSTORE=YES -DWITH_UNITTESTS=YES ' + - '-DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO ' + - '-DPLUGIN_CONNECT=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' + - '-DPLUGIN_GSSAPI=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' + - '-DWITH_EMBEDDED_SERVER=NO -DWITH_WSREP=NO -DWITH_COREDUMPS=ON'; +local mtr_suite_list = "basic,bugfixes"; +local mtr_full_set = "basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly"; -local clang_version = '16'; -local gcc_version = '11'; - -local clang_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-' + clang_version + ' 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-' + clang_version + ' && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 '; - - -local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison ' + - 'snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool ' + - 'policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel ' + - 'expect createrepo python3 '; - -local rockylinux8_build_deps = "dnf install -y 'dnf-command(config-manager)' " + - '&& dnf config-manager --set-enabled powertools ' + - '&& dnf install -y gcc-toolset-' + gcc_version + ' libarchive cmake lz4-devel ' + - '&& . /opt/rh/gcc-toolset-' + gcc_version + '/enable '; - - -local rockylinux9_build_deps = "dnf install -y 'dnf-command(config-manager)' " + - '&& dnf config-manager --set-enabled crb ' + - '&& dnf install -y pcre2-devel lz4-devel gcc gcc-c++'; - -local debian11_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' && ' + clang_update_alternatives; -local ubuntu20_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' && ' + clang_update_alternatives; - -local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata libssl-dev && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; -local turnon_clang = 'export CC=/usr/bin/clang; export CXX=/usr/bin/clang++ '; -local bootstrap_deps = 'apt-get -y update && apt-get -y install build-essential automake libboost-all-dev bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl '; - -local mtr_suite_list = 'basic,bugfixes'; -local mtr_full_set = 'basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly'; - -local platformMap(platform, arch) = - local platform_map = { - 'rockylinux:8': rockylinux8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package', - 'rockylinux:9': rockylinux9_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux9 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package', - 'debian:11': bootstrap_deps + ' && ' + deb_build_deps + ' && ' + debian11_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=bullseye' debian/autobake-deb.sh", - 'ubuntu:20.04': bootstrap_deps + ' && ' + deb_build_deps + ' && ' + ubuntu20_04_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh", - 'ubuntu:22.04': bootstrap_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=jammy' debian/autobake-deb.sh", - }; - local result = std.strReplace(std.strReplace(platform, ':', ''), '/', '-'); - 'export CLICOLOR_FORCE=1; ' + platform_map[platform] + ' | storage/columnstore/columnstore/build/ansi2txt.sh ' + result + '/build.log'; +local upgrade_test_lists = { + rockylinux8: { + arm64: ["10.6.4-1", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"], + amd64: ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + }, + rockylinux9: { + arm64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + amd64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + }, + debian12: { + arm64: [], + amd64: [], + }, + "ubuntu20.04": { + arm64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + amd64: ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + }, + "ubuntu22.04": { + arm64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + amd64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + }, + "ubuntu24.04": + { + arm64: [], + amd64: [], + }, +}; local testRun(platform) = local platform_map = { - 'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', - 'rockylinux:9': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', - 'debian:11': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', - 'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', - 'ubuntu:22.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', + "rockylinux:8": "ctest3 -R columnstore: -j $(nproc) --output-on-failure", + "rockylinux:9": "ctest3 -R columnstore: -j $(nproc) --output-on-failure", + "debian:12": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", + "ubuntu:20.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", + "ubuntu:22.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", + "ubuntu:24.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", + }; platform_map[platform]; +local gcc_version = "11"; + +local rockylinux8_deps = "dnf install -y 'dnf-command(config-manager)' " + + "&& dnf config-manager --set-enabled powertools " + + "&& dnf install -y gcc-toolset-" + gcc_version + " libarchive cmake " + + "&& . /opt/rh/gcc-toolset-" + gcc_version + "/enable "; + +local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " + + "&& dnf config-manager --set-enabled crb " + + "&& dnf install -y gcc gcc-c++"; + +local rockylinux_common_deps = " && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel"; + +local deb_deps = rewrite_ubuntu_mirror + "apt-get clean && apt-get update && apt-get install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev"; local testPreparation(platform) = local platform_map = { - 'rockylinux:8': rockylinux8_build_deps + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', - 'rockylinux:9': rockylinux9_build_deps + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel', - 'debian:11': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', - 'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', - 'ubuntu:22.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake', + "rockylinux:8": rockylinux8_deps + rockylinux_common_deps, + "rockylinux:9": rockylinux9_deps + rockylinux_common_deps, + "debian:12": deb_deps, + "ubuntu:20.04": deb_deps, + "ubuntu:22.04": deb_deps, + "ubuntu:24.04": deb_deps, }; platform_map[platform]; -// server points to the commit in bb-10.6.14-9-cs-23.02.10-1 of ES -local Pipeline(branch, platform, event, arch='amd64', server='0c6b89c694a88aae764b152045bf5bdb234d7edf') = { - local pkg_format = if (std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb', - local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd', - local mtr_path = if (pkg_format == 'rpm') then '/usr/share/mysql-test' else '/usr/share/mysql/mysql-test', - local cmapi_path = '/usr/share/columnstore/cmapi', - local etc_path = '/etc/columnstore', - local socket_path = if (pkg_format == 'rpm') then '/var/lib/mysql/mysql.sock' else '/run/mysqld/mysqld.sock', - local config_path_prefix = if (pkg_format == 'rpm') then '/etc/my.cnf.d/' else '/etc/mysql/mariadb.conf.d/50-', - local img = if (platform == 'rockylinux:8') then platform else 'detravi/' + std.strReplace(platform, '/', '-'), - local branch_ref = if (branch == any_branch) then 'develop' else branch, +local make_clickable_link(link) = "echo -e '\\e]8;;" + link + "\\e\\\\" + link + "\\e]8;;\\e\\\\'"; +local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}", + make_clickable_link("https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=:${DRONE_STAGE_MACHINE};v=3;$case=tags:true%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:launchTime")]; + +local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = { + local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb", + local init = if (pkg_format == "rpm") then "/usr/lib/systemd/systemd" else "systemd", + local mtr_path = if (pkg_format == "rpm") then "/usr/share/mysql-test" else "/usr/share/mysql/mysql-test", + local cmapi_path = "/usr/share/columnstore/cmapi", + local etc_path = "/etc/columnstore", + local socket_path = if (pkg_format == "rpm") then "/var/lib/mysql/mysql.sock" else "/run/mysqld/mysqld.sock", + local config_path_prefix = if (pkg_format == "rpm") then "/etc/my.cnf.d/" else "/etc/mysql/mariadb.conf.d/50-", + local img = if (platform == "rockylinux:8") then platform else "detravi/" + std.strReplace(platform, "/", "-"), + local branch_ref = if (branch == any_branch) then "develop-23.02" else branch, // local regression_tests = if (std.startsWith(platform, 'debian') || std.startsWith(platform, 'ubuntu:20')) then 'test000.sh' else 'test000.sh,test001.sh', - local branchp = if (branch == '**') then '' else branch + '/', - local brancht = if (branch == '**') then '' else branch + '-', - local result = std.strReplace(std.strReplace(platform, ':', ''), '/', '-'), + local branchp = if (branch == "**") then "" else branch + "/", + local brancht = if (branch == "**") then "" else branch + "-", + local platformKey = std.strReplace(std.strReplace(platform, ":", ""), "/", "-"), + local result = platformKey + if customBuildEnvCommandsMapKey != "" then "_" + customBuildEnvCommandsMapKey else "", - local publish_pkg_url = 'https://cspkg.s3.amazonaws.com/index.html?prefix=' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/', + local packages_url = "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server, + local publish_pkg_url = "https://cspkg.s3.amazonaws.com/index.html?prefix=" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/" + result + "/", + local repo_pkg_url_no_res = "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/", - local packages_url = 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server, + local container_tags = if (event == "cron") then [brancht + std.strReplace(event, "_", "-") + "${DRONE_BUILD_NUMBER}", brancht] else [brancht + std.strReplace(event, "_", "-") + "${DRONE_BUILD_NUMBER}"], + local container_version = branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch, - local container_tags = if (event == 'cron') then [brancht + std.strReplace(event, '_', '-') + '${DRONE_BUILD_NUMBER}', brancht] else [brancht + std.strReplace(event, '_', '-') + '${DRONE_BUILD_NUMBER}'], - local container_version = branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch, + local server_remote = if (std.endsWith(server, "enterprise")) then "https://github.com/mariadb-corporation/MariaDBEnterprise" else "https://github.com/MariaDB/server", - local server_remote = if (std.endsWith(server, 'enterprise')) then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server', - - local sccache_arch = if (arch == 'amd64') then 'x86_64' else 'aarch64', - local get_sccache = 'curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.0/sccache-v0.3.0-' + sccache_arch + '-unknown-linux-musl.tar.gz ' + - '&& tar xzf sccache.tar.gz ' + - '&& install sccache*/sccache /usr/local/bin/', + local sccache_arch = if (arch == "amd64") then "x86_64" else "aarch64", + local get_sccache = ["echo getting sccache...", + rewrite_ubuntu_mirror, + "(apt-get clean && apt-get update -y && apt-get install -y curl || yum install -y curl || true)", + "curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz &&", + "tar xzf sccache.tar.gz", + "install sccache*/sccache /usr/local/bin/ && echo sccache installed"], local pipeline = self, - publish(step_prefix='pkg', eventp=event + '/${DRONE_BUILD_NUMBER}'):: { - name: 'publish ' + step_prefix, - depends_on: [std.strReplace(step_prefix, ' latest', '')], - image: 'plugins/s3-sync', + publish(step_prefix="pkg", eventp=event + "/${DRONE_BUILD_NUMBER}"):: { + name: "publish " + step_prefix, + depends_on: [std.strReplace(step_prefix, " latest", ""), "createrepo"], + image: "amazon/aws-cli", when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, - settings: { - bucket: 'cspkg', - access_key: { - from_secret: 'aws_access_key_id', + environment: { + AWS_ACCESS_KEY_ID: { + from_secret: "aws_access_key_id", }, - secret_key: { - from_secret: 'aws_secret_access_key', + AWS_SECRET_ACCESS_KEY: { + from_secret: "aws_secret_access_key", }, - source: result, - // branchp has slash if not empty - target: branchp + eventp + '/' + server + '/' + arch + '/' + result, - delete: 'true', }, + commands: [ + "ls " + result, + '[ -z "$(ls -A "' + result + '")" ] && echo Nothing to publish! && exit 1', + + "aws s3 sync " + result + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors", + 'echo "Data uploaded to: ' + publish_pkg_url + '"', + make_clickable_link(publish_pkg_url), + "rm -rf " + result + "/*", + ], }, - local regression_tests = if (event == 'cron') then [ - 'test000.sh', - 'test001.sh', - 'test005.sh', - 'test006.sh', - 'test007.sh', - 'test008.sh', - 'test009.sh', - 'test010.sh', - 'test011.sh', - 'test012.sh', - 'test013.sh', - 'test014.sh', - 'test023.sh', - 'test201.sh', - 'test202.sh', - 'test203.sh', - 'test204.sh', - 'test210.sh', - 'test211.sh', - 'test212.sh', - 'test297.sh', - 'test299.sh', + local regression_tests = if (event == "cron") then [ + "test000.sh", + "test001.sh", + "test005.sh", + "test006.sh", + "test007.sh", + "test008.sh", + "test009.sh", + "test010.sh", + "test011.sh", + "test012.sh", + "test013.sh", + "test014.sh", + "test023.sh", + "test201.sh", + "test202.sh", + "test203.sh", + "test204.sh", + "test210.sh", + "test211.sh", + "test212.sh", + // "test222.sh", FIXME: restore the test + "test297.sh", + "test299.sh", ] else [ - 'test000.sh', - 'test001.sh', + "test000.sh", + "test001.sh", ], + local mdb_server_versions = upgrade_test_lists[platformKey][arch], + local indexes(arr) = std.range(0, std.length(arr) - 1), - regression(name, depends_on):: { - name: name, - depends_on: depends_on, - image: 'docker:git', - volumes: [pipeline._volumes.docker], - when: { - status: ['success', 'failure'], - }, - [if (name != 'test000.sh' && name != 'test001.sh') then 'failure']: 'ignore', - environment: { - REGRESSION_TIMEOUT: { - from_secret: 'regression_timeout', - }, - }, - commands: [ - 'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} mkdir -p reg-logs', - "docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c 'sleep 4800 && eu-stack -p `pidof PrimProc` -n 0 | tee /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/prim_proc_callstacks.txt' & ", - 'docker exec --env PRESERVE_LOGS=true -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + ' || ./regression_logs.sh ' + name + '"', - 'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} cat go.log || echo "missing go.log"', - ], - }, + local execInnerDocker(command, containerName, flags="") = + "docker exec " + flags + " -t " + containerName + " " + command, + + local execInnerDockerNoTTY(command, containerName, flags="") = + "docker exec " + flags + " " + containerName + " " + command, + + local getContainerName(stepname) = stepname + "$${DRONE_BUILD_NUMBER}", + + local installCmapi(containerName, pkg_format) = + if (pkg_format == "deb") then execInnerDocker('bash -c "apt-get clean && apt-get update -y && apt-get install -y mariadb-columnstore-cmapi"', containerName) + else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName), + + local prepareTestStage(containerName, result, do_setup) = + 'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/prepare_test_stage.sh" + + " --container-name " + containerName + + " --docker-image " + img + + " --result-path " + result + + " --packages-url " + packages_url + + " --do-setup " + std.toString(do_setup) + '"', + + local reportTestStage(containerName, result, stage) = 'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/report_test_stage.sh " + containerName + " " + result + " " + stage + '"', + _volumes:: { mdb: { - name: 'mdb', - path: '/mdb', + name: "mdb", + path: "/mdb", }, docker: { - name: 'docker', - path: '/var/run/docker.sock', + name: "docker", + path: "/var/run/docker.sock", }, }, smoke:: { - name: 'smoke', - depends_on: ['publish pkg'], - image: 'docker', - volumes: [pipeline._volumes.docker], + name: "smoke", + depends_on: ["publish pkg"], + image: "docker", + volumes: [pipeline._volumes.mdb, pipeline._volumes.docker], commands: [ - 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', - if (pkg_format == 'rpm') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gdb gawk epel-release which rsyslog hostname procps-ng"' else 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y gdb gawk rsyslog hostname procps wget"', - if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} mkdir core', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} chmod 777 core', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_smoke_core_dump.%p"', - 'docker cp core_dumps/. smoke$${DRONE_BUILD_NUMBER}:/', - 'docker cp setup-repo.sh smoke$${DRONE_BUILD_NUMBER}:/', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} /setup-repo.sh', - if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore"' else 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine"', - 'sleep $${SMOKE_DELAY_SECONDS:-1s}', - // start mariadb and mariadb-columnstore services and run simple query - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb-columnstore', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"', - // restart mariadb and mariadb-columnstore services and run simple query again - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl restart mariadb', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl restart mariadb-columnstore', - 'sleep 10', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} mariadb -e "insert into test.t1 values (2); select * from test.t1"', + prepareTestStage(getContainerName("smoke"), result, true), + "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_smoke.sh " + getContainerName("smoke"), ], }, - mtr:: { - name: 'mtr', - depends_on: ['smoke'], - image: 'docker:git', + smokelog:: { + name: "smokelog", + depends_on: ["smoke"], + image: "docker", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], + commands: [ + reportTestStage(getContainerName("smoke"), result, "smoke"), + ], + when: { + status: ["success", "failure"], + }, + }, + upgrade(version):: { + name: "upgrade-test from " + version, + depends_on: ["regressionlog"], + image: "docker", volumes: [pipeline._volumes.docker], environment: { - MTR_SUITE_LIST: '${MTR_SUITE_LIST:-' + mtr_suite_list + '}', - MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}', + UPGRADE_TOKEN: { + from_secret: "es_token", + }, }, commands: [ - 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --shm-size=500m --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', - if (std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget tar lz4 procps-ng"', - if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', - if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y procps wget tar liblz4-tool"', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} mkdir core', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} chmod 777 core', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_mtr_core_dump.%p"', - 'docker cp core_dumps/. mtr$${DRONE_BUILD_NUMBER}:/', - if (std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb epel-release diffutils which rsyslog hostname patch perl cracklib-dicts procps-ng"' else '', - if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', - if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget gawk gdb rsyslog hostname patch"' else '', - 'docker cp setup-repo.sh mtr$${DRONE_BUILD_NUMBER}:/', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} /setup-repo.sh', - if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore mariadb-test"' else 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"', - 'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} chown -R mysql:mysql ' + mtr_path, + // why do we mount cgroups here, but miss it on other steps? + prepareTestStage(getContainerName("upgrade") + version, result, false), + if (pkg_format == "deb") + then execInnerDocker('bash -c "./upgrade_setup_deb.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', + getContainerName("upgrade") + version), + if (pkg_format == "rpm") + then execInnerDocker('bash -c "./upgrade_setup_rpm.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', + getContainerName("upgrade") + version), + ], + }, + upgradelog:: { + name: "upgradelog", + depends_on: std.map(function(p) "upgrade-test from " + p, mdb_server_versions), + image: "docker", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], + commands: + ["echo"] + + std.map( + function(ver) + reportTestStage( + getContainerName("upgrade") + ver, + result, + "upgrade_" + ver + ), + mdb_server_versions + ), + when: { + status: ["success", "failure"], + }, + }, + mtr:: { + name: "mtr", + depends_on: ["smoke"], + image: "docker:git", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], + environment: { + MTR_SUITE_LIST: "${MTR_SUITE_LIST:-" + mtr_suite_list + "}", + MTR_FULL_SUITE: "${MTR_FULL_SUITE:-false}", + }, + commands: [ + prepareTestStage(getContainerName("mtr"), result, true), + "docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:" + mtr_path + "/suite/", + execInnerDocker("chown -R mysql:mysql " + mtr_path, getContainerName("mtr")), // disable systemd 'ProtectSystem' (we need to write to /usr/share/) - "docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", - if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "echo character_set_server=latin1 >> /etc/mysql/mariadb.conf.d/columnstore.cnf"' else 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "echo character_set_server=latin1 >> /etc/my.cnf.d/columnstore.cnf"', - if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "echo collation_server=latin1_swedish_ci >> /etc/mysql/mariadb.conf.d/columnstore.cnf"' else 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "echo collation_server=latin1_swedish_ci >> /etc/my.cnf.d/columnstore.cnf"', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl daemon-reload', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl start mariadb', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} mariadb -e "create database if not exists test;"', + execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", getContainerName("mtr")), + execInnerDocker("systemctl daemon-reload", getContainerName("mtr")), + execInnerDocker("systemctl start mariadb", getContainerName("mtr")), // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps. - //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 4G"', - //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 1G"', - //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=mtr$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl restart mariadb-columnstore', + execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", getContainerName("mtr")), + execInnerDocker('mariadb -e "create database if not exists test;"', getContainerName("mtr")), + execInnerDocker("systemctl restart mariadb-columnstore", getContainerName("mtr")), + // delay mtr for manual debugging on live instance - 'sleep $${MTR_DELAY_SECONDS:-1s}', + "sleep $${MTR_DELAY_SECONDS:-1s}", 'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")', - if (event == 'custom' || event == 'cron') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"', - if (event == 'custom' || event == 'cron') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"', - if (event == 'cron') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=' + std.join(',', std.map(function(x) 'columnstore/' + x, std.split(mtr_full_set, ','))) + '"' else 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"', + if (event == "custom" || event == "cron") then + execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"', + getContainerName("mtr")), + if (event == "custom" || event == "cron") then + execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"', + getContainerName("mtr")), + + if (event == "cron") then + execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + + " --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=" + + std.join(",", std.map(function(x) "columnstore/" + x, std.split(mtr_full_set, ","))), + getContainerName("mtr")) + '"' + else + execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"', + getContainerName("mtr")), ], }, mtrlog:: { - name: 'mtrlog', - depends_on: ['mtr'], - image: 'docker', - volumes: [pipeline._volumes.docker], + name: "mtrlog", + depends_on: ["mtr"], + image: "docker", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], commands: [ - 'echo "---------- start mariadb service logs ----------"', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"', - 'echo "---------- end mariadb service logs ----------"', - 'echo', - 'echo "---------- start columnstore debug log ----------"', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', - 'echo "---------- end columnstore debug log ----------"', - 'echo "---------- end columnstore debug log ----------"', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/logs.sh mtr"', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Mtr"', - 'docker cp mtr$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/', - 'docker cp mtr$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/', - 'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/' + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"', - 'docker stop mtr$${DRONE_BUILD_NUMBER} && docker rm mtr$${DRONE_BUILD_NUMBER} || echo "cleanup mtr failure"', + reportTestStage(getContainerName("mtr"), result, "mtr"), ], when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, }, prepare_regression:: { - name: 'prepare regression', - depends_on: ['mtr', 'publish pkg', 'publish cmapi build'], + name: "prepare regression", + depends_on: ["mtr", "publish pkg", "publish cmapi build"], when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, - image: 'docker:git', + image: "docker:git", volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], environment: { - REGRESSION_BRANCH_REF: '${DRONE_SOURCE_BRANCH}', + REGRESSION_BRANCH_REF: "${DRONE_SOURCE_BRANCH}", REGRESSION_REF_AUX: branch_ref, }, commands: [ @@ -332,134 +403,82 @@ local Pipeline(branch, platform, event, arch='amd64', server='0c6b89c694a88aae76 'echo "$$REGRESSION_REF"', // REGRESSION_REF can be empty if there is no appropriate branch in regression repository. // assign what is appropriate by default. - 'export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}', + "export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}", 'echo "$$REGRESSION_REF"', // clone regression test repo - 'git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test', + "git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test", // where are we now? - 'cd mariadb-columnstore-regression-test', - 'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD', - 'cd ..', - 'docker run --shm-size=500m --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', - if (pkg_format == 'rpm') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb gcc-c++ epel-release diffutils tar findutils lz4 wget which rsyslog hostname procps-ng elfutils"' else 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget tar liblz4-tool procps wget findutils gawk gdb rsyslog hostname g++"', - if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} mkdir core', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} chmod 777 core', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_regression_core_dump.%p"', - 'docker cp core_dumps/. regression$${DRONE_BUILD_NUMBER}:/', - 'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/', + "cd mariadb-columnstore-regression-test", + "git rev-parse --abbrev-ref HEAD && git rev-parse HEAD", + "cd ..", + prepareTestStage(getContainerName("regression"), result, true), + + "docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/", // list storage manager binary - 'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager', - 'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/', + "ls -la /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager", + "docker cp /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/", // check storage-manager unit test binary file - 'docker exec -t regression$${DRONE_BUILD_NUMBER} ls -l /storage-manager', + execInnerDocker("ls -l /storage-manager", getContainerName("regression")), // copy test data for regression test suite - 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', - 'docker cp setup-repo.sh regression$${DRONE_BUILD_NUMBER}:/', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} /setup-repo.sh', - if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore"' else 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine"', + execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', getContainerName("regression")), + // set mariadb lower_case_table_names=1 config option - 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf', + execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + "server.cnf", getContainerName("regression")), // set default client character set to utf-8 - 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf', - // Regression tests hacks to pass on debs - if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/character-set-server/d" ' + config_path_prefix + 'server.cnf', - if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/collation-server/d" ' + config_path_prefix + 'server.cnf', + execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + "client.cnf", getContainerName("regression")), + // Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps. - //'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 5G"', - //'docker exec -t regressin$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 2G"', - //'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=regression$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"', - // start mariadb and mariadb-columnstore services - 'docker exec -t regression$${DRONE_BUILD_NUMBER} systemctl start mariadb', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} systemctl restart mariadb-columnstore', + execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", getContainerName("regression")), + + execInnerDocker("systemctl start mariadb", getContainerName("regression")), + execInnerDocker("systemctl restart mariadb-columnstore", getContainerName("regression")), // delay regression for manual debugging on live instance - 'sleep $${REGRESSION_DELAY_SECONDS:-1s}', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} /usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester', + "sleep $${REGRESSION_DELAY_SECONDS:-1s}", + execInnerDocker("/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester", getContainerName("regression")), ], }, - smokelog:: { - name: 'smokelog', - depends_on: ['smoke'], - image: 'docker', + regression(name, depends_on):: { + name: name, + depends_on: depends_on, + image: "docker:git", volumes: [pipeline._volumes.docker], - commands: [ - 'echo "---------- start mariadb service logs ----------"', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"', - 'echo "---------- end mariadb service logs ----------"', - 'echo', - 'echo "---------- start columnstore debug log ----------"', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', - 'echo "---------- end columnstore debug log ----------"', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/logs.sh smoke"', - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Smoke"', - 'docker cp smoke$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/', - 'docker cp smoke$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/', - 'ls -l /drone/src/' + result, - 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"', - 'docker stop smoke$${DRONE_BUILD_NUMBER} && docker rm smoke$${DRONE_BUILD_NUMBER} || echo "cleanup smoke failure"', - ], when: { - status: ['success', 'failure'], + status: ["success", "failure"], + }, + [if (name != "test000.sh" && name != "test001.sh") then "failure"]: "ignore", + environment: { + REGRESSION_TIMEOUT: { + from_secret: "regression_timeout", + }, }, - }, - cmapilog:: { - name: 'cmapi log', - depends_on: ['cmapi test'], - image: 'docker', - volumes: [pipeline._volumes.docker], commands: [ - 'echo "---------- start mariadb service logs ----------"', - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"', - 'echo "---------- end mariadb service logs ----------"', - 'echo', - 'echo "---------- start columnstore debug log ----------"', - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', - 'echo "---------- end columnstore debug log ----------"', - 'echo "---------- start cmapi log ----------"', - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi cmapi_server.log"', - 'echo "---------- end cmapi log ----------"', - 'ls -l /drone/src/' + result, - 'docker stop cmapi$${DRONE_BUILD_NUMBER} && docker rm cmapi$${DRONE_BUILD_NUMBER} || echo "cleanup cmapi failure"', + execInnerDocker("mkdir -p reg-logs", getContainerName("regression"), "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"), + execInnerDocker("bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' & ", + getContainerName("regresion")), + execInnerDockerNoTTY('bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + " || ./regression_logs.sh " + name + '"', + getContainerName("regression"), + "--env PRESERVE_LOGS=true --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"), ], - when: { - status: ['success', 'failure'], - }, }, regressionlog:: { - name: 'regressionlog', + name: "regressionlog", depends_on: [regression_tests[std.length(regression_tests) - 1]], - image: 'docker', - volumes: [pipeline._volumes.docker], + image: "docker", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], commands: [ - 'echo "---------- start columnstore regression short report ----------"', - 'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} cat go.log || echo "missing go.log"', - 'echo "---------- end columnstore regression short report ----------"', - 'echo', - 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/' + result + '/', - 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"', - 'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"', - 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/logs.sh regression"', - 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Regression"', - 'docker cp regression$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/', - 'docker cp regression$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/', - 'docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/' + result + '/', - 'ls -l /drone/src/' + result, - 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"', - 'docker stop regression$${DRONE_BUILD_NUMBER} && docker rm regression$${DRONE_BUILD_NUMBER} || echo "cleanup regression failure"', + reportTestStage(getContainerName("regression"), result, "regression"), ], when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, }, dockerfile:: { - name: 'dockerfile', - depends_on: ['publish pkg', 'publish cmapi build'], + name: "dockerfile", + depends_on: ["publish pkg", "publish cmapi build"], //failure: 'ignore', - image: 'alpine/git', + image: "alpine/git", environment: { - DOCKER_BRANCH_REF: '${DRONE_SOURCE_BRANCH}', + DOCKER_BRANCH_REF: "${DRONE_SOURCE_BRANCH}", DOCKER_REF_AUX: branch_ref, }, commands: [ @@ -471,317 +490,291 @@ local Pipeline(branch, platform, event, arch='amd64', server='0c6b89c694a88aae76 'echo "$$DOCKER_REF"', // DOCKER_REF can be empty if there is no appropriate branch in docker repository. // assign what is appropriate by default. - 'export DOCKER_REF=$${DOCKER_REF:-$$DOCKER_REF_AUX}', + "export DOCKER_REF=$${DOCKER_REF:-$$DOCKER_REF_AUX}", 'echo "$$DOCKER_REF"', - 'git clone --branch $$DOCKER_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker', - 'touch docker/.secrets', + "git clone --branch $$DOCKER_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker", + "touch docker/.secrets", ], }, dockerhub:: { - name: 'dockerhub', - depends_on: ['dockerfile'], + name: "dockerhub", + depends_on: ["dockerfile"], //failure: 'ignore', - image: 'plugins/docker', + image: "plugins/docker", environment: { VERSION: container_version, - MCS_REPO: 'columnstore', - DEV: 'true', + MCS_REPO: "columnstore", + DEV: "true", // branchp has slash if not empty - MCS_BASEURL: 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/', - CMAPI_REPO: 'cmapi', - CMAPI_BASEURL: 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/', + MCS_BASEURL: "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/" + result + "/", + CMAPI_REPO: "cmapi", + CMAPI_BASEURL: "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/" + result + "/", }, settings: { - repo: 'mariadb/enterprise-columnstore-dev', - context: 'docker', - dockerfile: 'docker/Dockerfile', - build_args_from_env: ['VERSION', 'MCS_REPO', 'MCS_BASEURL', 'CMAPI_REPO', 'CMAPI_BASEURL', 'DEV'], + repo: "mariadb/enterprise-columnstore-dev", + context: "docker", + dockerfile: "docker/Dockerfile", + build_args_from_env: ["VERSION", "MCS_REPO", "MCS_BASEURL", "CMAPI_REPO", "CMAPI_BASEURL", "DEV"], tags: container_tags, username: { - from_secret: 'dockerhub_user', + from_secret: "dockerhub_user", }, password: { - from_secret: 'dockerhub_password', + from_secret: "dockerhub_password", }, }, }, - cmapipython:: { - name: 'cmapi python', - image: img, - volumes: [pipeline._volumes.mdb], - environment: { - PYTHON_URL_AMD64: 'https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst', - PYTHON_URL_ARM64: 'https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst', - }, - commands: [ - 'cd cmapi', - '%s install -y wget zstd findutils gcc' % if (pkg_format == 'rpm') then 'yum install -y epel-release && yum makecache && yum ' else 'apt update && apt', - 'wget -qO- $${PYTHON_URL_' + std.asciiUpper(arch) + '} | tar --use-compress-program=unzstd -xf - -C ./', - 'mv python pp && mv pp/install python', - 'chown -R root:root python', - if (platform == 'rockylinux:9') then 'yum install -y libxcrypt-compat', - if (arch == 'arm64') then 'export CC=gcc', - 'python/bin/pip3 install -t deps --only-binary :all -r requirements.txt', - './cleanup.sh', - 'cp cmapi_server/cmapi_server.conf cmapi_server/cmapi_server.conf.default', - ], - }, - cmapibuild:: { - name: 'cmapi build', - depends_on: ['cmapi python'], - image: img, - volumes: [pipeline._volumes.mdb], - environment: { - DEBIAN_FRONTEND: 'noninteractive', - }, - commands: [ - 'cd cmapi', - if (pkg_format == 'rpm') then 'yum install -y cmake make rpm-build libarchive createrepo findutils' else 'apt update && apt install --no-install-recommends -y cmake make dpkg-dev', - './cleanup.sh', - 'cmake -D' + std.asciiUpper(pkg_format) + '=1 . && make package', - 'mkdir ./' + result, - 'mv -v *.%s ./%s/' % [pkg_format, result], - if (pkg_format == 'rpm') then 'createrepo ./' + result else 'dpkg-scanpackages %s | gzip > ./%s/Packages.gz' % [result, result], - 'mkdir /drone/src/' + result, - 'yes | cp -vr ./%s /drone/src/' % result, - ], - }, cmapitest:: { - name: 'cmapi test', - depends_on: ['publish cmapi build', 'smoke'], - image: 'docker:git', - volumes: [pipeline._volumes.docker], + name: "cmapi test", + depends_on: ["publish cmapi build"], + image: "docker:git", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], environment: { - PYTHONPATH: '/usr/share/columnstore/cmapi/deps', + PYTHONPATH: "/usr/share/columnstore/cmapi/deps", }, commands: [ - 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env PYTHONPATH=$${PYTHONPATH} --name cmapi$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', - if (pkg_format == 'rpm') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y iproute sudo epel-release which rsyslog hostname procps-ng"' else 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y iproute2 rsyslog hostname procps sudo"', - if (pkg_format == 'deb') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', - if (platform == 'rockylinux:9') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y libxcrypt-compat"', - 'docker cp setup-repo.sh cmapi$${DRONE_BUILD_NUMBER}:/', - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} /setup-repo.sh', - if (pkg_format == 'deb') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore mariadb-columnstore-cmapi"' else 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine MariaDB-columnstore-cmapi"', - 'cd cmapi', - 'for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/$${i}/; done', - 'docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/', - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} systemctl start mariadb-columnstore-cmapi', + prepareTestStage(getContainerName("cmapi"), result, true), + installCmapi(getContainerName("cmapi"), pkg_format), + "cd cmapi", + "for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/$${i}/; done", + "docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/", + execInnerDocker("systemctl start mariadb-columnstore-cmapi", getContainerName("cmapi")), // set API key to /etc/columnstore/cmapi_server.conf - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "mcs cluster set api-key --key somekey123"', + execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', getContainerName("cmapi")), // copy cmapi conf file for test purposes (there are api key already set inside) - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} systemctl stop mariadb-columnstore-cmapi', - 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', + execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], getContainerName("cmapi")), + execInnerDocker("systemctl stop mariadb-columnstore-cmapi", getContainerName("cmapi")), + execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', getContainerName("cmapi")), ], }, + cmapilog:: { + name: "cmapilog", + depends_on: ["cmapi test"], + image: "docker", + volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], + commands: [ + reportTestStage(getContainerName("cmapi"), result, "cmapi"), + ], + when: { + status: ["success", "failure"], + }, + }, multi_node_mtr:: { - name: 'mtr', - depends_on: ['dockerhub'], + name: "mtr", + depends_on: ["dockerhub"], //failure: 'ignore', - image: 'docker', + image: "docker", volumes: [pipeline._volumes.docker], environment: { DOCKER_LOGIN: { - from_secret: 'dockerhub_user', + from_secret: "dockerhub_user", }, DOCKER_PASSWORD: { - from_secret: 'dockerhub_password', + from_secret: "dockerhub_password", }, - MCS_IMAGE_NAME: 'mariadb/enterprise-columnstore-dev:' + container_tags[0], + MCS_IMAGE_NAME: "mariadb/enterprise-columnstore-dev:" + container_tags[0], }, commands: [ - 'echo $$DOCKER_PASSWORD | docker login --username $$DOCKER_LOGIN --password-stdin', - 'cd docker', - 'cp .env_example .env', + "echo $$DOCKER_PASSWORD | docker login --username $$DOCKER_LOGIN --password-stdin", + "cd docker", + "cp .env_example .env", 'sed -i "/^MCS_IMAGE_NAME=/s/=.*/=${MCS_IMAGE_NAME}/" .env', 'sed -i "/^MAXSCALE=/s/=.*/=false/" .env', - 'docker-compose up -d', - 'docker exec mcs1 provision', - 'docker cp ../mysql-test/columnstore mcs1:' + mtr_path + '/suite/', - 'docker exec -t mcs1 chown mysql:mysql -R ' + mtr_path, + "docker-compose up -d", + "docker exec mcs1 provision mcs1 mcs2 mcs3", + "docker cp ../mysql-test/columnstore mcs1:" + mtr_path + "/suite/", + "docker exec -t mcs1 chown mysql:mysql -R " + mtr_path, 'docker exec -t mcs1 mariadb -e "create database if not exists test;"', // delay for manual debugging on live instance - 'sleep $${COMPOSE_DELAY_SECONDS:-1s}', - 'docker exec -t mcs1 bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"', + "sleep $${COMPOSE_DELAY_SECONDS:-1s}", + 'docker exec -t mcs1 bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"', ], }, - kind: 'pipeline', - type: 'docker', - name: std.join(' ', [branch, platform, event, arch, server]), + kind: "pipeline", + type: "docker", + name: std.join(" ", [branch, platform, event, arch, server, customBootstrapParams, customBuildEnvCommandsMapKey]), platform: { arch: arch }, // [if arch == 'arm64' then 'node']: { arch: 'arm64' }, clone: { depth: 10 }, steps: [ { - name: 'submodules', - image: 'alpine/git', + name: "submodules", + image: "alpine/git", commands: [ - 'git submodule update --init --recursive', - 'git config cmake.update-submodules no', - 'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD', + "git submodule update --init --recursive", + "git config cmake.update-submodules no", + "git rev-parse --abbrev-ref HEAD && git rev-parse HEAD", ], }, { - name: 'clone-mdb', - image: 'alpine/git', + name: "clone-mdb", + image: "alpine/git", volumes: [pipeline._volumes.mdb], environment: { - SERVER_REF: '${SERVER_REF:-' + 'bb-10.6.14-9-cs-23.02.10-1' + '}', - SERVER_REMOTE: '${SERVER_REMOTE:-' + server_remote + '}', - // server points to the commit in bb-10.6.14-9-cs-23.02.10-1 of ES - SERVER_SHA: '${SERVER_SHA:-' + '0c6b89c694a88aae764b152045bf5bdb234d7edf' + '}', + SERVER_REF: "${SERVER_REF:-" + server + "}", + SERVER_REMOTE: "${SERVER_REMOTE:-" + server_remote + "}", + SERVER_SHA: "${SERVER_SHA:-" + server + "}", }, - commands: [ - 'echo $$SERVER_REF', - 'echo $$SERVER_REMOTE', - 'mkdir -p /mdb/' + builddir + ' && cd /mdb/' + builddir, + commands: echo_running_on + + [ + "echo $$SERVER_REF", + "echo $$SERVER_REMOTE", + "mkdir -p /mdb/" + builddir + " && cd /mdb/" + builddir, 'git config --global url."https://github.com/".insteadOf git@github.com:', 'git -c submodule."storage/rocksdb/rocksdb".update=none -c submodule."wsrep-lib".update=none -c submodule."storage/columnstore/columnstore".update=none clone --recurse-submodules --depth 200 --branch $$SERVER_REF $$SERVER_REMOTE .', - 'git reset --hard $$SERVER_SHA', - 'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD', - 'git config cmake.update-submodules no', - 'rm -rf storage/columnstore/columnstore', - 'cp -r /drone/src /mdb/' + builddir + '/storage/columnstore/columnstore', + "git reset --hard $$SERVER_SHA", + "git rev-parse --abbrev-ref HEAD && git rev-parse HEAD", + "git config cmake.update-submodules no", + "rm -rf storage/columnstore/columnstore", + "cp -r /drone/src /mdb/" + builddir + "/storage/columnstore/columnstore", ], }, { - name: 'build', - depends_on: ['clone-mdb'], + name: "build", + depends_on: ["clone-mdb"], image: img, volumes: [pipeline._volumes.mdb], environment: { - DEBIAN_FRONTEND: 'noninteractive', - DEB_BUILD_OPTIONS: 'parallel=4', - DH_BUILD_DDEBS: '1', - BUILDPACKAGE_FLAGS: '-b', // Save time and produce only binary packages, not source + DEBIAN_FRONTEND: "noninteractive", AWS_ACCESS_KEY_ID: { - from_secret: 'aws_access_key_id', + from_secret: "aws_access_key_id", }, AWS_SECRET_ACCESS_KEY: { - from_secret: 'aws_secret_access_key', + from_secret: "aws_secret_access_key", }, - SCCACHE_BUCKET: 'cs-sccache', - SCCACHE_S3_USE_SSL: 'true', - SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + '${DRONE_PULL_REQUEST}', + SCCACHE_BUCKET: "cs-sccache", + SCCACHE_REGION: "us-east-1", + SCCACHE_S3_USE_SSL: "true", + SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + "${DRONE_PULL_REQUEST}", //SCCACHE_ERROR_LOG: '/tmp/sccache_log.txt', //SCCACHE_LOG: 'debug', }, commands: [ - 'cd /mdb/' + builddir, - 'mkdir ' + result, - "sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh", - if (std.startsWith(server, '10.6')) then "sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control", - // Remove Debian build flags that could prevent ColumnStore from building - "sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules", - // Disable dh_missing strict check for missing files - 'sed -i s/--fail-missing/--list-missing/ debian/rules', - // Tweak debian packaging stuff - 'for i in mariadb-backup mariadb-plugin libmariadbd; do sed -i "/Package: $i.*/,/^$/d" debian/control; done', - "sed -i 's/Depends: galera.*/Depends:/' debian/control", - 'for i in galera wsrep ha_sphinx embedded; do sed -i /$i/d debian/*.install; done', - // Install build dependencies for deb - if (pkg_format == 'deb') then "apt-cache madison liburing-dev | grep liburing-dev || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules && apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules && sed '/libfmt-dev/d' -i debian/control", - // Change plugin_maturity level - // "sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt", - if (pkg_format == 'deb') then 'apt update -y && apt install -y curl' else if (platform == 'rockylinux:9') then 'yum install -y curl-minimal' else 'yum install -y curl', - get_sccache, - testPreparation(platform), - // disable LTO for 22.04 for now - if (platform == 'ubuntu:22.04') then 'apt install -y lto-disabled-list && for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do echo "$i any" >> /usr/share/lto-disabled-list/lto-disabled-list; done && grep mariadb /usr/share/lto-disabled-list/lto-disabled-list', - platformMap(platform, arch), - 'sccache --show-stats', - // move engine and cmapi packages to one dir to make a repo - 'mv -v -t ./%s/ %s/*.%s /drone/src/cmapi/%s/*.%s ' % [result, if (pkg_format == 'rpm') then '.' else '..', pkg_format, result, pkg_format], - if (pkg_format == 'rpm') then 'createrepo ./' + result else 'dpkg-scanpackages %s | gzip > ./%s/Packages.gz' % [result, result], - // list storage manager binary - 'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager', - ], + "mkdir /mdb/" + builddir + "/" + result, + ] + + get_sccache + + customEnvCommands(customBuildEnvCommandsMapKey, builddir) + + [ + 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + + "--build-type RelWithDebInfo " + + "--distro " + platform + " " + + "--build-packages --install-deps --sccache -g " + + " " + customBootstrapParams + + " " + customBootstrapParamsForExisitingPipelines(platform) + " | " + + "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " + + "/mdb/" + builddir + "/" + result + '/build.log"', + "sccache --show-stats", + ], }, { - name: 'unittests', - depends_on: ['build'], + name: "cmapi build", + depends_on: ["clone-mdb"], image: img, volumes: [pipeline._volumes.mdb], environment: { - DEBIAN_FRONTEND: 'noninteractive', + DEBIAN_FRONTEND: "noninteractive", }, commands: [ - 'cd /mdb/' + builddir, + "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/build_cmapi.sh --distro " + platform + " --arch " + arch, + ], + }, + { + name: "createrepo", + depends_on: ["build", "cmapi build"], + image: img, + when: { + status: ["success", "failure"], + }, + volumes: [pipeline._volumes.mdb], + commands: [ + "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/createrepo.sh --result " + result, + ], + }, + { + name: "unittests", + depends_on: ["createrepo"], + image: img, + volumes: [pipeline._volumes.mdb], + environment: { + DEBIAN_FRONTEND: "noninteractive", + }, + commands: [ + "cd /mdb/" + builddir, testPreparation(platform), testRun(platform), ], }, { - name: 'pkg', - depends_on: ['unittests'], - image: 'alpine/git', + name: "pkg", + depends_on: ["unittests"], + image: "alpine/git", when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, volumes: [pipeline._volumes.mdb], + environment: { + SERVER_REF: "${SERVER_REF:-" + server + "}", + SERVER_REMOTE: "${SERVER_REMOTE:-" + server_remote + "}", + }, commands: [ - 'cd /mdb/' + builddir, + "cd /mdb/" + builddir, 'echo "engine: $DRONE_COMMIT" > buildinfo.txt', 'echo "server: $$(git rev-parse HEAD)" >> buildinfo.txt', 'echo "buildNo: $DRONE_BUILD_NUMBER" >> buildinfo.txt', - 'mv buildinfo.txt ./%s/' % result, - 'yes | cp -vr ./%s/. /drone/src/%s/' % [result, result], - 'ls -l /drone/src/' + result, + 'echo "serverBranch: $$SERVER_REF" >> buildinfo.txt', + 'echo "serverRepo: $$SERVER_REMOTE" >> buildinfo.txt', + 'echo "engineBranch: $DRONE_SOURCE_BRANCH" >> buildinfo.txt', + 'echo "engineRepo: https://github.com/$DRONE_REPO" >> buildinfo.txt', + "mv buildinfo.txt ./%s/" % result, + "yes | cp -vr ./%s/. /drone/src/%s/" % [result, result], + "ls -l /drone/src/" + result, 'echo "check columnstore package:"', - 'ls -l /drone/src/%s | grep columnstore' % result, + "ls -l /drone/src/%s | grep columnstore" % result, ], }, ] + - [pipeline.cmapipython] + [pipeline.cmapibuild] + - [pipeline.publish('cmapi build')] + + [pipeline.publish("cmapi build")] + [pipeline.publish()] + - [ - { - name: 'publish pkg url', - depends_on: ['publish pkg'], - image: 'alpine/git', - commands: [ - "echo -e '\\e]8;;" + publish_pkg_url + '\\e\\\\' + publish_pkg_url + "\\e]8;;\\e\\\\'", - ], - }, - ] + - (if (event == 'cron') then [pipeline.publish('pkg latest', 'latest')] else []) + + (if (event == "cron") then [pipeline.publish("pkg latest", "latest")] else []) + [pipeline.smoke] + [pipeline.smokelog] + - [pipeline.publish('smokelog')] + + [pipeline.publish("smokelog")] + [pipeline.cmapitest] + [pipeline.cmapilog] + - (if (platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish('mtr')] + [pipeline.mtrlog] + [pipeline.publish('mtrlog')]) + - (if (event == 'cron' && platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.publish('mtr latest', 'latest')] else []) + + [pipeline.publish("cmapilog")] + + (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) + [pipeline.prepare_regression] + - [pipeline.regression(regression_tests[i], [if (i == 0) then 'prepare regression' else regression_tests[i - 1]]) for i in indexes(regression_tests)] + + [pipeline.regression(regression_tests[i], [if (i == 0) then "prepare regression" else regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + - [pipeline.publish('regressionlog')] + - (if (event == 'cron') then [pipeline.publish('regressionlog latest', 'latest')] else []), + [pipeline.publish("regressionlog")] + + // [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + + // (if (std.length(mdb_server_versions) == 0) then [] else [pipeline.upgradelog] + [pipeline.publish("upgradelog")]) + + (if (event == "cron") then [pipeline.publish("regressionlog latest", "latest")] else []), - volumes: [pipeline._volumes.mdb { temp: {} }, pipeline._volumes.docker { host: { path: '/var/run/docker.sock' } }], + volumes: [pipeline._volumes.mdb { temp: {} }, pipeline._volumes.docker { host: { path: "/var/run/docker.sock" } }], trigger: { event: [event], branch: [branch], - } + (if event == 'cron' then { - cron: ['nightly-' + std.strReplace(branch, '.', '-')], - } else {}), + }, + // + (if event == 'cron' then { + // cron: ['nightly-' + std.strReplace(branch, '.', '-')], + // } else {}), }; local FinalPipeline(branch, event) = { - kind: 'pipeline', - name: std.join(' ', ['after', branch, event]), + kind: "pipeline", + name: std.join(" ", ["after", branch, event]), steps: [ { - name: 'notify', - image: 'plugins/slack', + name: "notify", + image: "plugins/slack", settings: { webhook: { - from_secret: 'slack_webhook', + from_secret: "slack_webhook", }, - template: '*' + event + (if event == 'pull_request' then ' ' else '') + - ' build <{{build.link}}|{{build.number}}> {{#success build.status}}succeeded{{else}}failed{{/success}}*.\n\n*Branch*: \n*Commit*: {{truncate build.message.title 100 }}\n*Author*: {{ build.author }}\n*Duration*: {{since build.started}}\n*Artifacts*: https://cspkg.s3.amazonaws.com/index.html?prefix={{build.branch}}/{{build.event}}/{{build.number}}', + template: "*" + event + (if event == "pull_request" then " " else "") + + " build <{{build.link}}|{{build.number}}> {{#success build.status}}succeeded{{else}}failed{{/success}}*.\n\n*Branch*: \n*Commit*: {{truncate build.message.title 100 }}\n*Author*: {{ build.author }}\n*Duration*: {{since build.started}}\n*Artifacts*: https://cspkg.s3.amazonaws.com/index.html?prefix={{build.branch}}/{{build.event}}/{{build.number}}", }, }, ], @@ -789,23 +782,23 @@ local FinalPipeline(branch, event) = { event: [event], branch: [branch], status: [ - 'success', - 'failure', + "success", + "failure", ], - } + (if event == 'cron' then { cron: ['nightly-' + std.strReplace(branch, '.', '-')] } else {}), - depends_on: std.map(function(p) std.join(' ', [branch, p, event, 'amd64', '10.6-enterprise']), platforms.develop) + - std.map(function(p) std.join(' ', [branch, p, event, 'arm64', '10.6-enterprise']), platforms_arm.develop), + } + (if event == "cron" then { cron: ["nightly-" + std.strReplace(branch, ".", "-")] } else {}), + depends_on: std.map(function(p) std.join(" ", [branch, p, event, "amd64", "10.6-enterprise", "", ""]), platforms.develop) + + std.map(function(p) std.join(" ", [branch, p, event, "arm64", "10.6-enterprise", "", ""]), platforms_arm.develop), }; [ - Pipeline(b, p, e, 'amd64', s) + Pipeline(b, p, e, "amd64", s) for b in std.objectFields(platforms) for p in platforms[b] for s in servers[b] for e in events ] + [ - Pipeline(b, p, e, 'arm64', s) + Pipeline(b, p, e, "arm64", s) for b in std.objectFields(platforms_arm) for p in platforms_arm[b] for s in servers[b] @@ -813,15 +806,25 @@ local FinalPipeline(branch, event) = { ] + [ - FinalPipeline(b, 'cron') + FinalPipeline(b, "cron") for b in std.objectFields(platforms) ] + [ - Pipeline(any_branch, p, 'custom', 'amd64', '10.6-enterprise') + Pipeline(any_branch, p, "custom", "amd64", "10.6-enterprise") for p in platforms_custom ] + [ - Pipeline(any_branch, p, 'custom', 'arm64', '10.6-enterprise') + Pipeline(any_branch, p, "custom", "arm64", "10.6-enterprise") for p in platforms_arm_custom ] ++ +[ + Pipeline(b, platform, triggeringEvent, a, server, "", buildenv) + for a in ["amd64"] + for b in std.objectFields(platforms) + for platform in ["ubuntu:24.04"] + for buildenv in std.objectFields(customEnvCommandsMap) + for triggeringEvent in events + for server in servers.develop +] diff --git a/CMakeLists.txt b/CMakeLists.txt index 58f875ce5..093fb1f93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,440 +1,87 @@ -if (CMAKE_VERSION VERSION_LESS 3.12.0) - MESSAGE_ONCE(CS_CMAKE "ColumnStore needs cmake >= 3.12") - RETURN() -ENDIF() +cmake_minimum_required(VERSION 3.13) -CMAKE_MINIMUM_REQUIRED(VERSION 3.12) +project(Columnstore) -PROJECT(Columnstore) -INCLUDE (CheckCXXSourceCompiles) - -# MariaDB server adds this flags by default for Debug builds, and this makes our external projects -# ABI incompatible with debug STL containers -STRING(REPLACE -D_GLIBCXX_DEBUG "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) -STRING(REPLACE -D_GLIBCXX_ASSERTIONS "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) - - -FOREACH(p CMP0135) - IF(POLICY ${p}) - CMAKE_POLICY(SET ${p} NEW) - ENDIF() -ENDFOREACH() - -IF(NOT INSTALL_LAYOUT) - IF(NOT CMAKE_BUILD_TYPE) - SET(CMAKE_BUILD_TYPE RELWITHDEBINFO CACHE STRING - "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) - ENDIF(NOT CMAKE_BUILD_TYPE) - SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -ENDIF() - -if(SERVER_BUILD_DIR) - if (NOT IS_ABSOLUTE ${SERVER_BUILD_DIR}) - set(SERVER_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SERVER_BUILD_DIR}) +foreach(p CMP0135 CMP0077) + if(POLICY ${p}) + cmake_policy(SET ${p} NEW) endif() - if(NOT EXISTS ${SERVER_BUILD_DIR}/CMakeCache.txt) - message(FATAL_ERROR "SERVER_BUILD_DIR parameter supplied but CMakeCache.txt not found in ${SERVER_BUILD_DIR}") - endif() - load_cache("${SERVER_BUILD_DIR}" READ_WITH_PREFIX SERVER_ MySQL_SOURCE_DIR MySQL_BINARY_DIR CMAKE_BUILD_TYPE CMAKE_INSTALL_PREFIX) +endforeach() - set(SERVER_BUILD_INCLUDE_DIR "${SERVER_MySQL_BINARY_DIR}/include" CACHE PATH "Location of server build include folder" FORCE) - set(SERVER_SOURCE_ROOT_DIR "${SERVER_MySQL_SOURCE_DIR}" CACHE PATH "Location of the server source folder" FORCE) - set(CMAKE_INSTALL_PREFIX "${SERVER_CMAKE_INSTALL_PREFIX}" CACHE PATH "Installation prefix" FORCE) - set(CMAKE_BUILD_TYPE ${SERVER_CMAKE_BUILD_TYPE} CACHE STRING "Build configuration type" FORCE) -endif() +set(PACKAGE columnstore) +set(PACKAGE_NAME columnstore) +set(PACKAGE_TARNAME columnstore) +set(PACKAGE_BUGREPORT support@mariadb.com) +set(PACKAGE_URL "") +set(PACKAGE_STRING columnstore-${PACKAGE_VERSION}) -SET_PROPERTY(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external) -LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) +set_property(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) -INCLUDE(check_compiler_flag) -INCLUDE(ExternalProject) -INCLUDE(CheckCXXSourceCompiles) +set(CMAKE_CXX_EXTENSIONS FALSE) +set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) +set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) -FIND_PACKAGE(Boost 1.84.0 COMPONENTS chrono filesystem program_options regex system thread) +# take a note, that install scripts are and services configurations are located oam/install_scripts/CMakeLists.txt -IF (NOT Boost_FOUND OR Boost_VERSION VERSION_GREATER 1.76.99) - MY_CHECK_CXX_COMPILER_FLAG("-std=c++20") -ENDIF() +set(WITH_COLUMNSTORE_LZ4 + AUTO + CACHE STRING "Build with lz4. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'" +) -IF (have_CXX__std_c__20) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20") -ELSE() - MY_CHECK_CXX_COMPILER_FLAG("-std=c++2a") - IF (have_CXX__std_c__2a) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a") - ELSE() - MESSAGE_ONCE(CS_NO_CXX20 "C++ Compiler does not understand -std=c++20") - return() - ENDIF() -ENDIF() +# order of includes is important +include(check_compiler_flag) +include(ExternalProject) +include(CheckCXXSourceCompiles) +include(packages) +include(boost) +include(thrift) +include(dirs) +include(includes) +include(libs) +include(ColumnstoreLibrary) +include(columnstore_version) +include(configureEngine) +include(compiler_flags) +include(misc) -# There is an inconsistency b/w default char signedness at ARM and x86. -# This flag explicitly sets char as signed. -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsigned-char") +set(COMPONENTS + dbcon/mysql + utils + oam/oamcpp + dbcon/execplan + dbcon/joblist + versioning + oam + writeengine/wrapper + writeengine/client + writeengine/xml + writeengine/redistribute + dbcon/ddlpackage + dbcon/ddlpackageproc + dbcon/dmlpackage + dbcon/dmlpackageproc + ddlproc + dmlproc + oamapps + primitives + tools + writeengine/server + writeengine/bulk + writeengine/splitter + storage-manager + datatypes + tests +) -UNSET(CMAKE_CXX_STANDARD) -SET(CMAKE_CXX_EXTENSIONS FALSE) -SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) -SET(CMAKE_POSITION_INDEPENDENT_CODE TRUE) -SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) -SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) -SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) -SET(WITH_COLUMNSTORE_LZ4 AUTO CACHE STRING "Build with lz4. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") +foreach(component ${COMPONENTS}) + add_subdirectory(${component}) +endforeach() -SET (ENGINE_SYSCONFDIR "/etc") -SET (ENGINE_DATADIR "/var/lib/columnstore") -SET (ENGINE_LOGDIR "/var/log/mariadb/columnstore") - -IF (INSTALL_LAYOUT) - # We are building from MariaDB server submodule if this is set - SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR}") - IF (INSTALL_SYSCONF2DIR) - SET(MARIADB_MYCNFDIR "${INSTALL_SYSCONF2DIR}") - ELSE () - SET(MARIADB_MYCNFDIR "/etc/mysql") - ENDIF () - SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR}") - SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR}") - SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") - SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR}") - SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR}") - SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") -ELSE () - IF (RPM) - SET(MARIADB_PLUGINDIR "/usr/lib64/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/my.cnf.d") - SET (ENGINE_LIBDIR "/usr/lib64") - SET (ENGINE_BINDIR "/usr/bin") - SET (ENGINE_INCDIR "/usr/include") - SET (ENGINE_MANDIR "/usr/share/man") - SET (ENGINE_SBINDIR "/usr/sbin") - SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") - ELSEIF (DEB) - SET(MARIADB_PLUGINDIR "/usr/lib/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/mysql/conf.d") - SET (ENGINE_LIBDIR "/usr/lib") - SET (ENGINE_BINDIR "/usr/bin") - SET (ENGINE_INCDIR "/usr/include") - SET (ENGINE_MANDIR "/usr/share/man") - SET (ENGINE_SBINDIR "/usr/sbin") - SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") - ELSE () - # TODO: prefix should probably apply here - SET(MARIADB_PLUGINDIR "/usr/local/lib/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/mysql") - SET (ENGINE_LIBDIR "/usr/local/lib") - SET (ENGINE_BINDIR "/usr/local/bin") - SET (ENGINE_INCDIR "/usr/local/include") - SET (ENGINE_MANDIR "/usr/local/man") - SET (ENGINE_SBINDIR "/usr/local/sbin") - SET (ENGINE_SUPPORTDIR "/usr/local/share/columnstore") - ENDIF () -ENDIF () - -SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - -INCLUDE(columnstore_version) -INCLUDE(misc) -INCLUDE(boost) -INCLUDE(thrift) - -FIND_PACKAGE(BISON) -IF (NOT BISON_FOUND) - MESSAGE_ONCE(CS_NO_BISON "bison not found!") - return() -ENDIF() - -check_cxx_source_compiles("#include \n void main(){}" HAS_STD_FILESYSTEM) -check_cxx_source_compiles("#include \n void main(){}" HAS_STD_EXPERIMENTAL_FILESYSTEM) - -SET (PACKAGE columnstore) -SET (PACKAGE_NAME columnstore) -SET (PACKAGE_TARNAME columnstore) -SET (PACKAGE_BUGREPORT support@mariadb.com) -SET (PACKAGE_URL "") - -SET (PACKAGE_STRING columnstore-${PACKAGE_VERSION}) - -INCLUDE (configureEngine) - -FIND_PROGRAM(LEX_EXECUTABLE flex DOC "path to the flex executable") -if(NOT LEX_EXECUTABLE) - FIND_PROGRAM(LEX_EXECUTABLE lex DOC "path to the lex executable") - if(NOT LEX_EXECUTABLE) - MESSAGE_ONCE(CS_NO_LEX "flex/lex not found!") - return() - endif() -endif() - -FIND_PACKAGE(LibXml2) - -if (NOT LIBXML2_FOUND) - MESSAGE_ONCE(CS_NO_LIBXML "Could not find a usable libxml2 development environment!") - return() -endif() - -find_package(Snappy) -if (NOT SNAPPY_FOUND) - MESSAGE_ONCE(CS_NO_SNAPPY "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") - return() -endif() - -FIND_PACKAGE(CURL) -if (NOT CURL_FOUND) - MESSAGE_ONCE(CS_NO_CURL "libcurl development headers not found") - return() -endif() - -FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") -if(NOT AWK_EXECUTABLE) - MESSAGE_ONCE(CS_NO_AWK "awk not found!") - return() -endif() - -SET(HAVE_LZ4 0 CACHE INTERNAL "") -IF (WITH_COLUMNSTORE_LZ4 STREQUAL "ON" OR WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") - FIND_PACKAGE(LZ4) - IF (NOT LZ4_FOUND) - IF (WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") - MESSAGE_ONCE(CS_LZ4 "LZ4 not found, building without LZ4") - ELSE() - MESSAGE(FATAL_ERROR "LZ4 not found.") - ENDIF() - ELSE() - MESSAGE_ONCE(CS_LZ4 "Building with LZ4") - SET(HAVE_LZ4 1 CACHE INTERNAL "") - ENDIF() -ELSE() - MESSAGE_ONCE(CS_LZ4 "Building without LZ4") -ENDIF() - -IF (NOT INSTALL_LAYOUT) - MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) - MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-DBOOST_BIND_GLOBAL_PLACEHOLDERS" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) - - # enable security hardening features, like most distributions do - # in our benchmarks that costs about ~1% of performance, depending on the load - IF(CMAKE_C_COMPILER_VERSION VERSION_LESS "4.6") - SET(security_default OFF) - ELSE() - SET(security_default ON) - ENDIF() - OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default}) - OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF) - IF(SECURITY_HARDENED AND NOT WITH_ASAN AND NOT WITH_UBSAN AND NOT WITH_TSAN AND NOT WITH_GPROF) - # security-enhancing flags - MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector --param=ssp-buffer-size=4") - MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO) - MY_CHECK_AND_SET_COMPILER_FLAG("-fexceptions") - IF(SECURITY_HARDENED_NEW) - MY_CHECK_AND_SET_COMPILER_FLAG("-mcet -fcf-protection") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector-strong") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-clash-protection") - ENDIF() - ENDIF() -ELSE () - # Remove visibility flag for now as it breaks Ubuntu 18.05 and we need to - # fix our libraries anyway - STRING(REPLACE "-fvisibility=hidden" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-D_DEBUG -O0" DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-vla" DEBUG) -ENDIF() - -IF (MASK_LONGDOUBLE) - MY_CHECK_AND_SET_COMPILER_FLAG("-DMASK_LONGDOUBLE") -ENDIF() - -if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-enum-enum-conversion -Wno-register -Wno-typedef-redefinition") -elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-register") -endif() - -IF (WITH_COLUMNSTORE_ASAN) - MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) - MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") -ENDIF() - -MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-copy" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) -MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-declarations" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) - -MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall -Wextra") -SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") -SET (ENGINE_DT_LIB datatypes) -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot boost_thread xml2 pthread rt ${ENGINE_DT_LIB}) -SET (ENGINE_OAM_LIBS oamcpp) -SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) -SET (PLUGIN_EXEC_LIBS execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querytele threadpool ${ENGINE_BRM_LIBS}) -SET (ENGINE_EXEC_LIBS joblist querystats libmysql_client ${PLUGIN_EXEC_LIBS}) -SET (PLUGIN_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage writeengine writeengineclient idbdatafile cacheutils) -SET (ENGINE_WRITE_LIBS ${PLUGIN_WRITE_LIBS} ${ENGINE_EXEC_LIBS}) - -IF (NOT INSTALL_LAYOUT) - SET (LIBMARIADB_BININC_DIR ${SERVER_BUILD_DIR}/libmariadb/include) - SET (LIBMARIADB_SRCINC_DIR ${SERVER_SOURCE_ROOT_DIR}/libmariadb/include) -ELSE() - SET (LIBMARIADB_BININC_DIR ${CMAKE_BINARY_DIR}/libmariadb/include) - SET (LIBMARIADB_SRCINC_DIR ${CMAKE_SOURCE_DIR}/libmariadb/include) -ENDIF() - -SET (ENGINE_COMMON_LDFLAGS "") - -IF (SERVER_BUILD_INCLUDE_DIR) - IF (NOT IS_ABSOLUTE ${SERVER_BUILD_INCLUDE_DIR}) - SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR}) - ENDIF() -ELSEIF (INSTALL_LAYOUT) - SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include) -ELSE() - SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include) -ENDIF() - -IF (SERVER_SOURCE_ROOT_DIR) - IF (NOT IS_ABSOLUTE ${SERVER_SOURCE_ROOT_DIR}) - SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR}) - ENDIF() -ELSEIF (INSTALL_LAYOUT) - SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR}) -ELSE() - SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..) -ENDIF() - -IF (NOT SERVER_BUILD_DIR) - SET (SERVER_BUILD_DIR ${SERVER_SOURCE_ROOT_DIR}) -ENDIF() - -MESSAGE_ONCE(SERVER_BUILD_INCLUDE_DIR "SERVER_BUILD_INCLUDE_DIR = ${SERVER_BUILD_INCLUDE_DIR}") -MESSAGE_ONCE(SERVER_SOURCE_ROOT_DIR "SERVER_SOURCE_ROOT_DIR = ${SERVER_SOURCE_ROOT_DIR}") - -IF (INSTALL_LAYOUT) - SET (MARIADB_CLIENT_LIBS libmariadb) -ELSE() - SET (MARIADB_CLIENT_LIBS -L${SERVER_BUILD_INCLUDE_DIR}/../libmariadb/libmariadb/ libmariadb.so) -ENDIF() - -IF (INSTALL_LAYOUT) - SET (MARIADB_STRING_LIBS dbug strings mysys) -ELSE() - SET (MARIADB_STRING_LIBS -L${SERVER_BUILD_INCLUDE_DIR}/../mysys/ libmysys.a -L${SERVER_BUILD_INCLUDE_DIR}/../strings/ libstrings.a -L${SERVER_BUILD_INCLUDE_DIR}/../dbug/ libdbug.a) -ENDIF() - -#SET (ENGINE_UTILS_BOOSTIDB_INCLUDE "{CMAKE_CURRENT_SOURCE_DIR}/utils/boost_idb") -SET (ENGINE_UTILS_MESSAGEQCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/messageqcpp") -SET (ENGINE_WE_SHARED_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/shared") -SET (ENGINE_UTILS_IDBDATAFILE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/idbdatafile") -SET (ENGINE_UTILS_LOGGINGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/loggingcpp" "${CMAKE_CURRENT_BINARY_DIR}/utils/loggingcpp") -SET (ENGINE_UTILS_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/configcpp") -SET (ENGINE_UTILS_COMPRESS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/compress") -SET (ENGINE_VERSIONING_BRM_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/versioning/BRM") -SET (ENGINE_UTILS_ROWGROUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rowgroup") -SET (ENGINE_UTILS_COMMON_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/common") -SET (ENGINE_UTILS_DATACONVERT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/dataconvert") -SET (ENGINE_UTILS_RWLOCK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rwlock") -SET (ENGINE_UTILS_FUNCEXP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/funcexp") -SET (ENGINE_UTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils") -SET (ENGINE_OAM_OAMCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/oam/oamcpp") -SET (ENGINE_DBCON_DDLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackageproc") -SET (ENGINE_DBCON_DDLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackage") -SET (ENGINE_DBCON_EXECPLAN_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/execplan") -SET (ENGINE_UTILS_STARTUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/startup") -SET (ENGINE_DBCON_JOBLIST_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/joblist") -SET (ENGINE_WE_WRAPPER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/wrapper") -SET (ENGINE_WE_SERVER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/server") -SET (ENGINE_DBCON_DMLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackage") -SET (ENGINE_WE_CLIENT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/client") -SET (ENGINE_DBCON_DMLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackageproc") -SET (ENGINE_UTILS_CACHEUTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/cacheutils") -SET (ENGINE_UTILS_MYSQLCL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/mysqlcl_idb") -SET (ENGINE_UTILS_QUERYTELE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querytele") -SET (ENGINE_UTILS_JOINER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/joiner") -SET (ENGINE_UTILS_THREADPOOL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/threadpool") -SET (ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchloader") -SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup") -SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") -SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") -SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") -SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") -SET (ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache") -SET (ENGINE_PRIMPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/primproc") -SET (ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") -SET (ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") -IF (PCRE_INCLUDES) - SET (ENGINE_SERVER_PCRE_INCLUDE "${PCRE_INCLUDES}") -ELSE () - SET (ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre") -ENDIF () -SET (ENGINE_SERVER_WSREP_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/include") -SET (ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/wsrep-API/v26/") -SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk") - -SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) - -SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) - -ADD_SUBDIRECTORY(dbcon/mysql) -IF(NOT TARGET columnstore) - RETURN() -ENDIF() - -ADD_SUBDIRECTORY(utils) -ADD_SUBDIRECTORY(oam/oamcpp) -ADD_SUBDIRECTORY(dbcon/execplan) -ADD_SUBDIRECTORY(dbcon/joblist) -ADD_SUBDIRECTORY(versioning) -ADD_SUBDIRECTORY(oam) -ADD_SUBDIRECTORY(writeengine/wrapper) -ADD_SUBDIRECTORY(writeengine/client) -ADD_SUBDIRECTORY(writeengine/xml) -ADD_SUBDIRECTORY(writeengine/redistribute) -ADD_SUBDIRECTORY(dbcon/ddlpackage) -ADD_SUBDIRECTORY(dbcon/ddlpackageproc) -ADD_SUBDIRECTORY(dbcon/dmlpackage) -ADD_SUBDIRECTORY(dbcon/dmlpackageproc) -ADD_SUBDIRECTORY(ddlproc) -ADD_SUBDIRECTORY(dmlproc) -ADD_SUBDIRECTORY(oamapps) -ADD_SUBDIRECTORY(primitives) -ADD_SUBDIRECTORY(tools) -ADD_SUBDIRECTORY(writeengine/server) -ADD_SUBDIRECTORY(writeengine/bulk) -ADD_SUBDIRECTORY(writeengine/splitter) -ADD_SUBDIRECTORY(storage-manager) -ADD_SUBDIRECTORY(datatypes) -ADD_SUBDIRECTORY(tests) - -find_package(Git QUIET) - -IF (GIT_FOUND AND EXISTS ${ENGINE_SRC_DIR}/.git) - exec_program("git" - ${CMAKE_CURRENT_SOURCE_DIR} - ARGS "describe --match=NeVeRmAtCh --always --dirty" - OUTPUT_VARIABLE GIT_VERSION) -ELSE () - SET(GIT_VERSION "source") -ENDIF () - -# releasenum is used by external scripts for various tasks. Leave it alone. -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h) - -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) - -IF (INSTALL_LAYOUT) - # Do this or when MariaDB builds us we don't have GenError which is required for these - ADD_DEPENDENCIES(udf_mysql GenError) - ADD_DEPENDENCIES(funcexp GenError) - ADD_DEPENDENCIES(oamcpp GenError) - ADD_DEPENDENCIES(regr_mysql GenError) -ELSE () - # MariaDB has its own packaging routines - INCLUDE(cpackEngineRPM) - INCLUDE(cpackEngineDEB) -ENDIF () +add_dependencies(udf_mysql GenError) +add_dependencies(funcexp GenError) +add_dependencies(oamcpp GenError) +add_dependencies(regr_mysql GenError) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 4c8b7a3ea..680eec342 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -4,153 +4,171 @@ # - the server's source code is two directories above the MCS engine source. # - the script is to be run under root. +set -o pipefail + +export CLICOLOR_FORCE=1 #cmake output + +INSTALL_PREFIX="/usr/" +DATA_DIR="/var/lib/mysql/data" +CMAKE_BIN_NAME=cmake +CTEST_BIN_NAME=ctest + +RPM_CONFIG_DIR="/etc/my.cnf.d" +DEB_CONFIG_DIR="/etc/mysql/mariadb.conf.d" +CONFIG_DIR=$RPM_CONFIG_DIR + SCRIPT_LOCATION=$(dirname "$0") -MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../..) +MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..) +COLUMSNTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../) -source $SCRIPT_LOCATION/utils.sh - -if [ "$EUID" -ne 0 ] - then error "Please run script as root to install MariaDb to system paths" - exit 1 -fi - -message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" +DEFAULT_MARIA_BUILD_PATH=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH")) BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") -DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky") +DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9") -cd $SCRIPT_LOCATION -CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) -cd - > /dev/null +GCC_VERSION="11" +MDB_CMAKE_FLAGS=() +source "$SCRIPT_LOCATION"/utils.sh -optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE +echo "Arguments received: $@" + +optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true +optparse.define short=a long=build-path desc="Path for build output" variable=MARIA_BUILD_PATH default=$DEFAULT_MARIA_BUILD_PATH +optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true +optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true +optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true optparse.define short=d long=distro desc="Choose your OS: ${DISTRO_OPTIONS[*]}" variable=OS optparse.define short=D long=install-deps desc="Install dependences" variable=INSTALL_DEPS default=false value=true -optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true -optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true -optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true -optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true -optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH -optparse.define short=D long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true -optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true -optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true -optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true -optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true -optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" -optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true +optparse.define short=F long=custom-cmake-flags desc="Add custom cmake flags" variable=CUSTOM_CMAKE_FLAGS +optparse.define short=f long=do-not-freeze-revision desc="Disable revision freezing, or do not set 'update none' for columnstore submodule in MDB repository" variable=DO_NOT_FREEZE_REVISION default=false value=true +optparse.define short=g long=alien desc="Turn off maintainer mode (ex. -Werror)" variable=MAINTAINER_MODE default=true value=false optparse.define short=G long=draw-deps desc="Draw dependencies graph" variable=DRAW_DEPS default=false value=true -optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true -optparse.define short=n long=no-clean-install desc="Do not perform a clean install (keep existing db files)" variable=NO_CLEAN default=false value=true optparse.define short=j long=parallel desc="Number of paralles for build" variable=CPUS default=$(getconf _NPROCESSORS_ONLN) -optparse.define short=F long=show-build-flags desc="Print CMake flags, while build" variable=PRINT_CMAKE_FLAGS default=false +optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true +optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true +optparse.define short=n long=no-clean-install desc="Do not perform a clean install (keep existing db files)" variable=NO_CLEAN default=false value=true +optparse.define short=o long=recompile-only variable=RECOMPILE_ONLY default=false value=true +optparse.define short=O long=static desc="Build all with static libraries" variable=STATIC_BUILD default=false value=true +optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true +optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" +optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false +optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE default=false value=true +optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true +optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE +optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true +optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true +optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true +optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true +optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true +optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true -source $( optparse.build ) +source $(optparse.build) + +message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal" if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS MCS_BUILD_TYPE=$selectedChoice fi -if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "CentOS" ]]; then +if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then + echo "OS is empty, trying to detect..." detect_distro fi -INSTALL_PREFIX="/usr/" -DATA_DIR="/var/lib/mysql/data" -CMAKE_BIN_NAME=cmake -CTEST_BIN_NAME=ctest -CONFIG_DIR="/etc/my.cnf.d" - -if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then - CONFIG_DIR="/etc/mysql/mariadb.conf.d" +pkg_format="deb" +if [[ "$OS" == *"rocky"* ]]; then + pkg_format="rpm" fi - -select_branch() -{ - cd $SCRIPT_LOCATION - CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) - - if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then - if [[ $BRANCH = "" ]]; then - getChoice -q "Select your branch" -o BRANCHES - BRANCH=$selectedChoice - fi - if [[ $BRANCH != $CURRENT_BRANCH ]]; then - message "Selecting $BRANCH branch for Columnstore" - git checkout $BRANCH - fi - - message "Turning off Columnstore submodule auto update via gitconfig" - cd $MDB_SOURCE_PATH - git config submodule.storage/columnstore/columnstore.update none - cd - > /dev/null +install_deps() { + if [[ $INSTALL_DEPS = false ]]; then + return fi - - cd - > /dev/null - message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch" -} - -install_deps() -{ message_split - message "Installing deps" - if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then - apt-get -y update - apt-get -y install build-essential automake libboost-all-dev bison cmake \ - libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ - libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ - libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev \ - liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev graphviz + prereq="" + RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \ + snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \ + rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \ + cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel" - elif [[ $OS = 'CentOS' || $OS = 'Rocky' || $OS = 'Fedora' ]]; then - if [[ "$OS_VERSION" == "7" ]]; then - yum -y install cmake3 epel-release centos-release-scl - CMAKE_BIN_NAME=cmake3 - CTEST_BIN_NAME=ctest3 - else - yum -y install cmake - fi - if [ $OS = 'Rocky' ]; then - yum -y groupinstall "Development Tools" && yum config-manager --set-enabled powertools - yum install -y checkpolicy - fi - if [[ $OS != 'Fedora' ]]; then - yum -y install epel-release - fi + DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \ + bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ + libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \ + libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \ + graphviz devscripts ccache equivs eatmydata curl" - yum install -y bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ - vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts \ - jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel lzo-devel xz-devel lz4-devel bzip2-devel \ - pcre2-devel flex graphviz libaio-devel openssl-devel flex + if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then + command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \ + dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \ + ${RPM_BUILD_DEPS}" + elif + [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]] + then + command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \ + dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}" + + elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then + command="${DEB_BUILD_DEPS}" else - error "Unsupported OS $OS" - exit 17 + echo "Unsupported OS: $OS" + exit 17 fi + + if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then + if [ -f /.dockerenv ]; then + change_ubuntu_mirror us + fi + command="${command} lto-disabled-list" + fi + + eval "$prereq" + message "Installing dependencies for $OS" + retry_eval 5 "$command" } -stop_service() -{ +install_deps + +cd $COLUMSNTORE_SOURCE_PATH +COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD) +message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch" + +cd $MDB_SOURCE_PATH +MARIADB_BRANCH=$(git rev-parse --abbrev-ref HEAD) +message "MariaDB will be built from $color_yellow$MARIADB_BRANCH$color_cyan branch" +cd - >/dev/null + +if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then + MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}" +fi + +disable_git_restore_frozen_revision() { + cd $MDB_SOURCE_PATH + git config submodule.storage/columnstore/columnstore.update none + cd - >/dev/null +} + +DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot" + +stop_service() { message_split message "Stopping MariaDB services" systemctl stop mariadb systemctl stop mariadb-columnstore + systemctl stop mcs-storagemanager } -check_service() -{ - if systemctl is-active --quiet $1; then +check_service() { + if systemctl is-active --quiet "$1"; then message "$1 $color_normal[$color_green OK $color_normal]" else message "$1 $color_normal[$color_red Fail $color_normal]" - service $1 status + service "$1" status fi } -start_service() -{ +start_service() { message_split message "Starting MariaDB services" systemctl start mariadb-columnstore @@ -166,212 +184,362 @@ start_service() check_service mcs-writeengineserver } -clean_old_installation() -{ +start_storage_manager_if_needed() { + if [[ $CLOUD_STORAGE_ENABLED = true ]]; then + export MCS_USE_S3_STORAGE=1 + message_split + message "Starting Storage Manager service" + systemctl start mcs-storagemanager + check_service mcs-storagemanager + fi +} + +clean_old_installation() { message_split message "Cleaning old installation" rm -rf /var/lib/columnstore/data1/* rm -rf /var/lib/columnstore/data/ rm -rf /var/lib/columnstore/local/ - rm -f /var/lib/columnstore/storagemanager/storagemanager-lock - rm -f /var/lib/columnstore/storagemanager/cs-initialized + rm -rf /var/lib/columnstore/storagemanager/* rm -rf /var/log/mariadb/columnstore/* rm -rf /tmp/* - rm -rf $REPORT_PATH + rm -rf "$REPORT_PATH" rm -rf /var/lib/mysql rm -rf /var/run/mysqld rm -rf $DATA_DIR - rm -rf /etc/mysql + rm -rf $CONFIG_DIR } -build() -{ - message_split - message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode" +modify_packaging() { + echo "Modifying_packaging..." + cd $MDB_SOURCE_PATH - local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes - -DPLUGIN_COLUMNSTORE=YES - -DPLUGIN_MROONGA=NO - -DPLUGIN_ROCKSDB=NO - -DPLUGIN_TOKUDB=NO - -DPLUGIN_CONNECT=NO - -DPLUGIN_SPIDER=NO - -DPLUGIN_OQGRAPH=NO - -DPLUGIN_SPHINX=NO - -DWITH_EMBEDDED_SERVER=OFF - -DBUILD_CONFIG=mysql_release - -DWITH_WSREP=OFF - -DWITH_SSL=system - -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX - -DCMAKE_EXPORT_COMPILE_COMMANDS=1 - " + if [[ $pkg_format == "deb" ]]; then + sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh + fi + #disable LTO for 22.04 for now + if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then + for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do + echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list + done && + grep mariadb /usr/share/lto-disabled-list/lto-disabled-list + fi - if [[ $SKIP_UNIT_TESTS = true ]] ; then - warn "Unittests are not build" + if [[ $pkg_format == "deb" ]]; then + apt-cache madison liburing-dev | grep liburing-dev || { + sed 's/liburing-dev/libaio-dev/g' -i debian/control && + sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && + sed '/-DWITH_URING=yes/d' -i debian/rules + } + apt-cache madison libpmem-dev | grep 'libpmem-dev' || { + sed '/libpmem-dev/d' -i debian/control && + sed '/-DWITH_PMEM/d' -i debian/rules + } + sed '/libfmt-dev/d' -i debian/control + # Remove Debian build flags that could prevent ColumnStore from building + sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules + + # Disable dh_missing strict check for missing files + sed -i 's/--fail-missing/--list-missing/' debian/rules + + # Tweak debian packaging stuff + for i in mariadb-plugin libmariadbd; do + sed -i "/Package: $i.*/,/^$/d" debian/control + done + + sed -i 's/Depends: galera.*/Depends:/' debian/control + + for i in galera wsrep ha_sphinx embedded; do + sed -i "/$i/d" debian/*.install + done + fi +} + +construct_cmake_flags() { + + MDB_CMAKE_FLAGS=( + -DBUILD_CONFIG=mysql_release + -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE + -DCMAKE_EXPORT_COMPILE_COMMANDS=1 + -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX + -DMYSQL_MAINTAINER_MODE=NO + -DPLUGIN_COLUMNSTORE=YES + -DPLUGIN_CONNECT=NO + -DPLUGIN_GSSAPI=NO + -DPLUGIN_MROONGA=NO + -DPLUGIN_OQGRAPH=NO + -DPLUGIN_ROCKSDB=NO + -DPLUGIN_SPHINX=NO + -DPLUGIN_SPIDER=NO + -DPLUGIN_TOKUDB=NO + -DWITH_EMBEDDED_SERVER=NO + -DWITH_SSL=system + -DWITH_SYSTEMD=yes + -DWITH_WSREP=NO + ) + + if [[ $MAINTAINER_MODE = true ]]; then + MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES) + message "Columnstore maintainer mode on" else - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UNITTESTS=YES" + warn "Maintainer mode is disabled, be careful, alien" + fi + + if [[ $SKIP_UNIT_TESTS = true ]]; then + warn "Unittests are not build" + else + MDB_CMAKE_FLAGS+=(-DWITH_UNITTESTS=YES) message "Buiding with unittests" fi - if [[ $DRAW_DEPS = true ]] ; then + if [[ $DRAW_DEPS = true ]]; then warn "Generating dependendies graph to mariadb.dot" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} --graphviz=mariadb.dot" + MDB_CMAKE_FLAGS+=(--graphviz=$DEP_GRAPH_PATH) fi - if [[ $USE_NINJA = true ]] ; then + if [[ $USE_NINJA = true ]]; then warn "Using Ninja instead of Makefiles" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja" + MDB_CMAKE_FLAGS+=(-GNinja) fi - if [[ $ASAN = true ]] ; then + if [[ $STATIC_BUILD = true ]]; then + warn "Building all with static linkage" + MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_STATIC_LIBRARIES:BOOL=ON) + fi + + if [[ $ASAN = true ]]; then warn "Building with Address Sanitizer " - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + MDB_CMAKE_FLAGS+=(-DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}) fi - if [[ $TSAN = true ]] ; then + if [[ $TSAN = true ]]; then warn "Building with Thread Sanitizer" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + MDB_CMAKE_FLAGS+=(-DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}) + message "Setting vm.mmap_rnd_bits=30 for TSAN support" + sysctl vm.mmap_rnd_bits=30 fi - if [[ $UBSAN = true ]] ; then + if [[ $UBSAN = true ]]; then warn "Building with UB Sanitizer" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + MDB_CMAKE_FLAGS+=(-DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}) fi - if [[ $WITHOUT_COREDUMPS = true ]] ; then + if [[ $WITHOUT_COREDUMPS = true ]]; then warn "Cores are not dumped" else - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON" - warn Building with CoreDumps: /proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p - echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern + warn "Building with CoreDumps" + MDB_CMAKE_FLAGS+=(-DWITH_COREDUMPS=ON) + + if [ -f /.dockerenv ]; then + warn "Build is executed in Docker, core dumps saving path /proc/sys/kernel/core_pattern will not be configured!" + else + warn "/proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p" + echo "${REPORT_PATH}/core_%e.%p" >/proc/sys/kernel/core_pattern + fi fi - if [[ $MAKEFILE_VERBOSE = true ]] ; then + if [[ $MAKEFILE_VERBOSE = true ]]; then warn "Verbosing Makefile Commands" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" + MDB_CMAKE_FLAGS+=(-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON) fi - if [[ $RUN_BENCHMARKS = true ]] ; then - if [[ $MCS_BUILD_TYPE = 'Debug' ]] ; then + if [[ $SCCACHE = true ]]; then + warn "Use sccache" + MDB_CMAKE_FLAGS+=(-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache) + fi + + if [[ $RUN_BENCHMARKS = true ]]; then + if [[ $MCS_BUILD_TYPE = 'Debug' ]]; then error "Benchmarks will not be build in run in Debug build Mode" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" - $RUN_BENCHMARKS = false - elif [[ $OS != 'Ubuntu' && $OS != 'Debian' ]] ; then + MDB_CMAKE_FLAGS+=(-DWITH_MICROBENCHMARKS=NO) + RUN_BENCHMARKS=false + elif [[ $OS != *"ubuntu"* && $OS != *"debian"* ]]; then error "Benchmarks are now avaliable only at Ubuntu or Debian" - MAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" - $RUN_BENCHMARKS = false + MAKE_FLAGS="${MDB_CMAKE_FLAGS[@]} -DWITH_MICROBENCHMARKS=NO" + RUN_BENCHMARKS=false else message "Compile with microbenchmarks" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES" + MDB_CMAKE_FLAGS+=(-DWITH_MICROBENCHMARKS=YES) fi else - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" + MDB_CMAKE_FLAGS+=(-DWITH_MICROBENCHMARKS=NO) message "Buiding without microbenchmarks" fi - cd $MDB_SOURCE_PATH + if [[ "$OS" == *"rocky"* ]]; then + OS_VERSION=${OS//[^0-9]/} + MDB_CMAKE_FLAGS+=(-DRPM=rockylinux${OS_VERSION}) + elif [[ "$OS" == "debian:11" ]]; then + CODENAME="bullseye" + elif [[ "$OS" == "debian:12" ]]; then + CODENAME="bookworm" + elif [[ "$OS" == "ubuntu:20.04" ]]; then + CODENAME="focal" + elif [[ "$OS" == "ubuntu:22.04" ]]; then + CODENAME="jammy" + elif [[ "$OS" == "ubuntu:24.04" ]]; then + CODENAME="noble" + else + echo "Unsupported OS: $OS" + exit 17 + fi - if [[ $SKIP_SUBMODULES = true ]] ; then + if [[ -n "$CODENAME" ]]; then + MDB_CMAKE_FLAGS+=(-DDEB=${CODENAME}) + fi + + if [[ "$NO_CLEAN" == true ]]; then + message "Skipping Columnstore.etc config installation" + MDB_CMAKE_FLAGS+=(-DSKIP_CONFIG_INSTALLATION=ON) + fi + + MDB_CMAKE_FLAGS+=($CUSTOM_CMAKE_FLAGS) + + message "Building with flags" + newline_array "${MDB_CMAKE_FLAGS[@]}" +} + +init_submodules() { + if [[ $SKIP_SUBMODULES = true ]]; then warn "Skipping initialization of columnstore submodules" else message "Initialization of columnstore submodules" - cd storage/columnstore/columnstore - git submodule update --init - cd - > /dev/null + cd $MDB_SOURCE_PATH + git submodule update --init --recursive + cd - >/dev/null fi +} - if [[ $FORCE_CMAKE_CONFIG = true ]] ; then - warn "Erasing cmake cache" - rm -f "$MDB_SOURCE_PATH/CMakeCache.txt" - rm -rf "$MDB_SOURCE_PATH/CMakeFiles" - fi - - if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" - elif [ $OS = 'CentOS' ]; then - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" - elif [ $OS = 'Rocky' ]; then - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" - elif [ $OS = 'openSUSE' ]; then - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15" - fi - - if [[ $PRINT_CMAKE_FLAGS = true ]] ; then - message "Building with flags" - newline_array ${MDB_CMAKE_FLAGS[@]} - fi - - message "Configuring cmake silently" - ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS . | spinner - message_split - ${CMAKE_BIN_NAME} --build . -j $CPUS && \ - message "Installing silently" && - ${CMAKE_BIN_NAME} --install . | spinner 30 - +check_errorcode() { if [ $? -ne 0 ]; then message_split error "!!!! BUILD FAILED !!!!" message_split exit 1 fi - cd - > /dev/null + cd - >/dev/null } -check_user_and_group() -{ - if [ -z "$(grep mysql /etc/passwd)" ]; then - message "Adding user mysql into /etc/passwd" - useradd -r -U mysql -d /var/lib/mysql - fi - - if [ -z "$(grep mysql /etc/group)" ]; then - GroupID = `awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group` - message "Adding group mysql with id $GroupID" - groupadd -g GroupID mysql +generate_svgs() { + if [[ $DRAW_DEPS = true ]]; then + message_split + warn "Generating svgs with dependency graph to $DEP_GRAPH_PATH" + for f in $(ls "$DEP_GRAPH_PATH".* | grep -v ".svg"); do + dot -Tsvg -o "$f.svg" "$f" + done fi } -run_unit_tests() -{ +build_package() { + cd $MDB_SOURCE_PATH + + if [[ $pkg_format == "rpm" ]]; then + command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package" + else + export DEBIAN_FRONTEND="noninteractive" + export DEB_BUILD_OPTIONS="parallel=$(nproc)" + export DH_BUILD_DDEBS="1" + export BUILDPACKAGE_FLAGS="-b" + command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \ + CMAKEFLAGS=\"${MDB_CMAKE_FLAGS[@]}\" debian/autobake-deb.sh" + fi + + echo "Building a package for $OS" + echo "Build command: $command" + eval "$command" + + check_errorcode +} + +check_debian_install_file() { + message "checking debian/mariadb-plugin-columnstore.install" message_split - if [[ $SKIP_UNIT_TESTS = true ]] ; then + python3 $COLUMSNTORE_SOURCE_PATH/build/debian_install_file_compare.py \ + ${COLUMSNTORE_SOURCE_PATH}/debian/mariadb-plugin-columnstore.install \ + $MARIA_BUILD_PATH/mariadb-plugin-columnstore.install.generated +} + +build_binary() { + MARIA_BUILD_PATH=$(realpath "$MARIA_BUILD_PATH") + message_split + message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_cyan mode" + message "Compiled artifacts will be written to $color_yellow$MARIA_BUILD_PATH$color_cyan" + mkdir -p "$MARIA_BUILD_PATH" + + cd $MDB_SOURCE_PATH + + if [[ $FORCE_CMAKE_CONFIG = true ]]; then + warn "Erasing cmake cache" + rm -f "$MARIA_BUILD_PATH/CMakeCache.txt" + rm -rf "$MARIA_BUILD_PATH/CMakeFiles" + fi + + message "Configuring cmake silently" + ${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner + message_split + check_debian_install_file + generate_svgs + + ${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator && + message "Installing silently" && + ${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30 + + check_errorcode + + message "Adding symbol link to compile_commands.json to the source root" + ln -sf "$MARIA_BUILD_PATH/compile_commands.json" "$MDB_SOURCE_PATH" +} + +check_user_and_group() { + user=$1 + if [ -z "$(grep "$user" /etc/passwd)" ]; then + message "Adding user $user into /etc/passwd" + useradd -r -U "$user" -d /var/lib/mysql + fi + + if [ -z "$(grep "$user" /etc/group)" ]; then + GroupID=$(awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group) + message "Adding group $user with id $GroupID" + groupadd -g "$GroupID" "$user" + fi +} + +run_unit_tests() { + message_split + if [[ $SKIP_UNIT_TESTS = true ]]; then warn "Skipping unittests" else message "Running unittests" cd $MARIA_BUILD_PATH - ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress - cd - > /dev/null + ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure + cd - >/dev/null fi } -run_microbenchmarks_tests() -{ +run_microbenchmarks_tests() { message_split - if [[ $RUN_BENCHMARKS = false ]] ; then + if [[ $RUN_BENCHMARKS = false ]]; then warn "Skipping microbenchmarks" else message "Runnning microbenchmarks" - cd $MDB_SOURCE_PATH + cd $MARIA_BUILD_PATH ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress - cd - > /dev/null + cd - >/dev/null fi } -disable_plugins_for_bootstrap() -{ +disable_plugins_for_bootstrap() { find /etc -type f -exec sed -i 's/plugin-load-add=auth_gssapi.so//g' {} + find /etc -type f -exec sed -i 's/plugin-load-add=ha_columnstore.so//g' {} + } -enable_columnstore_back() -{ - echo plugin-load-add=ha_columnstore.so >> $CONFIG_DIR/columnstore.cnf +enable_columnstore_back() { + cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/dbcon/mysql/columnstore.cnf $CONFIG_DIR } -fix_config_files() -{ +fix_config_files() { message Fixing config files THREAD_STACK_SIZE="20M" @@ -380,133 +548,136 @@ fix_config_files() MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf - if [[ $ASAN = true ]] ; then + if [[ $ASAN = true ]]; then if grep -q thread_stack $COLUMNSTORE_CONFIG; then warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN" else - echo "thread_stack = ${THREAD_STACK_SIZE}" >> $COLUMNSTORE_CONFIG + echo "thread_stack = ${THREAD_STACK_SIZE}" >>$COLUMNSTORE_CONFIG message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG" fi if grep -q ASAN $MDB_SERVICE_FILE; then warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility" else - echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${REPORT_PATH}/asan.mariadb'" >> $MDB_SERVICE_FILE + echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${REPORT_PATH}/asan.mariadb'" >>$MDB_SERVICE_FILE message "ASAN options were added to $MDB_SERVICE_FILE" fi fi - if [[ $TSAN = true ]] ; then + if [[ $TSAN = true ]]; then if grep -q TSAN $MDB_SERVICE_FILE; then warn "MDB Server has TSAN options in $MDB_SERVICE_FILE, check it's compatibility" else - echo Environment="'TSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/tsan.mariadb'" >> $MDB_SERVICE_FILE + echo Environment="'TSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/tsan.mariadb'" >>$MDB_SERVICE_FILE message "TSAN options were added to $MDB_SERVICE_FILE" fi fi - if [[ $UBSAN = true ]] ; then + if [[ $UBSAN = true ]]; then if grep -q UBSAN $MDB_SERVICE_FILE; then warn "MDB Server has UBSAN options in $MDB_SERVICE_FILE, check it's compatibility" else - echo Environment="'UBSAN_OPTIONS=abort_on_error=0,print_stacktrace=true,log_path=${REPORT_PATH}/ubsan.mariadb'" >> $MDB_SERVICE_FILE + echo Environment="'UBSAN_OPTIONS=abort_on_error=0,print_stacktrace=true,log_path=${REPORT_PATH}/ubsan.mariadb'" >>$MDB_SERVICE_FILE message "UBSAN options were added to $MDB_SERVICE_FILE" fi fi - - message Reloading systemd - systemctl daemon-reload + if [[ $RECOMPILE_ONLY = false ]]; then + message Reloading systemd + systemctl daemon-reload + fi } -install() -{ - message_split - message "Installing MariaDB" - disable_plugins_for_bootstrap +make_dir() { + mkdir -p "$1" + chown mysql:mysql "$1" +} - mkdir -p $REPORT_PATH - chmod 777 $REPORT_PATH +install() { + if [[ $RECOMPILE_ONLY = false ]]; then - check_user_and_group + if [ "$EUID" -ne 0 ]; then + error "Please run script as root to install MariaDb to system paths" + exit 1 + fi - mkdir -p /etc/my.cnf.d + message_split + message "Installing MariaDB" + disable_plugins_for_bootstrap - bash -c 'echo "[client-server] -socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' + make_dir "$REPORT_PATH" + chmod 777 "$REPORT_PATH" - mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so - mkdir -p /var/lib/mysql - chown mysql:mysql /var/lib/mysql + check_user_and_group mysql + check_user_and_group syslog - message "Running mysql_install_db" - sudo -u mysql mysql_install_db --rpm --user=mysql > /dev/null - mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so + make_dir $CONFIG_DIR - enable_columnstore_back + echo "[client-server] + socket=/run/mysqld/mysqld.sock" >$CONFIG_DIR/socket.cnf - mkdir -p /etc/columnstore + make_dir /var/lib/mysql - cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml - cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + message "Running mysql_install_db" + sudo -u mysql mysql_install_db --rpm --user=mysql >/dev/null - cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/ - cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + enable_columnstore_back - if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then - mkdir -p /usr/share/mysql - mkdir -p /etc/mysql/ - cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh - cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start - > /etc/mysql/debian.cnf + make_dir /etc/columnstore + + if [[ "$NO_CLEAN" == false ]]; then + cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml + cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + fi + + cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + + if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then + make_dir /usr/share/mysql + make_dir /etc/mysql/ + cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh + cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start + >/etc/mysql/debian.cnf + fi + + fix_config_files + + if [ -d "$DEBCONFIG_DIR" ]; then + message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR" + cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR" + fi + + make_dir /var/lib/columnstore/data1 + make_dir /var/lib/columnstore/data1/systemFiles + make_dir /var/lib/columnstore/data1/systemFiles/dbrm + make_dir /run/mysqld/ + make_dir $DATA_DIR + + chmod +x $INSTALL_PREFIX/bin/mariadb* + + ldconfig + + start_storage_manager_if_needed + + message "Running columnstore-post-install" + make_dir /var/lib/columnstore/local + columnstore-post-install --rpmmode=install + message "Running install_mcs_mysql" + install_mcs_mysql.sh fi - fix_config_files - - if [ -d "/etc/mysql/mariadb.conf.d/" ]; then - message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d" - cp -rp /etc/mysql/mariadb.conf.d/* /etc/my.cnf.d - fi - - if [ -d "/etc/mysql/conf.d/" ]; then - message "Copying configs from /etc/mysql/conf.d/ to /etc/my.cnf.d" - cp -rp /etc/mysql/conf.d/* /etc/my.cnf.d - fi - - mkdir -p /var/lib/columnstore/data1 - mkdir -p /var/lib/columnstore/data1/systemFiles - mkdir -p /var/lib/columnstore/data1/systemFiles/dbrm - mkdir -p /run/mysqld/ - - mkdir -p $DATA_DIR - chown -R mysql:mysql $DATA_DIR - chown -R mysql:mysql /var/lib/columnstore/ - chown -R mysql:mysql /run/mysqld/ - - chmod +x $INSTALL_PREFIX/bin/mariadb* - - ldconfig - - message "Running columnstore-post-install" - mkdir -p /var/lib/columnstore/local - columnstore-post-install --rpmmode=install - message "Running install_mcs_mysql" - install_mcs_mysql.sh - chown -R syslog:syslog /var/log/mariadb/ chmod 777 /var/log/mariadb/ chmod 777 /var/log/mariadb/columnstore } - -smoke() -{ - if [[ $SKIP_SMOKE = false ]] ; then +smoke() { + if [[ $SKIP_SMOKE = false ]]; then message_split message "Creating test database" mariadb -e "create database if not exists test;" message "Selecting magic numbers" - MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql` - if [[ $MAGIC == '42' ]] ; then + MAGIC=$(mysql -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql) + if [[ $MAGIC == '42' ]]; then message "Great answer correct!" else warn "Smoke failed, answer is '$MAGIC'" @@ -514,36 +685,30 @@ smoke() fi } +if [[ $DO_NOT_FREEZE_REVISION = false ]]; then + disable_git_restore_frozen_revision +fi -generate_svgs() -{ - if [[ $DRAW_DEPS = true ]] ; then - message_split - warn "Generating svgs with dependency graph to $REPORT_PATH" - for f in $MDB_SOURCE_PATH/mariadb.dot.*; - do dot -Tsvg -o $REPORT_PATH/`basename $f`.svg $f; - done +construct_cmake_flags +init_submodules + +if [[ $BUILD_PACKAGES = false ]]; then + stop_service + + if [[ $NO_CLEAN = false ]]; then + clean_old_installation fi -} - -select_branch - -if [[ $INSTALL_DEPS = true ]] ; then - install_deps + build_binary + run_unit_tests + run_microbenchmarks_tests + install + if [[ $RESTART_SERVICES = true ]]; then + start_service + smoke + fi +else + modify_packaging + build_package fi -stop_service - -if [[ $NO_CLEAN = false ]] ; then - clean_old_installation -fi - -build -run_unit_tests -run_microbenchmarks_tests -install -start_service -smoke -generate_svgs - message_splitted "FINISHED" diff --git a/build/build_cmapi.sh b/build/build_cmapi.sh new file mode 100755 index 000000000..241b3fc35 --- /dev/null +++ b/build/build_cmapi.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# This script installs dependencies and builds cmapi package +# Should be executed by root +# Should reside in build/ (or storage/columnstore/columnstore/build), as all the paths a relative to script's location + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +COLUMNSTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../) +MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..) + +source "$SCRIPT_LOCATION"/utils.sh + +optparse.define short=d long=distro desc="distro" variable=OS +optparse.define short=a long=arch desc="architecture" variable=ARCH +source $(optparse.build) +echo "Arguments received: $@" + +if [ "$EUID" -ne 0 ]; then + error "Please run script as root" + exit 1 +fi + +if [[ -z "${OS:-}" || -z "${ARCH:-}" ]]; then + echo "Please provide provide --distro and --arch parameters, e.g. ./build_cmapi.sh --distro ubuntu:22.04 --arch amd64" + exit 1 +fi + +pkg_format="deb" +if [[ "$OS" == *"rocky"* ]]; then + pkg_format="rpm" +fi + +if [[ "$ARCH" == "arm64" ]]; then + export CC=gcc #TODO: what it is for? +fi + +on_exit() { + if [[ $? -eq 0 ]]; then + echo "Cmapi package has been build successfully." + else + echo "Cmapi package build failed!" + fi +} +trap on_exit EXIT + +install_deps() { + echo "Installing dependencies..." + + cd "$COLUMNSTORE_SOURCE_PATH"/cmapi + + if [[ "$OS" == "rockylinux:9" ]]; then + retry_eval 5 "dnf install -q -y libxcrypt-compat yum-utils" + retry_eval 5 "dnf config-manager --set-enabled devel && dnf update -q -y" #to make redhat-lsb-core available for rocky 9 + fi + + if [[ "$pkg_format" == "rpm" ]]; then + retry_eval 5 "dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive" + else + retry_eval 5 "apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release" + fi + + if [ "$ARCH" == "amd64" ]; then + PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst" + elif [ "$ARCH" == "arm64" ]; then + PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst" + else + echo "Unsupported architecture: $ARCH" + exit 1 + fi + + rm -rf python pp + wget -qO- "$PYTHON_URL" | tar --use-compress-program=unzstd -xf - -C ./ + + mv python pp + mv pp/install python + chown -R root:root python + + python/bin/pip3 install -t deps --only-binary :all -r requirements.txt + cp cmapi_server/cmapi_server.conf cmapi_server/cmapi_server.conf.default +} + +build_cmapi() { + cd "$COLUMNSTORE_SOURCE_PATH"/cmapi + ./cleanup.sh + cmake -D"${pkg_format^^}"=1 -DSERVER_DIR="$MDB_SOURCE_PATH" . && make package +} +install_deps +build_cmapi diff --git a/build/createrepo.sh b/build/createrepo.sh new file mode 100755 index 000000000..7bc2673c8 --- /dev/null +++ b/build/createrepo.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# This script creates a repository from packages in result directory +# Should be executed by root + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") + +source "$SCRIPT_LOCATION"/utils.sh +optparse.define short=R long=result desc="Directory with packages" variable=RESULT +source $(optparse.build) +echo "Arguments received: $@" + +BUILDDIR="verylongdirnameforverystrangecpackbehavior" +COLUMNSTORE_RPM_PACKAGES_PATH="/mdb/${BUILDDIR}/*.rpm" +CMAPI_RPM_PACKAGES_PATH="/mdb/${BUILDDIR}/storage/columnstore/columnstore/cmapi/*.rpm" + +COLUMNSTORE_DEB_PACKAGES_PATH="/mdb/*.deb" +CMAPI_DEB_PACKAGES_PATH="/mdb/${BUILDDIR}/storage/columnstore/columnstore/cmapi/*.deb" + +if [ "$EUID" -ne 0 ]; then + error "Please run script as root" + exit 1 +fi + +if [[ -z "${RESULT:-}" ]]; then + echo "Please provide provide --result parameter, e.g. ./createrepo.sh --result ubuntu24.04" + exit 1 +fi + +cd "/mdb/${BUILDDIR}" + +if ! compgen -G "/mdb/${BUILDDIR}/*.rpm" >/dev/null && + ! compgen -G "/mdb/${BUILDDIR}/storage/columnstore/columnstore/cmapi/*.rpm" >/dev/null && + ! compgen -G "/mdb/${BUILDDIR}/*.deb" >/dev/null && + ! compgen -G "/mdb/${BUILDDIR}/storage/columnstore/columnstore/cmapi/*.deb" >/dev/null; then + echo "None of the cmapi or columnstore packages found. Failing!" + exit 1 +fi + +echo "Adding columnstore packages to repository..." +if [[ $(compgen -G "$COLUMNSTORE_RPM_PACKAGES_PATH") ]]; then + mv -v $COLUMNSTORE_RPM_PACKAGES_PATH "./${RESULT}/" +elif [[ $(compgen -G "$COLUMNSTORE_DEB_PACKAGES_PATH") ]]; then + mv -v $COLUMNSTORE_DEB_PACKAGES_PATH "./${RESULT}/" +else + echo "Columnstore packages are not found!" +fi + +echo "Adding cmapi packages to repository..." +if [[ $(compgen -G "$CMAPI_RPM_PACKAGES_PATH") ]]; then + mv -v $CMAPI_RPM_PACKAGES_PATH "./${RESULT}/" +elif [[ $(compgen -G "$CMAPI_DEB_PACKAGES_PATH") ]]; then + mv -v $CMAPI_DEB_PACKAGES_PATH "./${RESULT}/" +else + echo "Cmapi packages are not found!" +fi + +if [[ $(compgen -G "./${RESULT}/*.rpm") ]]; then + retry_eval 5 "dnf install -q -y createrepo" + createrepo "./${RESULT}" +else + retry_eval 5 "apt update && apt install -y dpkg-dev" + dpkg-scanpackages "${RESULT}" | gzip >"./${RESULT}/Packages.gz" +fi + +mkdir -p "/drone/src/${RESULT}" +cp -vrf "./${RESULT}/." "/drone/src/${RESULT}" diff --git a/build/debian_install_file_compare.py b/build/debian_install_file_compare.py new file mode 100644 index 000000000..eeeb8e415 --- /dev/null +++ b/build/debian_install_file_compare.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import argparse +import fnmatch +import sys + +def read_patterns(filename): + with open(filename, 'r') as f: + return [line.strip() for line in f if line.strip()] + +def read_paths_with_comments(filename): + paths = [] + with open(filename, 'r') as f: + for line in f: + raw = line.rstrip('\n') + if not raw.strip(): + continue + # Split into path and comment + if '#' in raw: + path, comment = raw.split('#', 1) + path = path.strip() + comment = comment.strip() + else: + path = raw.strip() + comment = '' + if path: + paths.append((raw, path, comment)) + return paths + +def print_red(msg): + if sys.stdout.isatty(): + print("\033[31m" + msg + "\033[0m") + else: + print(msg) + +def main(): + parser = argparse.ArgumentParser(description="Compare wildcard patterns to file paths.") + parser.add_argument('patterns_file', help='File with wildcard patterns (one per line)') + parser.add_argument('paths_file', help='File with file paths (one per line)') + args = parser.parse_args() + + patterns = read_patterns(args.patterns_file) + paths_with_comments = read_paths_with_comments(args.paths_file) + + # Track which paths are matched (by index) + path_matched = [False] * len(paths_with_comments) + + # Collect patterns that don't match any path + unmatched_patterns = [] + for pattern in patterns: + matched = False + for i, (raw, path, comment) in enumerate(paths_with_comments): + if fnmatch.fnmatch(path, pattern): + matched = True + path_matched[i] = True + if not matched: + unmatched_patterns.append(pattern) + + if unmatched_patterns: + print_red("The files declared in debian/mariadb-plugin-columnstore.install " + " are not added to CMakeLists via columnstore_* statements see cmake/ColumnstoreLibrary.cmake") + for pattern in unmatched_patterns: + print(f"- {pattern}") + + # Collect paths that weren't matched by any pattern + unmatched_paths = [] + for (raw, path, comment), matched in zip(paths_with_comments, path_matched): + if not matched: + unmatched_paths.append((path, comment)) + + if unmatched_paths: + print_red("The files added via columnstore_* statements from cmake/ColumnstoreLibrary.cmake " + "are missing in debian/mariadb-plugin-columnstore.install file") + for path, comment in unmatched_paths: + if comment: + print(f"- {path} {comment}") + else: + print(f"- {path}") + +if __name__ == "__main__": + main() diff --git a/build/prepare_test_stage.sh b/build/prepare_test_stage.sh new file mode 100755 index 000000000..73a2a5e6d --- /dev/null +++ b/build/prepare_test_stage.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +# Should be executed by root +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +COLUMNSTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../) + +source "$SCRIPT_LOCATION"/utils.sh + +echo "Arguments received: $@" + +optparse.define short=c long=container-name desc="Name of the Docker container to run tests in" variable=CONTAINER_NAME +optparse.define short=i long=docker-image desc="Docker image name to start container from" variable=DOCKER_IMAGE +optparse.define short=r long=result-path desc="Name suffix used in core dump file path" variable=RESULT +optparse.define short=s long=do-setup desc="Run setup-repo.sh inside the container" variable=DO_SETUP +optparse.define short=u long=packages-url desc="Packages url" variable=PACKAGES_URL +source $(optparse.build) + +if [[ "$EUID" -ne 0 ]]; then + error "Please run script as root" + exit 1 +fi + +if [[ -z "${CONTAINER_NAME:-}" || -z "${DOCKER_IMAGE:-}" || -z "${RESULT:-}" || -z "${DO_SETUP:-}" || -z "${PACKAGES_URL:-}" ]]; then + echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true" + exit 1 +fi + +start_container() { + if [[ "$RESULT" == *rocky* ]]; then + SYSTEMD_PATH="/usr/lib/systemd/systemd" + MTR_PATH="/usr/share/mysql-test" + else + SYSTEMD_PATH="systemd" + MTR_PATH="/usr/share/mysql/mysql-test" + fi + + docker_run_args=( + --env OS="$RESULT" + --env PACKAGES_URL="$PACKAGES_URL" + --env DEBIAN_FRONTEND=noninteractive + --env MCS_USE_S3_STORAGE=0 + --name "$CONTAINER_NAME" + --ulimit core=-1 + --privileged + --detach + ) + + if [[ "$CONTAINER_NAME" == *smoke* ]]; then + docker_run_args+=(--memory 3g) + elif [[ "$CONTAINER_NAME" == *mtr* ]]; then + docker_run_args+=(--shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH") + elif [[ "$CONTAINER_NAME" == *cmapi* ]]; then + docker_run_args+=(--env PYTHONPATH="${PYTHONPATH}") + elif [[ "$CONTAINER_NAME" == *upgrade* ]]; then + docker_run_args+=(--env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro) + elif [[ "$CONTAINER_NAME" == *regression* ]]; then + docker_run_args+=(--shm-size=500m --memory 12g) + else + echo "Unknown container type: $CONTAINER_NAME" + exit 1 + fi + + docker_run_args+=("$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target) + + docker run "${docker_run_args[@]}" + sleep 5 + + bash "$COLUMNSTORE_SOURCE_PATH"/core_dumps/docker-awaiter.sh "$CONTAINER_NAME" + + if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then + error "Container '$CONTAINER_NAME' has not started!" + exit 1 + fi +} + +start_container + +if [[ "$RESULT" != *rocky* ]]; then + execInnerDocker "$CONTAINER_NAME" 'sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d' +fi + +#list_cgroups +echo "Docker CGroups opts here" +ls -al /sys/fs/cgroup/cgroup.controllers || true +ls -al /sys/fs/cgroup/ || true +ls -al /sys/fs/cgroup/memory || true + +execInnerDocker "$CONTAINER_NAME" 'echo Inner Docker CGroups opts here' +execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/cgroup.controllers || true' +execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/ || true' +execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/memory || true' + +# Prepare core dump directory inside container +execInnerDocker "$CONTAINER_NAME" 'mkdir -p core && chmod 777 core' +docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/. "$CONTAINER_NAME":/ +docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh "$CONTAINER_NAME":/ +docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh "$CONTAINER_NAME":/ + +if [[ "$DO_SETUP" == "true" ]]; then + execInnerDocker "$CONTAINER_NAME" '/setup-repo.sh' +fi + +# install deps +if [[ "$RESULT" == *rocky* ]]; then + execInnerDockerWithRetry "$CONTAINER_NAME" 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' +else + change_ubuntu_mirror_in_docker "$CONTAINER_NAME" "us" + execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' +fi + +# Configure core dump naming pattern +execInnerDocker "$CONTAINER_NAME" 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"' + +#Install columnstore in container +echo "Installing columnstore..." +if [[ "$RESULT" == *rocky* ]]; then + execInnerDockerWithRetry "$CONTAINER_NAME" 'yum install -y MariaDB-columnstore-engine MariaDB-test' +else + execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' +fi + +sleep 5 +echo "PrepareTestStage completed in $CONTAINER_NAME" diff --git a/build/report_test_stage.sh b/build/report_test_stage.sh new file mode 100755 index 000000000..23546a382 --- /dev/null +++ b/build/report_test_stage.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +set -eo pipefail + +CONTAINER_NAME=$1 +RESULT=$2 +STAGE=$3 + +SCRIPT_LOCATION=$(dirname "$0") +source "$SCRIPT_LOCATION"/utils.sh + +echo "Arguments received: $@" + +if [[ "$EUID" -ne 0 ]]; then + error "Please run script as root" + exit 1 +fi + +if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then + error "Container '${CONTAINER_NAME}' is not running." + exit 1 +fi + +if [[ "$RESULT" == *rocky* ]]; then + SYSTEMD_PATH="/usr/lib/systemd/systemd" + MTR_PATH="/usr/share/mysql-test" +else + SYSTEMD_PATH="systemd" + MTR_PATH="/usr/share/mysql/mysql-test" +fi + +echo "Reporting test stage: ${STAGE} executed in ${CONTAINER_NAME} container" + +if [[ "${CONTAINER_NAME}" == *smoke* ]] || [[ "${CONTAINER_NAME}" == *mtr* ]] || [[ "${CONTAINER_NAME}" == *cmapi* ]]; then + # common logs for smoke, mtr, cmapi + echo "---------- start mariadb service logs ----------" + execInnerDocker "$CONTAINER_NAME" 'journalctl -u mariadb --no-pager || echo "mariadb service failure"' + echo "---------- end mariadb service logs ----------" + echo + echo "---------- start columnstore debug log ----------" + execInnerDocker "$CONTAINER_NAME" 'cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"' + echo "---------- end columnstore debug log ----------" + + if [[ "${CONTAINER_NAME}" == *mtr* ]]; then + echo + docker cp "${CONTAINER_NAME}:${MTR_PATH}/var/log" "/drone/src/${RESULT}/mtr-logs" || echo "missing ${MTR_PATH}/var/log" + fi + + if [[ "${CONTAINER_NAME}" == *cmapi* ]]; then + echo + echo "---------- start cmapi log ----------" + execInnerDocker "$CONTAINER_NAME" 'cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi_server.log"' + echo "---------- end cmapi log ----------" + fi + +elif [[ "${CONTAINER_NAME}" == *upgrade* ]]; then + # nothing to report here for upgrade + : + +elif [[ "${CONTAINER_NAME}" == *regression* ]]; then + echo "---------- start columnstore regression short report ----------" + execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; cat go.log || echo "missing go.log"' + echo "---------- end columnstore regression short report ----------" + echo + docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/" "/drone/src/${RESULT}/" || echo "missing regression logs" + docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs.tgz" + + execInnerDocker "$CONTAINER_NAME" 'tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/' + execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore || echo "failed to grab regression results"' + docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs2.tgz" + docker cp "${CONTAINER_NAME}:regressionQueries.tgz" "/drone/src/${RESULT}/" || echo "missing regressionQueries.tgz" + +else + echo "Unknown stage's container provided: ${CONTAINER_NAME}" + exit 1 +fi + +execInnerDocker "$CONTAINER_NAME" "/logs.sh ${STAGE}" +execInnerDocker "$CONTAINER_NAME" "/core_dump_check.sh core /core/ ${STAGE}" + +docker cp "${CONTAINER_NAME}:/core/" "/drone/src/${RESULT}/" +docker cp "${CONTAINER_NAME}:/unit_logs/" "/drone/src/${RESULT}/" + +execInnerDocker "$CONTAINER_NAME" "/core_dump_drop.sh core" +echo "Saved artifacts:" +ls -R "/drone/src/${RESULT}/" +echo "Done reporting ${STAGE}" + +cleanup() { + if [[ -n $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then + echo "Cleaning up container ${CONTAINER_NAME}..." + docker rm -f "${CONTAINER_NAME}" || echo "Can't remove container ${CONTAINER_NAME}!" + fi +} +#Remove the container on exit +trap cleanup EXIT diff --git a/build/run_smoke.sh b/build/run_smoke.sh new file mode 100755 index 000000000..c94164662 --- /dev/null +++ b/build/run_smoke.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +source "$SCRIPT_LOCATION"/utils.sh + +CONTAINER_NAME="$1" + +echo "Arguments received: $@" + +if [[ "$EUID" -ne 0 ]]; then + error "Please run script as root" + exit 1 +fi + +if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then + error "Container '${CONTAINER_NAME}' is not running." + exit 1 +fi + +message "Running smoke checks..." + +# start mariadb and mariadb-columnstore services and run simple query +execInnerDocker "$CONTAINER_NAME" 'systemctl start mariadb' +execInnerDocker "$CONTAINER_NAME" '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local' +execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb-columnstore' + +execInnerDocker "$CONTAINER_NAME" 'mariadb -e "CREATE DATABASE IF NOT EXISTS test; \ +CREATE TABLE test.t1 (a INT) ENGINE=Columnstore; \ +INSERT INTO test.t1 VALUES (1); \ +SELECT * FROM test.t1;"' + +# restart both services, wait a bit, then insert and select again +execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb' +execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb-columnstore' + +sleep 10 + +execInnerDocker "$CONTAINER_NAME" 'mariadb -e "INSERT INTO test.t1 VALUES (2); SELECT * FROM test.t1;"' diff --git a/build/update-clang-version.sh b/build/update-clang-version.sh new file mode 100644 index 000000000..4528561dc --- /dev/null +++ b/build/update-clang-version.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +function register_clang_version() { + local version=$1 + local priority=$2 + + update-alternatives \ + --verbose \ + --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ + --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ + --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ + --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ + --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ + --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ + --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ + --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ + --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ + --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ + --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ + --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ + --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ + --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ + --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ + --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ + --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ + --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ + --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ + --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ + --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ + --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ + --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ + --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ + --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ + --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} + + update-alternatives \ + --verbose \ + --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \ + --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ + --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ + --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ + --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ + --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ + --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ + --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ + --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ + --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ + --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ + --slave /usr/bin/lld lld /usr/bin/lld-${version} \ + --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ + --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ + --slave /usr/bin/llc llc /usr/bin/llc-${version} \ + --slave /usr/bin/lli lli /usr/bin/lli-${version} \ + --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ + --slave /usr/bin/opt opt /usr/bin/opt-${version} \ + --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ + --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ + --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ + --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} + +} + +register_clang_version $1 $2 diff --git a/build/utils.sh b/build/utils.sh index 9e9b72bf0..56b6ee3c7 100644 --- a/build/utils.sh +++ b/build/utils.sh @@ -1,154 +1,199 @@ -color_normal=$(tput sgr0) -color_bold=$(tput bold) -color_red="$color_bold$(tput setaf 1)" -color_green="$color_bold$(tput setaf 2)" -color_fawn=$(tput setaf 3); color_beige="$color_fawn" -color_yellow="$color_bold$color_fawn" -color_darkblue=$(tput setaf 4) +#!/bin/bash -color_blue="$color_bold$color_darkblue" -color_purple=$(tput setaf 5); color_magenta="$color_purple" -color_pink="$color_bold$color_purple" -color_darkcyan=$(tput setaf 6) -color_cyan="$color_bold$color_darkcyan" -color_gray=$(tput setaf 7) -color_darkgray="$color_bold"$(tput setaf 0) -color_white="$color_bold$color_gray" - - -if [[ $(tput colors) == '256' ]]; then - color_red=$(tput setaf 196) - color_yellow=$(tput setaf 228) - color_cyan=$(tput setaf 87) - color_green=$(tput setaf 156) - color_darkgray=$(tput setaf 59) +if [[ -n "$TERM" && "$TERM" != "dumb" && $(command -v tput) ]]; then + TPUT_AVAILABLE=true +else + TPUT_AVAILABLE=false fi -message() -{ - echo $color_cyan ・ $@$color_normal +if [[ $TPUT_AVAILABLE == true ]]; then + color_normal=$(tput sgr0) + color_bold=$(tput bold) + color_red="$color_bold$(tput setaf 1)" + color_green="$color_bold$(tput setaf 2)" + color_fawn=$(tput setaf 3) + color_beige="$color_fawn" + color_yellow="$color_bold$color_fawn" + color_darkblue=$(tput setaf 4) + + color_blue="$color_bold$color_darkblue" + color_purple=$(tput setaf 5) + color_magenta="$color_purple" + color_pink="$color_bold$color_purple" + color_darkcyan=$(tput setaf 6) + color_cyan="$color_bold$color_darkcyan" + color_gray=$(tput setaf 7) + color_darkgray="$color_bold"$(tput setaf 0) + color_white="$color_bold$color_gray" + + if [[ $(tput colors) == '256' ]]; then + color_red=$(tput setaf 196) + color_yellow=$(tput setaf 228) + color_cyan=$(tput setaf 87) + color_green=$(tput setaf 156) + color_darkgray=$(tput setaf 59) + fi + +else + # Basic attributes + color_normal="\e[0m" # sgr0 + color_bold="\e[1m" # bold + + # Standard 8-color palette + color_red="\e[1;31m" # bold + setaf 1 + color_green="\e[1;32m" # bold + setaf 2 + color_fawn="\e[33m" # setaf 3 + color_beige="$color_fawn" # alias + color_yellow="\e[1;33m" # bold + setaf 3 + color_darkblue="\e[34m" # setaf 4 + color_blue="\e[1;34m" # bold + setaf 4 + color_purple="\e[35m" # setaf 5 + color_magenta="$color_purple" # alias + color_pink="\e[1;35m" # bold + setaf 5 + color_darkcyan="\e[36m" # setaf 6 + color_cyan="\e[1;36m" # bold + setaf 6 + color_gray="\e[37m" # setaf 7 + color_darkgray="\e[1;30m" # bold + setaf 0 + color_white="\e[1;37m" # bold + setaf 7 + if [ "$TERM" = "xterm-256color" ] || [ "$COLORTERM" = "truecolor" ] || [ "$COLORTERM" = "24bit" ]; then + # Only set 256-color codes if actually in a 256-color terminal + color_red="\e[91m" # bright red + color_yellow="\e[93m" # light yellow + color_cyan="\e[96m" # bright cyan + color_green="\e[92m" # light green + color_darkgray="\e[90m" # dark gray + fi +fi + +message() { + echo -e $color_cyan ・ $@$color_normal } -warn() -{ - echo $color_yellow ・ $@$color_normal +warn() { + echo -e $color_yellow ・ $@$color_normal } -error() -{ - echo $color_red ・ $@$color_normal +error() { + echo -e $color_red ・ $@$color_normal } -message_split() -{ - echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal +message_split() { + echo -e $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal } -message_splitted() -{ - message_split +message_splitted() { + message_split + if [[ $TPUT_AVAILABLE == true ]]; then echo $color_green ・ $@$color_normal - message_split + else + echo "$@" + fi + message_split } - -colorify_array() -{ - PROMT="" - for a in "$@" - do - i=$((((i+1) % (123-106)) + 106)) - if [[ $(tput colors) == '256' ]]; then - PROMT="$PROMT $(tput setaf $i)$a$color_normal" - else - PROMT="$PROMT $a" - fi - done - echo $PROMT -} - - -newline_array() -{ - PROMT="" - for a in "$@" - do - PROMT="$PROMT$a\n" - done - echo -e $PROMT -} - - -function spinner -{ - freq=${1:-10} - points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷) - colored_points=($(colorify_array ${points[@]})) - len=${#points[@]} - point_num=0 - line_num=0 - while read data; do - line_num=$((line_num+1)) - if [[ $((line_num % freq)) = 0 ]]; then - point_num=$(((point_num + 1) % len )) - echo -ne "\r${colored_points[point_num]}" - fi - done; - echo -} - - -detect_distro() -{ - if [ -f /etc/os-release ]; then - . /etc/os-release - export OS=$NAME - export OS_VERSION=$VERSION_ID - elif type lsb_release >/dev/null 2>&1; then - # linuxbase.org - export OS=$(lsb_release -si) - export OS_VERSION=$(lsb_release -sr) - elif [ -f /etc/lsb-release ]; then - # For some versions of Debian/Ubuntu without lsb_release command - . /etc/lsb-release - export OS=$DISTRIB_ID - OS_VERSION=$DISTRIB_RELEASE - elif [ -f /etc/debian_version ]; then - # Older Debian/Ubuntu/etc. - OS=Debian - OS_VERSION=$(cat /etc/debian_version) +colorify_array() { + PROMT="" + for a in "$@"; do + if [[ $TPUT_AVAILABLE == true ]]; then + i=$((((i + 1) % (123 - 106)) + 106)) + if [[ $(tput colors) == '256' ]]; then + PROMT="$PROMT $(tput setaf $i)$a$color_normal" + else + PROMT="$PROMT $a" + fi else - # Fall back to uname, e.g. "Linux ", also works for BSD, etc. - OS=$(uname -s) - OS_VERSION=$(uname -r) + PROMT="$PROMT $a" fi - OS=$(echo $OS | cut -f 1 -d " ") - message "Detected $color_yellow$OS $OS_VERSION$color_normal" + done + echo $PROMT +} + +newline_array() { + PROMT="" + for a in "$@"; do + PROMT="$PROMT$a\n" + done + echo -e $PROMT +} + +function spinner() { + freq=${1:-10} + points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷) + colored_points=($(colorify_array ${points[@]})) + len=${#points[@]} + point_num=0 + line_num=0 + while read data; do + line_num=$((line_num + 1)) + if [[ $((line_num % freq)) = 0 ]]; then + point_num=$(((point_num + 1) % len)) + if [[ $TPUT_AVAILABLE == true ]]; then + echo -ne "\r${colored_points[point_num]}" + else + echo -ne "\r${points[point_num]}" + fi + fi + done + echo +} + +function onelinearizator() { + while read data; do + echo -ne "\r\e[K$data" + done + echo +} + +detect_distro() { + if [ -f /etc/os-release ]; then + . /etc/os-release + export OS=$NAME + export OS_VERSION=$VERSION_ID + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + export OS=$(lsb_release -si) + export OS_VERSION=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + export OS=$DISTRIB_ID + OS_VERSION=$DISTRIB_RELEASE + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + OS_VERSION=$(cat /etc/debian_version) + else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + OS=$(uname -s) + OS_VERSION=$(uname -r) + fi + OS=$(echo $OS | cut -f 1 -d " " | tr '[:upper:]' '[:lower:]')":"$OS_VERSION + message "Detected $color_yellow$OS $OS_VERSION$color_normal" } menuStr="" -function hideCursor(){ - printf "\033[?25l" +function hideCursor() { + printf "\e[?25l" # capture CTRL+C so cursor can be reset trap "showCursor && exit 0" 2 } -function showCursor(){ - printf "\033[?25h" +function showCursor() { + printf "\e[?25h" } -function clearLastMenu(){ +function clearLastMenu() { local msgLineCount=$(printf "$menuStr" | wc -l) # moves the curser up N lines so the output overwrites it - echo -en "\033[${msgLineCount}A" + echo -en "\e[${msgLineCount}A" # clear to end of screen to ensure there's no text left behind from previous input [ $1 ] && tput ed } -function renderMenu(){ +function renderMenu() { local start=0 local selector="" local instruction="$1" @@ -160,8 +205,8 @@ function renderMenu(){ # Get the longest item from the list so that we know how many spaces to add # to ensure there's no overlap from longer items when a list is scrolling up or down. - for (( i=0; i<$itemsLength; i++ )); do - if (( ${#menuItems[i]} > longest )); then + for ((i = 0; i < $itemsLength; i++)); do + if ((${#menuItems[i]} > longest)); then longest=${#menuItems[i]} fi done @@ -171,12 +216,12 @@ function renderMenu(){ listLength=$3 if [ $selectedIndex -ge $listLength ]; then - start=$(($selectedIndex+1-$listLength)) - listLength=$(($selectedIndex+1)) + start=$(($selectedIndex + 1 - $listLength)) + listLength=$(($selectedIndex + 1)) fi fi - for (( i=$start; i<$listLength; i++ )); do + for ((i = $start; i < $listLength; i++)); do local currItem="${menuItems[i]}" currItemLength=${#currItem} @@ -201,9 +246,9 @@ function renderMenu(){ printf "${menuStr}" } -function getChoice(){ - local KEY__ARROW_UP=$(echo -e "\033[A") - local KEY__ARROW_DOWN=$(echo -e "\033[B") +function getChoice() { + local KEY__ARROW_UP=$(echo -e "\e[A") + local KEY__ARROW_DOWN=$(echo -e "\e[B") local KEY__ENTER=$(echo -e "\n") local captureInput=true local displayHelp=false @@ -216,55 +261,55 @@ function getChoice(){ key="$1" case $key in - -h|--help) - displayHelp=true - shift - ;; - -i|--index) - selectedIndex=$2 - shift 2 - ;; - -m|--max) - maxViewable=$2 - shift 2 - ;; - -o|--options) - menuItems=$2[@] - menuItems=("${!menuItems}") - shift 2 - ;; - -q|--query) - instruction="$2" - shift 2 - ;; - *) - remainingArgs+=("$1") - shift - ;; + -h | --help) + displayHelp=true + shift + ;; + -i | --index) + selectedIndex=$2 + shift 2 + ;; + -m | --max) + maxViewable=$2 + shift 2 + ;; + -o | --options) + menuItems=$2[@] + menuItems=("${!menuItems}") + shift 2 + ;; + -q | --query) + instruction="$2" + shift 2 + ;; + *) + remainingArgs+=("$1") + shift + ;; esac done # just display help if $displayHelp; then - echo; + echo echo "Usage: getChoice [OPTION]..." echo "Renders a keyboard navigable menu with a visual indicator of what's selected." - echo; + echo echo " -h, --help Displays this message" echo " -i, --index The initially selected index for the options" echo " -m, --max Limit how many options are displayed" echo " -o, --options An Array of options for a User to choose from" echo " -q, --query Question or statement presented to the User" - echo; + echo echo "Example:" echo " foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")" - echo; + echo echo " getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4" echo " printf \"\\n First choice is '\${selectedChoice}'\\n\"" - echo; + echo echo " getChoice -q \"Select another option in case the first isn't available\" -o foodOptions" echo " printf \"\\n Second choice is '\${selectedChoice}'\\n\"" - echo; + echo return 0 fi @@ -282,105 +327,104 @@ function getChoice(){ hideCursor while $captureInput; do - read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters. + read -rsn3 key # `3` captures the escape (\e'), bracket ([), & type (A) characters. case "$key" in - "$KEY__ARROW_UP") - selectedIndex=$((selectedIndex-1)) - (( $selectedIndex < 0 )) && selectedIndex=$((itemsLength-1)) + "$KEY__ARROW_UP") + selectedIndex=$((selectedIndex - 1)) + (($selectedIndex < 0)) && selectedIndex=$((itemsLength - 1)) - renderMenu "$instruction" $selectedIndex $maxViewable true - ;; + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; - "$KEY__ARROW_DOWN") - selectedIndex=$((selectedIndex+1)) - (( $selectedIndex == $itemsLength )) && selectedIndex=0 + "$KEY__ARROW_DOWN") + selectedIndex=$((selectedIndex + 1)) + (($selectedIndex == $itemsLength)) && selectedIndex=0 - renderMenu "$instruction" $selectedIndex $maxViewable true - ;; + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; - "$KEY__ENTER") - clearLastMenu true - showCursor - captureInput=false - ;; + "$KEY__ENTER") + clearLastMenu true + showCursor + captureInput=false + ;; esac done } - -function optparse.throw_error(){ +function optparse.throw_error() { local message="$1" - error "OPTPARSE: ERROR: $message" - exit 1 + error "OPTPARSE: ERROR: $message" + exit 1 } # ----------------------------------------------------------------------------------------------------------------------------- -function optparse.define(){ - if [ $# -lt 3 ]; then - optparse.throw_error "optparse.define [] [] []" - fi - for option_id in $( seq 1 $# ) ; do - local option="$( eval "echo \$$option_id")" - local key="$( echo $option | awk -F "=" '{print $1}' )"; - local value="$( echo $option | awk -F "=" '{print $2}' )"; +function optparse.define() { + if [ $# -lt 3 ]; then + optparse.throw_error "optparse.define [] [] []" + fi + for option_id in $(seq 1 $#); do + local option="$(eval "echo \$$option_id")" + local key="$(echo $option | awk -F "=" '{print $1}')" + local value="$(echo $option | awk -F "=" '{print $2}')" - #essentials: shortname, longname, description - if [ "$key" = "short" ]; then - local shortname="$value" - if [ ${#shortname} -ne 1 ]; then - optparse.throw_error "short name expected to be one character long" - fi - local short="-${shortname}" - elif [ "$key" = "long" ]; then - local longname="$value" - if [ ${#longname} -lt 2 ]; then - optparse.throw_error "long name expected to be atleast one character long" - fi - local long="--${longname}" - elif [ "$key" = "desc" ]; then - local desc="$value" - elif [ "$key" = "default" ]; then - local default="$value" - elif [ "$key" = "variable" ]; then - local variable="$value" - elif [ "$key" = "value" ]; then - local val="$value" - fi - done + #essentials: shortname, longname, description + if [ "$key" = "short" ]; then + local shortname="$value" + if [ ${#shortname} -ne 1 ]; then + optparse.throw_error "short name expected to be one character long" + fi + local short="-${shortname}" + elif [ "$key" = "long" ]; then + local longname="$value" + if [ ${#longname} -lt 2 ]; then + optparse.throw_error "long name expected to be atleast one character long" + fi + local long="--${longname}" + elif [ "$key" = "desc" ]; then + local desc="$value" + elif [ "$key" = "default" ]; then + local default="$value" + elif [ "$key" = "variable" ]; then + local variable="$value" + elif [ "$key" = "value" ]; then + local val="$value" + fi + done - if [ "$variable" = "" ]; then - optparse.throw_error "You must give a variable for option: ($short/$long)" - fi + if [ "$variable" = "" ]; then + optparse.throw_error "You must give a variable for option: ($short/$long)" + fi - if [ "$val" = "" ]; then - val="\$OPTARG" - fi + if [ "$val" = "" ]; then + val="\$OPTARG" + fi - # build OPTIONS and help - optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")" - if [ "$default" != "" ]; then - optparse_usage="${optparse_usage} [default:$default]" - fi - optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;" - if [ "$default" != "" ]; then - optparse_defaults="${optparse_defaults}#NL${variable}=${default}" - fi - optparse_arguments_string="${optparse_arguments_string}${shortname}" - if [ "$val" = "\$OPTARG" ]; then - optparse_arguments_string="${optparse_arguments_string}:" - fi - optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;" + # build OPTIONS and help + optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")" + if [ "$default" != "" ]; then + optparse_usage="${optparse_usage} [default:$default]" + fi + optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;" + if [ "$default" != "" ]; then + optparse_defaults="${optparse_defaults}#NL${variable}=${default}" + fi + optparse_arguments_string="${optparse_arguments_string}${shortname}" + if [ "$val" = "\$OPTARG" ]; then + optparse_arguments_string="${optparse_arguments_string}:" + fi + optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;" } # ----------------------------------------------------------------------------------------------------------------------------- -function optparse.build(){ - local build_file="$(mktemp -t "optparse-XXXXXX.tmp")" +function optparse.build() { + local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")" - # Building getopts header here + # Building getopts header here - # Function usage - cat << EOF > $build_file + # Function usage + cat <$build_file function usage(){ cat << XXX usage: \$0 [OPTIONS] @@ -428,19 +472,108 @@ done rm $build_file EOF - local -A o=( ['#NL']='\n' ['#TB']='\t' ) + local -A o=(['#NL']='\n' ['#TB']='\t') - for i in "${!o[@]}"; do - sed -i "s/${i}/${o[$i]}/g" $build_file - done + for i in "${!o[@]}"; do + sed -i "s/${i}/${o[$i]}/g" $build_file + done - # Unset global variables - unset optparse_usage - unset optparse_process - unset optparse_arguments_string - unset optparse_defaults - unset optparse_contractions + # Unset global variables + unset optparse_usage + unset optparse_process + unset optparse_arguments_string + unset optparse_defaults + unset optparse_contractions - # Return file name to parent - echo "$build_file" -} \ No newline at end of file + # Return file name to parent + echo "$build_file" +} + +function retry_eval() { + if [ "$#" -lt 2 ]; then + error "Usage: retry_eval " + return 1 + fi + + local max_retries=$1 + shift # Remove max_retries from arguments + local attempt=1 + local initial_delay=1 + + while [ "$attempt" -le "$max_retries" ]; do + message_split + message "Attempt $attempt of $max_retries: $*" + if eval "$@"; then + message "Command '$@' done" + message_split + return 0 + fi + if [ "$attempt" -lt "$max_retries" ]; then + delay=$((initial_delay * 2 ** (attempt - 1))) + warn "Retrying command "$@" in $delay seconds..." + message_split + sleep "$delay" + fi + ((attempt++)) + done + + error "Max retries reached for command: $*" + message_split + exit 13 +} + +function execInnerDocker() { + local container_name=$1 + shift 1 # Remove first arg (container_name) + + docker exec -t "$container_name" bash -c "$@" + local dockerCommandExitCode=$? + + if [[ $dockerCommandExitCode -ne 0 ]]; then + error "Command \"$@\" failed in container \"$container_name\"" + exit $dockerCommandExitCode + fi +} + +function change_ubuntu_mirror() { + local region="$1" + message "Changing Ubuntu mirror to $region" + sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list 2>/dev/null || true + sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || true + cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null | grep archive || true + message_split +} + +function execInnerDockerWithRetry() { + local max_retries=5 + local container_name=$1 + shift 1 + + local cmd=("$@") + local attempt=1 + local dockerCommandExitCode=0 + + local docker_funcs=$(declare -f retry_eval color_normal color_cyan color_yellow color_red error warn message message_split) + + # Build the full command to execute in docker + local full_command="$docker_funcs; retry_eval $max_retries \"${cmd[*]}\"" + + # Execute the command in docker + docker exec -t "$container_name" bash -c "$full_command" + dockerCommandExitCode=$? + + if [[ $dockerCommandExitCode -ne 0 ]]; then + error "Command \"${cmd[*]}\" failed in container \"$container_name\" after $max_retries attempts" + return $dockerCommandExitCode + fi + + return 0 +} + +change_ubuntu_mirror_in_docker() { + local container_name=$1 + local region=$2 + local docker_funcs=$(declare -f color_normal color_cyan color_yellow color_red error warn message message_split change_ubuntu_mirror) + + execInnerDocker "$container_name" "$docker_funcs; change_ubuntu_mirror ${region}" +} diff --git a/cmake-format.py b/cmake-format.py new file mode 100644 index 000000000..2d232d51b --- /dev/null +++ b/cmake-format.py @@ -0,0 +1,241 @@ +# ---------------------------------- +# Options affecting listfile parsing +# ---------------------------------- +with section("parse"): + + # Specify structure for custom cmake functions + additional_commands = { 'foo': { 'flags': ['BAR', 'BAZ'], + 'kwargs': {'DEPENDS': '*', 'HEADERS': '*', 'SOURCES': '*'}}} + + # Override configurations per-command where available + override_spec = {} + + # Specify variable tags. + vartags = [] + + # Specify property tags. + proptags = [] + +# ----------------------------- +# Options affecting formatting. +# ----------------------------- +with section("format"): + + # Disable formatting entirely, making cmake-format a no-op + disable = False + + # How wide to allow formatted cmake files + line_width = 120 + + # How many spaces to tab for indent + tab_size = 4 + + # If true, lines are indented using tab characters (utf-8 0x09) instead of + # space characters (utf-8 0x20). In cases where the layout would + # require a fractional tab character, the behavior of the fractional + # indentation is governed by + use_tabchars = False + + # If is True, then the value of this variable indicates how + # fractional indentions are handled during whitespace replacement. If set to + # 'use-space', fractional indentation is left as spaces (utf-8 0x20). If set + # to `round-up` fractional indentation is replaced with a single tab character + # (utf-8 0x09) effectively shifting the column to the next tabstop + fractional_tab_policy = 'use-space' + + # If an argument group contains more than this many sub-groups (parg or kwarg + # groups) then force it to a vertical layout. + max_subgroups_hwrap = 2 + + # If a positional argument group contains more than this many arguments, then + # force it to a vertical layout. + max_pargs_hwrap = 6 + + # If a cmdline positional group consumes more than this many lines without + # nesting, then invalidate the layout (and nest) + max_rows_cmdline = 2 + + # If true, separate flow control names from their parentheses with a space + separate_ctrl_name_with_space = False + + # If true, separate function names from parentheses with a space + separate_fn_name_with_space = False + + # If a statement is wrapped to more than one line, than dangle the closing + # parenthesis on its own line. + dangle_parens = True + + # If the trailing parenthesis must be 'dangled' on its on line, then align it + # to this reference: `prefix`: the start of the statement, `prefix-indent`: + # the start of the statement, plus one indentation level, `child`: align to + # the column of the arguments + dangle_align = 'prefix' + + # If the statement spelling length (including space and parenthesis) is + # smaller than this amount, then force reject nested layouts. + min_prefix_chars = 4 + + # If the statement spelling length (including space and parenthesis) is larger + # than the tab width by more than this amount, then force reject un-nested + # layouts. + max_prefix_chars = 10 + + # If a candidate layout is wrapped horizontally but it exceeds this many + # lines, then reject the layout. + max_lines_hwrap = 2 + + # What style line endings to use in the output. + line_ending = 'unix' + + # Format command names consistently as 'lower' or 'upper' case + command_case = 'canonical' + + # Format keywords consistently as 'lower' or 'upper' case + keyword_case = 'unchanged' + + # A list of command names which should always be wrapped + always_wrap = [] + + # If true, the argument lists which are known to be sortable will be sorted + # lexicographicall + enable_sort = True + + # If true, the parsers may infer whether or not an argument list is sortable + # (without annotation). + autosort = False + + # By default, if cmake-format cannot successfully fit everything into the + # desired linewidth it will apply the last, most aggressive attempt that it + # made. If this flag is True, however, cmake-format will print error, exit + # with non-zero status code, and write-out nothing + require_valid_layout = False + + # A dictionary mapping layout nodes to a list of wrap decisions. See the + # documentation for more information. + layout_passes = {} + +# ------------------------------------------------ +# Options affecting comment reflow and formatting. +# ------------------------------------------------ +with section("markup"): + + # What character to use for bulleted lists + bullet_char = '*' + + # What character to use as punctuation after numerals in an enumerated list + enum_char = '.' + + # If comment markup is enabled, don't reflow the first comment block in each + # listfile. Use this to preserve formatting of your copyright/license + # statements. + first_comment_is_literal = False + + # If comment markup is enabled, don't reflow any comment block which matches + # this (regex) pattern. Default is `None` (disabled). + literal_comment_pattern = None + + # Regular expression to match preformat fences in comments default= + # ``r'^\s*([`~]{3}[`~]*)(.*)$'`` + fence_pattern = '^\\s*([`~]{3}[`~]*)(.*)$' + + # Regular expression to match rulers in comments default= + # ``r'^\s*[^\w\s]{3}.*[^\w\s]{3}$'`` + ruler_pattern = '^\\s*[^\\w\\s]{3}.*[^\\w\\s]{3}$' + + # If a comment line matches starts with this pattern then it is explicitly a + # trailing comment for the preceding argument. Default is '#<' + explicit_trailing_pattern = '#<' + + # If a comment line starts with at least this many consecutive hash + # characters, then don't lstrip() them off. This allows for lazy hash rulers + # where the first hash char is not separated by space + hashruler_min_length = 10 + + # If true, then insert a space between the first hash char and remaining hash + # chars in a hash ruler, and normalize its length to fill the column + canonicalize_hashrulers = True + + # enable comment markup parsing and reflow + enable_markup = True + +# ---------------------------- +# Options affecting the linter +# ---------------------------- +with section("lint"): + + # a list of lint codes to disable + disabled_codes = [] + + # regular expression pattern describing valid function names + function_pattern = '[0-9a-z_]+' + + # regular expression pattern describing valid macro names + macro_pattern = '[0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with global + # (cache) scope + global_var_pattern = '[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with global + # scope (but internal semantic) + internal_var_pattern = '_[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with local + # scope + local_var_pattern = '[a-z][a-z0-9_]+' + + # regular expression pattern describing valid names for privatedirectory + # variables + private_var_pattern = '_[0-9a-z_]+' + + # regular expression pattern describing valid names for public directory + # variables + public_var_pattern = '[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for function/macro + # arguments and loop variables. + argument_var_pattern = '[a-z][a-z0-9_]+' + + # regular expression pattern describing valid names for keywords used in + # functions or macros + keyword_pattern = '[A-Z][0-9A-Z_]+' + + # In the heuristic for C0201, how many conditionals to match within a loop in + # before considering the loop a parser. + max_conditionals_custom_parser = 2 + + # Require at least this many newlines between statements + min_statement_spacing = 1 + + # Require no more than this many newlines between statements + max_statement_spacing = 2 + max_returns = 6 + max_branches = 12 + max_arguments = 5 + max_localvars = 15 + max_statements = 50 + +# ------------------------------- +# Options affecting file encoding +# ------------------------------- +with section("encode"): + + # If true, emit the unicode byte-order mark (BOM) at the start of the file + emit_byteorder_mark = False + + # Specify the encoding of the input file. Defaults to utf-8 + input_encoding = 'utf-8' + + # Specify the encoding of the output file. Defaults to utf-8. Note that cmake + # only claims to support utf-8 so be careful when using anything else + output_encoding = 'utf-8' + +# ------------------------------------- +# Miscellaneous configurations options. +# ------------------------------------- +with section("misc"): + + # A dictionary containing any per-command configuration overrides. Currently + # only `command_case` is supported. + per_command = {} + diff --git a/cmake/ColumnstoreLibrary.cmake b/cmake/ColumnstoreLibrary.cmake new file mode 100644 index 000000000..36dbc6813 --- /dev/null +++ b/cmake/ColumnstoreLibrary.cmake @@ -0,0 +1,105 @@ +set(DEBIAN_INSTALL_FILE "${CMAKE_BINARY_DIR}/mariadb-plugin-columnstore.install.generated") +file(WRITE ${DEBIAN_INSTALL_FILE} "#File is generated by ColumnstoreLibrary.cmake, do not edit\n") + +macro(add_to_debian_install_file file_path) + string(SUBSTRING "${file_path}" 1 -1 BINARY_ENTRY) + file(RELATIVE_PATH CMAKEFILE "${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore" ${CMAKE_CURRENT_LIST_FILE}) + string(STRIP "${BINARY_ENTRY}" BINARY_ENTRY) + + file(APPEND ${DEBIAN_INSTALL_FILE} "${BINARY_ENTRY} # added in ${CMAKEFILE}\n") +endmacro() + +function(get_target_output_filename TARGET_NAME OUTPUT_VAR) + # 1. Get the target's OUTPUT_NAME (falls back to TARGET_NAME) + get_target_property(OUTPUT_NAME ${TARGET_NAME} OUTPUT_NAME) + if(NOT OUTPUT_NAME) + set(OUTPUT_NAME "${TARGET_NAME}") + endif() + + # 1. Get the correct suffix based on target type + get_target_property(TARGET_TYPE ${TARGET_NAME} TYPE) + if(TARGET_TYPE STREQUAL "EXECUTABLE") + set(SUFFIX "${CMAKE_EXECUTABLE_SUFFIX}") + set(PREFIX "") # No prefix for executables + elseif(TARGET_TYPE STREQUAL "SHARED_LIBRARY") + set(SUFFIX "${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(PREFIX "lib") + elseif(TARGET_TYPE STREQUAL "STATIC_LIBRARY") + set(SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(PREFIX "lib") + else() + message(WARNING "Unknown target type for ${TARGET_NAME}") + set(SUFFIX "") + endif() + + # 1. Combine into the final filename + set(${OUTPUT_VAR} + "${PREFIX}${OUTPUT_NAME}${SUFFIX}" + PARENT_SCOPE + ) +endfunction() + +macro(columnstore_install_target target destination) + install( + TARGETS ${target} + DESTINATION ${destination} + COMPONENT columnstore-engine + ) + get_target_output_filename(${target} OUTPUT_FILENAME) + add_to_debian_install_file("${destination}/${OUTPUT_FILENAME}") +endmacro() + +macro(columnstore_install_file file destination) + install( + FILES ${file} + DESTINATION ${destination} + COMPONENT columnstore-engine + ) + get_filename_component(FILENAME ${file} NAME) + add_to_debian_install_file("${destination}/${FILENAME}") +endmacro() + +macro(columnstore_install_program file destination) + install( + PROGRAMS ${file} + DESTINATION ${destination} + COMPONENT columnstore-engine + ) + get_filename_component(FILENAME ${file} NAME) + add_to_debian_install_file( + "${destination}/${FILENAME} + " + ) +endmacro() + +macro(columnstore_static_library libname) + add_definitions(-fPIC -DPIC) + add_library(${libname} STATIC ${ARGN}) +endmacro() + +macro(columnstore_shared_library libname) + add_library(${libname} SHARED ${ARGN}) + columnstore_install_target(${libname} ${ENGINE_LIBDIR}) +endmacro() + +macro(columnstore_library libname) + if(COLUMNSTORE_STATIC_LIBRARIES) + columnstore_static_library(${libname} ${ARGN}) + else() + columnstore_shared_library(${libname} ${ARGN}) + endif() +endmacro() + +macro(columnstore_mysql_plugin_library libname) + add_library(${libname} SHARED ${ARGN}) + columnstore_install_target(${libname} ${MARIADB_PLUGINDIR}) +endmacro() + +macro(columnstore_link libname) + target_link_libraries(${libname} ${ARGN}) +endmacro() + +macro(columnstore_executable executable_name) + add_executable(${executable_name} ${ARGN}) + columnstore_install_target(${executable_name} ${ENGINE_BINDIR}) +endmacro() diff --git a/cmake/boost.1.88.named_proxy.hpp.patch b/cmake/boost.1.88.named_proxy.hpp.patch new file mode 100644 index 000000000..30db7ecc2 --- /dev/null +++ b/cmake/boost.1.88.named_proxy.hpp.patch @@ -0,0 +1,10 @@ ++++ a/boost/interprocess/detail/named_proxy.hpp 2025-04-14 16:24:12.018395298 +0000 ++++ b/boost/interprocess/detail/named_proxy.hpp 2025-04-14 16:24:12.018395298 +0000 +@@ -89,6 +89,7 @@ + } BOOST_INTERPROCESS_CATCH_END + } + ++ virtual ~CtorArgN() {} + private: + template + void construct(void *mem, true_, const index_tuple&) diff --git a/cmake/columnstore_version.cmake b/cmake/columnstore_version.cmake index c8efb4092..cb9f85086 100644 --- a/cmake/columnstore_version.cmake +++ b/cmake/columnstore_version.cmake @@ -29,6 +29,7 @@ MACRO(GET_COLUMNSTORE_VERSION) SET(CMAPI_VERSION_MAJOR ${CS_MAJOR_VERSION}) SET(CMAPI_VERSION_MINOR ${CS_MINOR_VERSION}) SET(CMAPI_VERSION_PATCH ${CS_PATCH_VERSION}${CS_EXTRA_VERSION}) + SET(CMAPI_VERSION_RELEASE ${CS_RELEASE_VERSION}) ENDIF() IF(NOT "${CS_MAJOR_VERSION}" MATCHES "[0-9]+" OR @@ -39,11 +40,6 @@ IF(NOT "${CS_MAJOR_VERSION}" MATCHES "[0-9]+" OR SET(VERSION "${CS_MAJOR_VERSION}.${CS_MINOR_VERSION}.${CS_PATCH_VERSION}${CS_EXTRA_VERSION}") MESSAGE("== MariaDB-Columnstore ${VERSION}") - IF (NOT INSTALL_LAYOUT) - SET(CPACK_PACKAGE_VERSION_MAJOR ${CS_MAJOR_VERSION}) - SET(CPACK_PACKAGE_VERSION_MINOR ${CS_MINOR_VERSION}) - SET(CPACK_PACKAGE_VERSION_PATCH ${CS_PATCH_VERSION}${CS_EXTRA_VERSION}) - ENDIF () SET(PACKAGE_VERSION "${CS_MAJOR_VERSION}.${CS_MINOR_VERSION}.${CS_PATCH_VERSION}${CS_EXTRA_VERSION}") SET(PACKAGE_RELEASE "${CS_RELEASE_VERSION}") MATH(EXPR MCSVERSIONHEX "${CS_MAJOR_VERSION} * 256 + ${CS_MINOR_VERSION}" OUTPUT_FORMAT HEXADECIMAL) diff --git a/cmake/compiler_flags.cmake b/cmake/compiler_flags.cmake new file mode 100644 index 000000000..312574a80 --- /dev/null +++ b/cmake/compiler_flags.cmake @@ -0,0 +1,120 @@ +macro(SET_FLAGS) + foreach(F ${ARGV}) + my_check_and_set_compiler_flag(${F} DEBUG RELWITHDEBINFO MINSIZEREL) + endforeach() +endmacro() + +macro(SET_FLAGS_DEBUG) + foreach(F ${ARGV}) + my_check_and_set_compiler_flag(${F} DEBUG) + endforeach() +endmacro() + +macro(SET_FLAGS_RELEASE) + foreach(F ${ARGV}) + my_check_and_set_compiler_flag(${F} RELWITHDEBINFO) + endforeach() +endmacro() + +# C++ standard { +if(have_CXX__std_c__20) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20") +else() + my_check_cxx_compiler_flag("-std=c++2a") + if(have_CXX__std_c__2a) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a") + else() + message_once(CS_NO_CXX20 "C++ Compiler does not understand -std=c++20") + return() + endif() +endif() + +unset(CMAKE_CXX_STANDARD) +# } end C++ standard + +# Hacks to keep alive with MariaDB server { +string(REPLACE -D_GLIBCXX_DEBUG "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) +string(REPLACE -D_GLIBCXX_ASSERTIONS "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) +# } end hacks + +if(WITH_COLUMNSTORE_ASAN) + set(WERROR_FLAG) +else() + set(WERROR_FLAG -Werror) +endif() + +# Maintainer flags, works when build is done via bootstrap_mcs.sh { +set(COLUMNSTORE_MAINTAINER_FLAGS ${WERROR_FLAG}) +# } end Maintainer flags + +# Release, Debug and common flags { +set(FLAGS_ALL + -Wall + -Wextra + -ggdb3 + -fno-omit-frame-pointer + -fno-strict-aliasing + -fsigned-char + -msse4.2 + -DHAVE_CONFIG_H + -DBOOST_BIND_GLOBAL_PLACEHOLDERS +) + +set(FLAGS_RELEASE -O3 -DDBUG_OFF) + +set(FLAGS_DEBUG -O0 -D_DEBUG) +# } end Release, Debug and common flags + +# linker flags { +set(ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") +# } end linker flags + +# compiler specific flags { +set(CLANG_FLAGS + # suppressed warnings + -Wno-unused-parameter + -Wno-cast-function-type-strict + -Wno-deprecated-copy + -Wno-deprecated-declarations + -Wno-deprecated-enum-enum-conversion + -Wno-format-truncation + -Wno-register + -Wno-typedef-redefinition + -Wno-missing-template-arg-list-after-template-kw + -Wno-unused-parameter +) + +set(GNU_FLAGS # suppressed warnings + -Wno-deprecated-copy -Wno-deprecated-declarations -Wno-format-truncation -Wno-register -Wno-unused-parameter +) + +# } end compiler specific flags + +# Sanitizers { +set(ASAN_FLAGS -U_FORTIFY_SOURCE -fsanitize=address -fsanitize-address-use-after-scope -fPIC) +# } end Sanitizers + +# configured by cmake/configureEngine.cmake { +if(MASK_LONGDOUBLE) + my_check_and_set_compiler_flag("-DMASK_LONGDOUBLE") +endif() +# } end configured by cmake/configureEngine.cmake + +# apply them all +set_flags(${FLAGS_ALL}) +set_flags_debug(${FLAGS_DEBUG}) +set_flags_release(${FLAGS_RELEASE}) + +if(COLUMNSTORE_MAINTAINER) + set_flags(${COLUMNSTORE_MAINTAINER_FLAGS}) +endif() + +if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set_flags(${CLANG_FLAGS}) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set_flags(${GNU_FLAGS}) +endif() + +if(WITH_COLUMNSTORE_ASAN) + set_flags(${ASAN_FLAGS}) +endif() diff --git a/cmake/configureEngine.cmake b/cmake/configureEngine.cmake index 305f9f9a8..cd8bff26a 100644 --- a/cmake/configureEngine.cmake +++ b/cmake/configureEngine.cmake @@ -1,147 +1,143 @@ - - # # Tests for header files # -INCLUDE (CheckIncludeFiles) -INCLUDE (CheckIncludeFileCXX) -INCLUDE (CheckCSourceCompiles) -INCLUDE (CheckCXXSourceRuns) -INCLUDE (CheckCXXSourceCompiles) -INCLUDE (CheckStructHasMember) -INCLUDE (CheckLibraryExists) -INCLUDE (CheckFunctionExists) -INCLUDE (CheckCCompilerFlag) -INCLUDE (CheckCXXCompilerFlag) -INCLUDE (CheckCXXSourceRuns) -INCLUDE (CheckSymbolExists) -INCLUDE (CheckCXXSymbolExists) -INCLUDE (CheckTypeSize) +include(CheckIncludeFiles) +include(CheckIncludeFileCXX) +include(CheckCSourceCompiles) +include(CheckCXXSourceRuns) +include(CheckCXXSourceCompiles) +include(CheckStructHasMember) +include(CheckLibraryExists) +include(CheckFunctionExists) +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) +include(CheckCXXSourceRuns) +include(CheckSymbolExists) +include(CheckCXXSymbolExists) +include(CheckTypeSize) -CHECK_INCLUDE_FILE_CXX (alloca.h HAVE_ALLOCA_H) -CHECK_INCLUDE_FILE_CXX (arpa/inet.h HAVE_ARPA_INET_H) -CHECK_INCLUDE_FILE_CXX (dlfcn.h HAVE_DLFCN_H) -CHECK_INCLUDE_FILE_CXX (fcntl.h HAVE_FCNTL_H) -CHECK_INCLUDE_FILE_CXX (inttypes.h HAVE_INTTYPES_H) -CHECK_INCLUDE_FILE_CXX (limits.h HAVE_LIMITS_H) -CHECK_INCLUDE_FILE_CXX (malloc.h HAVE_MALLOC_H) -CHECK_INCLUDE_FILE_CXX (memory.h HAVE_MEMORY_H) -CHECK_INCLUDE_FILE_CXX (ncurses.h HAVE_NCURSES_H) -CHECK_INCLUDE_FILE_CXX (netdb.h HAVE_NETDB_H) -CHECK_INCLUDE_FILE_CXX (netinet/in.h HAVE_NETINET_IN_H) -CHECK_INCLUDE_FILE_CXX (stddef.h HAVE_STDDEF_H) -CHECK_INCLUDE_FILE_CXX (stdint.h HAVE_STDINT_H) -CHECK_INCLUDE_FILE_CXX (stdlib.h HAVE_STDLIB_H) -CHECK_INCLUDE_FILE_CXX (strings.h HAVE_STRINGS_H) -CHECK_INCLUDE_FILE_CXX (string.h HAVE_STRING_H) -CHECK_INCLUDE_FILE_CXX (syslog.h HAVE_SYSLOG_H) -CHECK_INCLUDE_FILE_CXX (sys/file.h HAVE_SYS_FILE_H) -CHECK_INCLUDE_FILE_CXX (sys/mount.h HAVE_SYS_MOUNT_H) -CHECK_INCLUDE_FILE_CXX (sys/select.h HAVE_SYS_SELECT_H) -CHECK_INCLUDE_FILE_CXX (sys/socket.h HAVE_SYS_SOCKET_H) -CHECK_INCLUDE_FILE_CXX (sys/statfs.h HAVE_SYS_STATFS_H) -CHECK_INCLUDE_FILE_CXX (sys/stat.h HAVE_SYS_STAT_H) -CHECK_INCLUDE_FILE_CXX (sys/timeb.h HAVE_SYS_TIMEB_H) -CHECK_INCLUDE_FILE_CXX (sys/time.h HAVE_SYS_TIME_H) -CHECK_INCLUDE_FILE_CXX (sys/types.h HAVE_SYS_TYPES_H) -CHECK_INCLUDE_FILE_CXX (sys/wait.h HAVE_SYS_WAIT_H) -CHECK_INCLUDE_FILE_CXX (unistd.h HAVE_UNISTD_H) -CHECK_INCLUDE_FILE_CXX (utime.h HAVE_UTIME_H) -CHECK_INCLUDE_FILE_CXX (values.h HAVE_VALUES_H) -CHECK_INCLUDE_FILE_CXX (vfork.h HAVE_VFORK_H) -CHECK_INCLUDE_FILE_CXX (wchar.h HAVE_WCHAR_H) -CHECK_INCLUDE_FILE_CXX (wctype.h HAVE_WCTYPE_H) -CHECK_INCLUDE_FILE_CXX (zlib.h HAVE_ZLIB_H) +check_include_file_cxx(alloca.h HAVE_ALLOCA_H) +check_include_file_cxx(arpa/inet.h HAVE_ARPA_INET_H) +check_include_file_cxx(dlfcn.h HAVE_DLFCN_H) +check_include_file_cxx(fcntl.h HAVE_FCNTL_H) +check_include_file_cxx(inttypes.h HAVE_INTTYPES_H) +check_include_file_cxx(limits.h HAVE_LIMITS_H) +check_include_file_cxx(malloc.h HAVE_MALLOC_H) +check_include_file_cxx(memory.h HAVE_MEMORY_H) +check_include_file_cxx(ncurses.h HAVE_NCURSES_H) +check_include_file_cxx(netdb.h HAVE_NETDB_H) +check_include_file_cxx(netinet/in.h HAVE_NETINET_IN_H) +check_include_file_cxx(stddef.h HAVE_STDDEF_H) +check_include_file_cxx(stdint.h HAVE_STDINT_H) +check_include_file_cxx(stdlib.h HAVE_STDLIB_H) +check_include_file_cxx(strings.h HAVE_STRINGS_H) +check_include_file_cxx(string.h HAVE_STRING_H) +check_include_file_cxx(syslog.h HAVE_SYSLOG_H) +check_include_file_cxx(sys/file.h HAVE_SYS_FILE_H) +check_include_file_cxx(sys/mount.h HAVE_SYS_MOUNT_H) +check_include_file_cxx(sys/select.h HAVE_SYS_SELECT_H) +check_include_file_cxx(sys/socket.h HAVE_SYS_SOCKET_H) +check_include_file_cxx(sys/statfs.h HAVE_SYS_STATFS_H) +check_include_file_cxx(sys/stat.h HAVE_SYS_STAT_H) +check_include_file_cxx(sys/timeb.h HAVE_SYS_TIMEB_H) +check_include_file_cxx(sys/time.h HAVE_SYS_TIME_H) +check_include_file_cxx(sys/types.h HAVE_SYS_TYPES_H) +check_include_file_cxx(sys/wait.h HAVE_SYS_WAIT_H) +check_include_file_cxx(unistd.h HAVE_UNISTD_H) +check_include_file_cxx(utime.h HAVE_UTIME_H) +check_include_file_cxx(values.h HAVE_VALUES_H) +check_include_file_cxx(vfork.h HAVE_VFORK_H) +check_include_file_cxx(wchar.h HAVE_WCHAR_H) +check_include_file_cxx(wctype.h HAVE_WCTYPE_H) +check_include_file_cxx(zlib.h HAVE_ZLIB_H) -CHECK_FUNCTION_EXISTS (_getb67 GETB1) -CHECK_FUNCTION_EXISTS (GETB67 GETB2) -CHECK_FUNCTION_EXISTS (getb67 GETB3) +check_function_exists(_getb67 GETB1) +check_function_exists(GETB67 GETB2) +check_function_exists(getb67 GETB3) -IF(GETB1) - SET (CRAY_STACKSEG_END 1) -ELSEIF(GETB2) - SET (CRAY_STACKSEG_END 1) -ELSEIF(GETB3) - SET (CRAY_STACKSEG_END 1) -ENDIF() +if(GETB1) + set(CRAY_STACKSEG_END 1) +elseif(GETB2) + set(CRAY_STACKSEG_END 1) +elseif(GETB3) + set(CRAY_STACKSEG_END 1) +endif() -CHECK_FUNCTION_EXISTS (alarm HAVE_ALARM) -CHECK_FUNCTION_EXISTS (btowc HAVE_BTOWC) -CHECK_FUNCTION_EXISTS (dup2 HAVE_DUP2) -CHECK_FUNCTION_EXISTS (error_at_line HAVE_ERROR_AT_LINE) -CHECK_FUNCTION_EXISTS (floor HAVE_FLOOR) -CHECK_FUNCTION_EXISTS (fork HAVE_FORK) -CHECK_FUNCTION_EXISTS (ftime HAVE_FTIME) -CHECK_FUNCTION_EXISTS (ftruncate HAVE_FTRUNCATE) -CHECK_FUNCTION_EXISTS (getenv HAVE_DECL_GETENV) -CHECK_FUNCTION_EXISTS (gethostbyname HAVE_GETHOSTBYNAME) -CHECK_FUNCTION_EXISTS (getpagesize HAVE_GETPAGESIZE) -CHECK_FUNCTION_EXISTS (gettimeofday HAVE_GETTIMEOFDAY) -CHECK_FUNCTION_EXISTS (inet_ntoa HAVE_INET_NTOA) -CHECK_FUNCTION_EXISTS (isascii HAVE_ISASCII) -CHECK_FUNCTION_EXISTS (localtime_r HAVE_LOCALTIME_R) -CHECK_FUNCTION_EXISTS (malloc HAVE_MALLOC) -CHECK_FUNCTION_EXISTS (mbsrtowcs HAVE_MBSRTOWCS) -CHECK_FUNCTION_EXISTS (memchr HAVE_MEMCHR) -CHECK_FUNCTION_EXISTS (memmove HAVE_MEMMOVE) -CHECK_FUNCTION_EXISTS (mempcpy HAVE_MEMPCPY) -CHECK_FUNCTION_EXISTS (memset HAVE_MEMSET) -CHECK_FUNCTION_EXISTS (mkdir HAVE_MKDIR) -CHECK_FUNCTION_EXISTS (mktime HAVE_MKTIME) -CHECK_FUNCTION_EXISTS (pow HAVE_POW) -CHECK_FUNCTION_EXISTS (regcomp HAVE_REGCOMP) -CHECK_FUNCTION_EXISTS (rmdir HAVE_RMDIR) -CHECK_FUNCTION_EXISTS (select HAVE_SELECT) -CHECK_FUNCTION_EXISTS (setenv HAVE_SETENV) -CHECK_FUNCTION_EXISTS (setlocale HAVE_SETLOCALE) -CHECK_FUNCTION_EXISTS (socket HAVE_SOCKET) -CHECK_FUNCTION_EXISTS (stat HAVE_STAT) -CHECK_FUNCTION_EXISTS (strcasecmp HAVE_STRCASECMP) -CHECK_FUNCTION_EXISTS (strchr HAVE_STRCHR) -CHECK_FUNCTION_EXISTS (strcspn HAVE_STRCSPN) -CHECK_FUNCTION_EXISTS (strdup HAVE_STRDUP) -CHECK_FUNCTION_EXISTS (strerror HAVE_STRERROR) -CHECK_FUNCTION_EXISTS (strerror_r HAVE_STRERROR_R) -CHECK_FUNCTION_EXISTS (strftime HAVE_STRFTIME) -CHECK_FUNCTION_EXISTS (strrchr HAVE_STRRCHR) -CHECK_FUNCTION_EXISTS (strspn HAVE_STRSPN) -CHECK_FUNCTION_EXISTS (strstr HAVE_STRSTR) -CHECK_FUNCTION_EXISTS (strtod HAVE_STRTOD) -CHECK_FUNCTION_EXISTS (strtol HAVE_STRTOL) -CHECK_FUNCTION_EXISTS (strtoul HAVE_STRTOUL) -CHECK_FUNCTION_EXISTS (strtoull HAVE_STRTOULL) -CHECK_FUNCTION_EXISTS (utime HAVE_UTIME) -CHECK_FUNCTION_EXISTS (vfork HAVE_VFORK) -CHECK_FUNCTION_EXISTS (wmempcpy HAVE_WMEMPCPY) +check_function_exists(alarm HAVE_ALARM) +check_function_exists(btowc HAVE_BTOWC) +check_function_exists(dup2 HAVE_DUP2) +check_function_exists(error_at_line HAVE_ERROR_AT_LINE) +check_function_exists(floor HAVE_FLOOR) +check_function_exists(fork HAVE_FORK) +check_function_exists(ftime HAVE_FTIME) +check_function_exists(ftruncate HAVE_FTRUNCATE) +check_function_exists(getenv HAVE_DECL_GETENV) +check_function_exists(gethostbyname HAVE_GETHOSTBYNAME) +check_function_exists(getpagesize HAVE_GETPAGESIZE) +check_function_exists(gettimeofday HAVE_GETTIMEOFDAY) +check_function_exists(inet_ntoa HAVE_INET_NTOA) +check_function_exists(isascii HAVE_ISASCII) +check_function_exists(localtime_r HAVE_LOCALTIME_R) +check_function_exists(malloc HAVE_MALLOC) +check_function_exists(mbsrtowcs HAVE_MBSRTOWCS) +check_function_exists(memchr HAVE_MEMCHR) +check_function_exists(memmove HAVE_MEMMOVE) +check_function_exists(mempcpy HAVE_MEMPCPY) +check_function_exists(memset HAVE_MEMSET) +check_function_exists(mkdir HAVE_MKDIR) +check_function_exists(mktime HAVE_MKTIME) +check_function_exists(pow HAVE_POW) +check_function_exists(regcomp HAVE_REGCOMP) +check_function_exists(rmdir HAVE_RMDIR) +check_function_exists(select HAVE_SELECT) +check_function_exists(setenv HAVE_SETENV) +check_function_exists(setlocale HAVE_SETLOCALE) +check_function_exists(socket HAVE_SOCKET) +check_function_exists(stat HAVE_STAT) +check_function_exists(strcasecmp HAVE_STRCASECMP) +check_function_exists(strchr HAVE_STRCHR) +check_function_exists(strcspn HAVE_STRCSPN) +check_function_exists(strdup HAVE_STRDUP) +check_function_exists(strerror HAVE_STRERROR) +check_function_exists(strerror_r HAVE_STRERROR_R) +check_function_exists(strftime HAVE_STRFTIME) +check_function_exists(strrchr HAVE_STRRCHR) +check_function_exists(strspn HAVE_STRSPN) +check_function_exists(strstr HAVE_STRSTR) +check_function_exists(strtod HAVE_STRTOD) +check_function_exists(strtol HAVE_STRTOL) +check_function_exists(strtoul HAVE_STRTOUL) +check_function_exists(strtoull HAVE_STRTOULL) +check_function_exists(utime HAVE_UTIME) +check_function_exists(vfork HAVE_VFORK) +check_function_exists(wmempcpy HAVE_WMEMPCPY) -CHECK_CXX_SYMBOL_EXISTS (alloca alloca.h HAVE_ALLOCA) -CHECK_CXX_SYMBOL_EXISTS (strerror_r string.h HAVE_DECL_STRERROR_R) -CHECK_CXX_SYMBOL_EXISTS (tm sys/time.h TM_IN_SYS_TIME) +check_cxx_symbol_exists(alloca alloca.h HAVE_ALLOCA) +check_cxx_symbol_exists(strerror_r string.h HAVE_DECL_STRERROR_R) +check_cxx_symbol_exists(tm sys/time.h TM_IN_SYS_TIME) -#AC_TYPE_SIGNAL -CHECK_TYPE_SIZE (ptrdiff_t PTRDIFF_T) -CHECK_TYPE_SIZE (_Bool __BOOL) -CHECK_TYPE_SIZE (mode_t mode_t_test) -IF(NOT HAVE_mode_t_test) -SET (mode_t int) -ENDIF() -CHECK_TYPE_SIZE(off_t off_t_test) -IF(NOT HAVE_off_t_test) -SET (off_t long int) -ENDIF() -CHECK_TYPE_SIZE(pid_t pid_t_test) -IF(NOT HAVE_pid_t_test) -SET (pid_t int) -ENDIF() -CHECK_TYPE_SIZE(size_t size_t_test) -IF(NOT HAVE_size_t_test) -SET (size_t unsigned int) -ENDIF() +# AC_TYPE_SIGNAL +check_type_size(ptrdiff_t PTRDIFF_T) +check_type_size(_Bool __BOOL) +check_type_size(mode_t mode_t_test) +if(NOT HAVE_mode_t_test) + set(mode_t int) +endif() +check_type_size(off_t off_t_test) +if(NOT HAVE_off_t_test) + set(off_t long int) +endif() +check_type_size(pid_t pid_t_test) +if(NOT HAVE_pid_t_test) + set(pid_t int) +endif() +check_type_size(size_t size_t_test) +if(NOT HAVE_size_t_test) + set(size_t unsigned int) +endif() - - -CHECK_CXX_SOURCE_COMPILES( -"#include +check_cxx_source_compiles( + "#include #include #include #include @@ -152,71 +148,72 @@ main () ; return 0; -}" STDC_HEADERS) +}" + STDC_HEADERS +) +set(TEST_INCLUDES " +#include " +) -SET (TEST_INCLUDES -" -#include ") - -IF (HAVE_SYS_TYPES_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_SYS_STAT_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (STDC_HEADERS) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include - # include ") -ELSE() - IF() - SET ( TEST_INCLUDES +if(HAVE_SYS_TYPES_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_SYS_STAT_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(STDC_HEADERS) + set(TEST_INCLUDES "${TEST_INCLUDES} - # include ") - ENDIF() -ENDIF() -IF (HAVE_STRING_H) - IF(NOT STDC_HEADERS) - IF (HAVE_MEMORY_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") - ENDIF() - ENDIF() - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_STRINGS_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_INTTYPES_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_STDINT_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_UNISTD_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() + # include + # include " + ) +else() + if() + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) + endif() +endif() +if(HAVE_STRING_H) + if(NOT STDC_HEADERS) + if(HAVE_MEMORY_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) + endif() + endif() + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_STRINGS_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_INTTYPES_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_STDINT_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_UNISTD_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() - -CHECK_CXX_SOURCE_COMPILES( -" +check_cxx_source_compiles( + " ${TEST_INCLUDES} # include int @@ -225,11 +222,12 @@ main () mbstate_t x; return sizeof x; ; return 0; -}" HAVE_MBSTATE_T) +}" + HAVE_MBSTATE_T +) - -CHECK_CXX_SOURCE_RUNS( -" +check_cxx_source_runs( + " ${TEST_INCLUDES} int main () @@ -238,14 +236,15 @@ struct stat sbuf; return stat (\"\", &sbuf) == 0; ; return 0; -}" STAT_EMPTY_STRING_BUG) -IF (NOT STAT_EMPTY_STRING_BUG) -SET (HAVE_STAT_EMPTY_STRING_BUG 1) -ENDIF() +}" + STAT_EMPTY_STRING_BUG +) +if(NOT STAT_EMPTY_STRING_BUG) + set(HAVE_STAT_EMPTY_STRING_BUG 1) +endif() - -CHECK_CXX_SOURCE_COMPILES( - " +check_cxx_source_compiles( + " ${TEST_INCLUDES} #include #ifndef bool @@ -304,28 +303,32 @@ main () ; return 0; -}" HAVE_STDBOOL_H) +}" + HAVE_STDBOOL_H +) -IF (HAVE_UTIME_H) -CHECK_CXX_SOURCE_COMPILES( -"${TEST_INCLUDES} +if(HAVE_UTIME_H) + check_cxx_source_compiles( + "${TEST_INCLUDES} # include int main () { struct stat s, t; - return ! (stat (\"conftest.data\", &s) == 0 - && utime (\"conftest.data\", 0) == 0 - && stat (\"conftest.data\", &t) == 0 + return ! (stat (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.data\", &s) == 0 + && utime (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.data\", 0) == 0 + && stat (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.data\", &t) == 0 && t.st_mtime >= s.st_mtime && t.st_mtime - s.st_mtime < 120); ; return 0; -}" HAVE_UTIME_NULL) -ENDIF() +}" + HAVE_UTIME_NULL + ) +endif() -CHECK_CXX_SOURCE_COMPILES( -" +check_cxx_source_compiles( + " ${TEST_INCLUDES} int main () @@ -334,10 +337,12 @@ main () return fork () < 0; ; return 0; -}" HAVE_WORKING_FORK) +}" + HAVE_WORKING_FORK +) -CHECK_CXX_SOURCE_COMPILES( -"${TEST_INCLUDES} +check_cxx_source_compiles( + "${TEST_INCLUDES} #include #ifdef HAVE_VFORK_H include @@ -424,14 +429,16 @@ main () || fstat(fileno(stdout), &st) != 0 ); } -}" HAVE_WORKING_VFORK) +}" + HAVE_WORKING_VFORK +) -IF (NOT HAVE_WORKING_VFORK) -SET (VFORK fork) -ENDIF() +if(NOT HAVE_WORKING_VFORK) + set(VFORK fork) +endif() -CHECK_CXX_SOURCE_COMPILES( -" +check_cxx_source_compiles( + " #include #include @@ -441,22 +448,24 @@ main () return *(signal (0, 0)) (0) == 1; ; return 0; -}" RET_SIGNAL_TYPES) -IF (RET_SIGNAL_TYPES) -SET (RETSIGTYPE int) -ELSE() -SET (RETSIGTYPE void) -ENDIF() +}" + RET_SIGNAL_TYPES +) +if(RET_SIGNAL_TYPES) + set(RETSIGTYPE int) +else() + set(RETSIGTYPE void) +endif() -#IF(NOT LSTAT_FOLLOWS_SLASHED_SYMLINK) -EXECUTE_PROCESS( +# IF(NOT LSTAT_FOLLOWS_SLASHED_SYMLINK) +execute_process( COMMAND rm -f conftest.sym conftest.file COMMAND touch conftest.file COMMAND ln -s conftest.file conftest.sym - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) -CHECK_CXX_SOURCE_RUNS( -" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) +check_cxx_source_runs( + " ${TEST_INCLUDES} int main () @@ -465,31 +474,30 @@ struct stat sbuf; /* Linux will dereference the symlink and fail, as required by POSIX. That is better in the sense that it means we will not have to compile and use the lstat wrapper. */ - return lstat (\"conftest.sym/\", &sbuf) == 0; + return lstat (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.sym/\", &sbuf) == 0; ; return 0; -}" LSTAT_FOLLOWS_SLASHED_SYMLINK) +}" + LSTAT_FOLLOWS_SLASHED_SYMLINK +) +set(SELECT_INCLUDES ${TEST_INCLUDES}) +if(HAVE_SYS_SELECT_H) + set(SELECT_INCULDES "${SELECT_INCLUDES} +# include " + ) +endif() +if(HAVE_SYS_SOCKET_H) + set(SELECT_INCULDES "${SELECT_INCLUDES} +# include " + ) +endif() - -SET (SELECT_INCLUDES ${TEST_INCLUDES}) -IF (HAVE_SYS_SELECT_H) -SET (SELECT_INCULDES -"${SELECT_INCLUDES} -# include ") -ENDIF() -IF (HAVE_SYS_SOCKET_H) -SET (SELECT_INCULDES -"${SELECT_INCLUDES} -# include ") -ENDIF() - - -FOREACH( ARG234 "fd_set *" "int *" "void *") -FOREACH( ARG1 "int" "size_t" "unsigned long int" "unsigned int") -FOREACH( ARG5 "struct timeval *" "const struct timeval *") -CHECK_CXX_SOURCE_COMPILES( -" +foreach(ARG234 "fd_set *" "int *" "void *") + foreach(ARG1 "int" "size_t" "unsigned long int" "unsigned int") + foreach(ARG5 "struct timeval *" "const struct timeval *") + check_cxx_source_compiles( + " ${SELECT_INCLUDES} #ifdef HAVE_SYS_SELECT_H # include @@ -507,26 +515,27 @@ extern int select (${ARG1}, ; return 0; } -" SELECT_ARGS) -IF(SELECT_ARGS) -SET (SELECT_TYPE_ARG1 ${ARG1}) -SET (SELECT_TYPE_ARG234 ${ARG234}) -SET (SELECT_TYPE_ARG5 ${ARG5}) -BREAK() -ENDIF() -ENDFOREACH() -IF(SELECT_ARGS) -BREAK() -ENDIF() -ENDFOREACH() -IF(SELECT_ARGS) -BREAK() -ENDIF() -ENDFOREACH() - - -CHECK_CXX_SOURCE_COMPILES( " + SELECT_ARGS + ) + if(SELECT_ARGS) + set(SELECT_TYPE_ARG1 ${ARG1}) + set(SELECT_TYPE_ARG234 ${ARG234}) + set(SELECT_TYPE_ARG5 ${ARG5}) + break() + endif() + endforeach() + if(SELECT_ARGS) + break() + endif() + endforeach() + if(SELECT_ARGS) + break() + endif() +endforeach() + +check_cxx_source_compiles( + " #include #include @@ -549,13 +558,15 @@ int main() { return 0; } -" STATS_MACROS_CHECK) -IF (NOT STATS_MACROS_CHECK) -SET (STAT_MACROS_BROKEN 1) -ENDIF() - -CHECK_CXX_SOURCE_COMPILES( " + STATS_MACROS_CHECK +) +if(NOT STATS_MACROS_CHECK) + set(STAT_MACROS_BROKEN 1) +endif() + +check_cxx_source_compiles( + " ${TEST_INCLUDES} int main () @@ -569,10 +580,12 @@ main () ; return 0; } -" STRERROR_R_CHAR_P) - -CHECK_CXX_SOURCE_COMPILES( " + STRERROR_R_CHAR_P +) + +check_cxx_source_compiles( + " #include #include #include @@ -585,11 +598,12 @@ return 0; ; return 0; } -" TIME_WITH_SYS_TIME) - - -CHECK_CXX_SOURCE_COMPILES( " + TIME_WITH_SYS_TIME +) + +check_cxx_source_compiles( + " int main () { @@ -647,13 +661,15 @@ main () ; return 0; } -" CONST_CONFORM_CHECK) -IF (NOT CONST_CONFORM_CHECK) -SET (const "") -ENDIF() - -CHECK_CXX_SOURCE_COMPILES( " + CONST_CONFORM_CHECK +) +if(NOT CONST_CONFORM_CHECK) + set(const "") +endif() + +check_cxx_source_compiles( + " int main () { @@ -664,15 +680,17 @@ return !x && !y; ; return 0; } -" WORKING_VOLATILE) -IF (NOT WORKING_VOLATILE) -SET (volatile "") -ENDIF() - -FOREACH (RESTRICT_KW __restrict __restrict__ _Restrict restrict) - -CHECK_CXX_SOURCE_COMPILES( " + WORKING_VOLATILE +) +if(NOT WORKING_VOLATILE) + set(volatile "") +endif() + +foreach(RESTRICT_KW __restrict __restrict__ _Restrict restrict) + + check_cxx_source_compiles( + " typedef int * int_ptr; int foo (int_ptr ${RESTRICT_KW} ip) { return ip[0]; @@ -687,53 +705,78 @@ int s[1]; ; return 0; } -" RESTRICT_CHECK) -IF (RESTRICT_CHECK) -SET (restrict ${RESTRICT_KW}) -BREAK() -ENDIF() -ENDFOREACH() - - -FOREACH(INLINE_KW inline __inline__ __inline) -CHECK_CXX_SOURCE_COMPILES( " + RESTRICT_CHECK + ) + if(RESTRICT_CHECK) + set(restrict ${RESTRICT_KW}) + break() + endif() +endforeach() + +foreach(INLINE_KW inline __inline__ __inline) + check_cxx_source_compiles( + " #ifndef __cplusplus typedef int foo_t; static ${INLINE_KW} foo_t static_foo () {return 0; } ${INLINE_KW} foo_t foo () {return 0; } int main (){return 0;} #endif -" INLINE) - -IF (INLINE) -SET (inline ${INLINE_KW}) -BREAK() -ENDIF() -ENDFOREACH() - -IF (NOT INLINE) -SET (inline "") -ENDIF() - -EXECUTE_PROCESS( - COMMAND rm -f conftest.data conftest.file conftest.sym - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +" + INLINE ) - -CHECK_CXX_SOURCE_RUNS(" + + if(INLINE) + set(inline ${INLINE_KW}) + break() + endif() +endforeach() + +if(NOT INLINE) + set(inline "") +endif() + +execute_process(COMMAND rm -f conftest.data conftest.file conftest.sym WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +check_cxx_source_runs( + " #include int main() { - // If long double is 16 bytes and digits and exponent are 64 and 16384 respectively, then we need to mask out the + // If long double is 16 bytes and digits and exponent are 64 and 16384 respectively, then we need to mask out the // unused bits, as they contain garbage. There are times we test for equality by memcmp of a buffer containing, // in part, the long double set here. Garbage bytes will adversly affect that compare. // Note: There may be compilers that store 80 bit floats in 12 bytes. We do not account for that here. I don't believe // there are any modern Linux compilers that do that as a default. Windows uses 64 bits, so no masking is needed. - if (std::numeric_limits::digits == 64 - && std::numeric_limits::max_exponent == 16384 + if (std::numeric_limits::digits == 64 + && std::numeric_limits::max_exponent == 16384 && sizeof(long double) == 16) return 0; - return 1; + return 1; }" -MASK_LONGDOUBLE) + MASK_LONGDOUBLE +) + +find_package(Git QUIET) + +if(GIT_FOUND AND EXISTS ${ENGINE_SRC_DIR}/.git) + execute_process( + COMMAND git describe --match=NeVeRmAtCh --always --dirty + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE GIT_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +else() + set(GIT_VERSION "source") +endif() + +# releasenum is used by external scripts for various tasks. Leave it alone. +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) +columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/build/releasenum ${ENGINE_SUPPORTDIR}) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) + +columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine ${ENGINE_SUPPORTDIR}) diff --git a/cmake/dirs.cmake b/cmake/dirs.cmake new file mode 100644 index 000000000..6cd876736 --- /dev/null +++ b/cmake/dirs.cmake @@ -0,0 +1,20 @@ +set(ENGINE_SYSCONFDIR "/etc") +set(ENGINE_DATADIR "/var/lib/columnstore") +set(ENGINE_LOGDIR "/var/log/mariadb/columnstore") +set(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR}") +set(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR}") +set(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR}") +set(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") +set(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR}") +set(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR}") +set(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") +set(ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +if(INSTALL_SYSCONF2DIR) + set(MARIADB_MYCNFDIR "${INSTALL_SYSCONF2DIR}") +else() + set(MARIADB_MYCNFDIR "/etc/mysql") +endif() + +set(SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR}) +set(SERVER_BUILD_DIR ${CMAKE_BINARY_DIR}) diff --git a/cmake/includes.cmake b/cmake/includes.cmake new file mode 100644 index 000000000..0d3b593b3 --- /dev/null +++ b/cmake/includes.cmake @@ -0,0 +1,108 @@ +message_once(SERVER_BUILD_INCLUDE_DIR "SERVER_BUILD_INCLUDE_DIR = ${SERVER_BUILD_INCLUDE_DIR}") +message_once(SERVER_SOURCE_ROOT_DIR "SERVER_SOURCE_ROOT_DIR = ${SERVER_SOURCE_ROOT_DIR}") + +set(ENGINE_UTILS_MESSAGEQCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/messageqcpp") +set(ENGINE_WE_SHARED_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/shared") +set(ENGINE_UTILS_IDBDATAFILE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/idbdatafile") +set(ENGINE_UTILS_LOGGINGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/loggingcpp" + "${CMAKE_CURRENT_BINARY_DIR}/utils/loggingcpp" +) +set(ENGINE_UTILS_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/configcpp") +set(ENGINE_UTILS_COMPRESS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/compress") +set(ENGINE_VERSIONING_BRM_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/versioning/BRM") +set(ENGINE_UTILS_ROWGROUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rowgroup") +set(ENGINE_UTILS_COMMON_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/common") +set(ENGINE_UTILS_DATACONVERT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/dataconvert") +set(ENGINE_UTILS_RWLOCK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rwlock") +set(ENGINE_UTILS_FUNCEXP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/funcexp") +set(ENGINE_UTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils") +set(ENGINE_OAM_OAMCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/oam/oamcpp") +set(ENGINE_DBCON_DDLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackageproc") +set(ENGINE_DBCON_DDLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackage") +set(ENGINE_DBCON_EXECPLAN_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/execplan") +set(ENGINE_UTILS_STARTUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/startup") +set(ENGINE_DBCON_JOBLIST_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/joblist") +set(ENGINE_WE_WRAPPER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/wrapper") +set(ENGINE_WE_SERVER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/server") +set(ENGINE_DBCON_DMLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackage") +set(ENGINE_WE_CLIENT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/client") +set(ENGINE_DBCON_DMLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackageproc") +set(ENGINE_UTILS_CACHEUTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/cacheutils") +set(ENGINE_UTILS_MYSQLCL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/mysqlcl_idb") +set(ENGINE_UTILS_QUERYTELE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querytele") +set(ENGINE_UTILS_JOINER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/joiner") +set(ENGINE_UTILS_THREADPOOL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/threadpool") +set(ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchloader") +set(ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup") +set(ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") +set(ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") +set(ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") +set(ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") +set(ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache") +set(ENGINE_PRIMPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/primproc") +set(ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") +set(ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") +if(PCRE_INCLUDE_DIRS) + set(ENGINE_SERVER_PCRE_INCLUDE "${PCRE_INCLUDE_DIRS}") +else() + set(ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre") +endif() +set(ENGINE_SERVER_WSREP_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/include") +set(ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/wsrep-API/v26/") +set(ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk") + +set(ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) + +set(ENGINE_COMMON_INCLUDES + ${ENGINE_DEFAULT_INCLUDES} + ${Boost_INCLUDE_DIRS} + ${LIBXML2_INCLUDE_DIR} + ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} + ${ENGINE_WE_SHARED_INCLUDE} + ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} + ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} + ${ENGINE_UTILS_CONFIGCPP_INCLUDE} + ${ENGINE_UTILS_COMPRESS_INCLUDE} + ${ENGINE_VERSIONING_BRM_INCLUDE} + ${ENGINE_UTILS_ROWGROUP_INCLUDE} + ${ENGINE_UTILS_COMMON_INCLUDE} + ${ENGINE_UTILS_DATACONVERT_INCLUDE} + ${ENGINE_UTILS_RWLOCK_INCLUDE} + ${ENGINE_UTILS_FUNCEXP_INCLUDE} + ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} + ${ENGINE_UTILS_INCLUDE} + ${ENGINE_OAM_OAMCPP_INCLUDE} + ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} + ${ENGINE_DBCON_DDLPKG_INCLUDE} + ${ENGINE_DBCON_EXECPLAN_INCLUDE} + ${ENGINE_UTILS_STARTUP_INCLUDE} + ${ENGINE_DBCON_JOBLIST_INCLUDE} + ${ENGINE_WE_WRAPPER_INCLUDE} + ${ENGINE_WE_SERVER_INCLUDE} + ${ENGINE_DBCON_DMLPKG_INCLUDE} + ${ENGINE_WE_CLIENT_INCLUDE} + ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} + ${ENGINE_UTILS_CACHEUTILS_INCLUDE} + ${ENGINE_UTILS_MYSQLCL_INCLUDE} + ${ENGINE_UTILS_QUERYTELE_INCLUDE} + ${ENGINE_UTILS_THRIFT_INCLUDE} + ${ENGINE_UTILS_JOINER_INCLUDE} + ${ENGINE_UTILS_THREADPOOL_INCLUDE} + ${ENGINE_UTILS_BATCHLDR_INCLUDE} + ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} + ${ENGINE_UTILS_QUERYSTATS_INCLUDE} + ${ENGINE_WE_CONFIGCPP_INCLUDE} + ${ENGINE_SERVER_SQL_INCLUDE} + ${ENGINE_SERVER_INCLUDE_INCLUDE} + ${ENGINE_SERVER_PCRE_INCLUDE} + ${ENGINE_SERVER_WSREP_API_INCLUDE} + ${ENGINE_SERVER_WSREP_INCLUDE} + ${ENGINE_UTILS_UDFSDK_INCLUDE} + ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} + ${ENGINE_DATATYPES_INCLUDE} +) + +set(LIBMARIADB_BININC_DIR ${CMAKE_BINARY_DIR}/libmariadb/include) +set(LIBMARIADB_SRCINC_DIR ${CMAKE_SOURCE_DIR}/libmariadb/include) +set(SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include) +set(SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR}) diff --git a/cmake/libs.cmake b/cmake/libs.cmake new file mode 100644 index 000000000..687cac600 --- /dev/null +++ b/cmake/libs.cmake @@ -0,0 +1,44 @@ +set(ENGINE_DT_LIB datatypes) +set(ENGINE_COMMON_LIBS + boost_thread + configcpp + idbboot + loggingcpp + messageqcpp + pthread + rt + xml2 + ${ENGINE_DT_LIB} +) +set(ENGINE_OAM_LIBS oamcpp) +set(ENGINE_BRM_LIBS brm cacheutils idbdatafile rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) + +set(PLUGIN_EXEC_LIBS + common + compress + dataconvert + execplan + funcexp + joiner + querytele + regr + rowgroup + threadpool + udfsdk + windowfunction + ${ENGINE_BRM_LIBS} +) +set(ENGINE_EXEC_LIBS joblist querystats libmysql_client ${PLUGIN_EXEC_LIBS}) +set(PLUGIN_WRITE_LIBS + cacheutils + ddlpackage + ddlpackageproc + dmlpackage + dmlpackageproc + idbdatafile + writeengine + writeengineclient +) +set(ENGINE_WRITE_LIBS ${PLUGIN_WRITE_LIBS} ${ENGINE_EXEC_LIBS}) +set(MARIADB_CLIENT_LIBS libmariadb) +set(MARIADB_STRING_LIBS dbug strings mysys) diff --git a/cmake/packages.cmake b/cmake/packages.cmake new file mode 100644 index 000000000..3fee24098 --- /dev/null +++ b/cmake/packages.cmake @@ -0,0 +1,64 @@ +find_package(BISON) +if(NOT BISON_FOUND) + message_once(CS_NO_BISON "bison not found!") + return() +endif() + +find_program(LEX_EXECUTABLE flex DOC "path to the flex executable") +if(NOT LEX_EXECUTABLE) + find_program(LEX_EXECUTABLE lex DOC "path to the lex executable") + if(NOT LEX_EXECUTABLE) + message_once(CS_NO_LEX "flex/lex not found!") + return() + endif() +endif() + +find_package(LibXml2) +if(NOT LIBXML2_FOUND) + message_once(CS_NO_LIBXML "Could not find a usable libxml2 development environment!") + return() +endif() + +find_package(Snappy) +if(NOT SNAPPY_FOUND) + message_once( + CS_NO_SNAPPY + "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian" + ) + return() +endif() + +find_package(CURL) +if(NOT CURL_FOUND) + message_once(CS_NO_CURL "libcurl development headers not found") + return() +endif() + +find_program(AWK_EXECUTABLE awk DOC "path to the awk executable") +if(NOT AWK_EXECUTABLE) + message_once(CS_NO_AWK "awk not found!") + return() +endif() + +set(HAVE_LZ4 + 0 + CACHE INTERNAL "" +) +if(WITH_COLUMNSTORE_LZ4 STREQUAL "ON" OR WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") + find_package(LZ4) + if(NOT LZ4_FOUND) + if(WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") + message_once(CS_LZ4 "LZ4 not found, building without LZ4") + else() + message(FATAL_ERROR "LZ4 not found.") + endif() + else() + message_once(CS_LZ4 "Building with LZ4") + set(HAVE_LZ4 + 1 + CACHE INTERNAL "" + ) + endif() +else() + message_once(CS_LZ4 "Building without LZ4") +endif() diff --git a/cmake/superbuild.md b/cmake/superbuild.md index 6bedaeabc..0a6fde43f 100644 --- a/cmake/superbuild.md +++ b/cmake/superbuild.md @@ -1,24 +1,24 @@ -CMake Super Build -================= -A super build is a process to download, build and install dependencies with cmake at configure time. This ensure dependencies are available during the initial configure stage. Its accomplished by executing separate cmake configure and build processes inline with the main project cmake which builds and installs the missing dependency. - -Rationale: ----------- -It maybe observed that ExternalProject accomplishes a similar task, however, the target of an ExternalProject is not available until after the build stage. Any scripting logic which requires the dependency during the configure stage will fail. The super build solves this by ensuring the dependency is built independent of the main projects configuration which uses it. - -Example: --------- -# In the context of the main projects cmake scripts, subshells of cmake are executed to configure and build the dependency -configure_file(some_dependency.CMakeLists.txt.in some_dep_dir\CMakeLists.txt @ONLY) # drop a top-level CMakeLists.txt in a folder for the dependency -execute_process(COMMAND ${CMAKE_COMMAND} . WORKING_DIRECTORY some_dep_dir) # execute configure stage of dependency against newly created CMakeLists.txt from above step -execute_process(COMMAND ${CMAKE_COMMAND} --build . WORKING_DIRECTORY some_dep_dir) # install the dependency -find_package(some_dependency) # the dependency should be installed and can be 'found' or used as appropriate - -NOTES ------ - o The bulk of the work is performed in the generated/copied CMakeLists.txt to download (optional), build and install the dependency. It typically contains the full set of ExternalProject statements and error handling. - o CMake scripts executed in a sub-process with execute_process are independent and share no state whatsoever with the calling process. There are two ways to share state with the sub-shell - - Wrap appropriate @VARIABLE@ decorations in the CMakeLists.in template which get substituted with values when configure_file is executed - - Pass them on the command line of the execute_process statement. e.g.: execute_process(COMMAND ${CMAKE_COMMAND} -DSOME_VAR=${SOME_VAL} -DANOTHER_VAR=${ANOTHER_VAL} ... - +CMake Super Build +================= +A super build is a process to download, build and install dependencies with cmake at configure time. This ensure dependencies are available during the initial configure stage. Its accomplished by executing separate cmake configure and build processes inline with the main project cmake which builds and installs the missing dependency. + +Rationale: +---------- +It maybe observed that ExternalProject accomplishes a similar task, however, the target of an ExternalProject is not available until after the build stage. Any scripting logic which requires the dependency during the configure stage will fail. The super build solves this by ensuring the dependency is built independent of the main projects configuration which uses it. + +Example: +-------- +# In the context of the main projects cmake scripts, subshells of cmake are executed to configure and build the dependency +configure_file(some_dependency.CMakeLists.txt.in some_dep_dir\CMakeLists.txt @ONLY) # drop a top-level CMakeLists.txt in a folder for the dependency +execute_process(COMMAND ${CMAKE_COMMAND} . WORKING_DIRECTORY some_dep_dir) # execute configure stage of dependency against newly created CMakeLists.txt from above step +execute_process(COMMAND ${CMAKE_COMMAND} --build . WORKING_DIRECTORY some_dep_dir) # install the dependency +find_package(some_dependency) # the dependency should be installed and can be 'found' or used as appropriate + +NOTES +----- + o The bulk of the work is performed in the generated/copied CMakeLists.txt to download (optional), build and install the dependency. It typically contains the full set of ExternalProject statements and error handling. + o CMake scripts executed in a sub-process with execute_process are independent and share no state whatsoever with the calling process. There are two ways to share state with the sub-shell + - Wrap appropriate @VARIABLE@ decorations in the CMakeLists.in template which get substituted with values when configure_file is executed + - Pass them on the command line of the execute_process statement. e.g.: execute_process(COMMAND ${CMAKE_COMMAND} -DSOME_VAR=${SOME_VAL} -DANOTHER_VAR=${ANOTHER_VAL} ... + x \ No newline at end of file diff --git a/cmake/thrift.cmake b/cmake/thrift.cmake index 3cce8ddbf..11e1533d6 100644 --- a/cmake/thrift.cmake +++ b/cmake/thrift.cmake @@ -1,29 +1,46 @@ +set(WITH_THRIFT + "bundled" + CACHE STRING "Which Thrift to use (possible values are 'bundled', 'system', or 'auto')" +) + +if(WITH_THRIFT STREQUAL "system" OR WITH_THRIFT STREQUAL "auto") + find_package(Thrift) + + if(Thrift_FOUND) + add_custom_target(external_thrift) + set(THRIFT_INCLUDE_DIR "${THRIFT_INCLUDE_DIR}") + set(THRIFT_LIBRARY "${THRIFT_LIBRARIES}") + return() + elseif(WITH_THRIFT STREQUAL "system") + message(FATAL_ERROR "System Thrift requested but not found!") + endif() +endif() + include(ExternalProject) set(INSTALL_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/thrift) -SET(THRIFT_INCLUDE_DIRS "${INSTALL_LOCATION}/include") -SET(THRIFT_LIBRARY_DIRS "${INSTALL_LOCATION}/lib") +set(THRIFT_INCLUDE_DIRS "${INSTALL_LOCATION}/include") +set(THRIFT_LIBRARY_DIRS "${INSTALL_LOCATION}/lib") set(THRIFT_LIBRARY ${THRIFT_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}thrift${CMAKE_STATIC_LIBRARY_SUFFIX}) - -ExternalProject_Add(external_thrift +ExternalProject_Add( + external_thrift URL https://github.com/apache/thrift/archive/refs/tags/v0.17.0.tar.gz URL_HASH SHA256=f5888bcd3b8de40c2c2ab86896867ad9b18510deb412cba3e5da76fb4c604c29 PREFIX ${INSTALL_LOCATION} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_LOCATION} - -DBUILD_COMPILER=YES - -DBUILD_CPP=YES - -DBUILD_C_GLIB=YES - -DBUILD_JAVA=NO - -DBUILD_JAVASCRIPT=NO - -DBUILD_KOTLIN=NO - -DBUILD_NODEJS=NO - -DBUILD_PYTHON=NO - -DBUILD_TESTING=NO - -DBUILD_SHARED_LIBS=NO - -DCMAKE_CXX_FLAGS:STRING="-fPIC" - -DBOOST_INCLUDEDIR=${Boost_INCLUDE_DIRS} - -DBOOST_LIBRARYDIR=${Boost_LIBRARY_DIRS} + -DBUILD_COMPILER=YES + -DBUILD_CPP=YES + -DBUILD_C_GLIB=YES + -DBUILD_JAVA=NO + -DBUILD_JAVASCRIPT=NO + -DBUILD_KOTLIN=NO + -DBUILD_NODEJS=NO + -DBUILD_PYTHON=NO + -DBUILD_TESTING=NO + -DBUILD_SHARED_LIBS=NO + -DCMAKE_CXX_FLAGS:STRING="-fPIC" + -DBOOST_ROOT=${BOOST_ROOT} BUILD_BYPRODUCTS "${THRIFT_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}thrift${CMAKE_STATIC_LIBRARY_SUFFIX}" EXCLUDE_FROM_ALL TRUE ) diff --git a/core_dumps/columnstore_schema.sql b/core_dumps/columnstore_schema.sql new file mode 100644 index 000000000..7cf37fac4 --- /dev/null +++ b/core_dumps/columnstore_schema.sql @@ -0,0 +1,55 @@ +DROP DATABASE IF EXISTS `columnstore_bts`; + +CREATE DATABASE `columnstore_bts`; + +USE `columnstore_bts`; + +CREATE TABLE `airlines` ( + `iata_code` varchar(2) DEFAULT NULL, + `airline` varchar(30) DEFAULT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci; + +CREATE TABLE `airports` ( + `iata_code` varchar(3) DEFAULT NULL, + `airport` varchar(80) DEFAULT NULL, + `city` varchar(30) DEFAULT NULL, + `state` varchar(2) DEFAULT NULL, + `country` varchar(30) DEFAULT NULL, + `latitude` decimal(11,4) DEFAULT NULL, + `longitude` decimal(11,4) DEFAULT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci; + +CREATE TABLE `flights` ( + `year` smallint(6) DEFAULT NULL, + `month` tinyint(4) DEFAULT NULL, + `day` tinyint(4) DEFAULT NULL, + `day_of_week` tinyint(4) DEFAULT NULL, + `fl_date` date DEFAULT NULL, + `carrier` varchar(2) DEFAULT NULL, + `tail_num` varchar(6) DEFAULT NULL, + `fl_num` smallint(6) DEFAULT NULL, + `origin` varchar(5) DEFAULT NULL, + `dest` varchar(5) DEFAULT NULL, + `crs_dep_time` varchar(4) DEFAULT NULL, + `dep_time` varchar(4) DEFAULT NULL, + `dep_delay` decimal(13,2) DEFAULT NULL, + `taxi_out` decimal(13,2) DEFAULT NULL, + `wheels_off` varchar(4) DEFAULT NULL, + `wheels_on` varchar(4) DEFAULT NULL, + `taxi_in` decimal(13,2) DEFAULT NULL, + `crs_arr_time` varchar(4) DEFAULT NULL, + `arr_time` varchar(4) DEFAULT NULL, + `arr_delay` decimal(13,2) DEFAULT NULL, + `cancelled` decimal(13,2) DEFAULT NULL, + `cancellation_code` varchar(20) DEFAULT NULL, + `diverted` decimal(13,2) DEFAULT NULL, + `crs_elapsed_time` decimal(13,2) DEFAULT NULL, + `actual_elapsed_time` decimal(13,2) DEFAULT NULL, + `air_time` decimal(13,2) DEFAULT NULL, + `distance` decimal(13,2) DEFAULT NULL, + `carrier_delay` decimal(13,2) DEFAULT NULL, + `weather_delay` decimal(13,2) DEFAULT NULL, + `nas_delay` decimal(13,2) DEFAULT NULL, + `security_delay` decimal(13,2) DEFAULT NULL, + `late_aircraft_delay` decimal(13,2) DEFAULT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci; \ No newline at end of file diff --git a/core_dumps/docker-awaiter.sh b/core_dumps/docker-awaiter.sh new file mode 100755 index 000000000..1562532e5 --- /dev/null +++ b/core_dumps/docker-awaiter.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +DOCKER_IMAGE=$1 + +zhdun() +{ + command=$1 + expected_result=$2 + waiting_message=$3 + retries=$4 + sleep_delay=$5 + result=$($command) + result="${result%%[[:cntrl:]]}" + retries_counter=1 + + while true; + do + if [ "$result" != "$expected_result" ]; then + echo $waiting_message " Status: " $result ", attempt: " $retries_counter + sleep $sleep_delay + else + echo Finished waiting for \'"$command"\' to return \'"$expected_result"\' + exit + fi + + if [ $retries_counter -ge $retries ]; then + echo "Tired to wait for retry, $retries_counter attemps were made" + exit + fi + retries_counter=$(($retries_counter + 1)) + done +} + +check_result="running" +check_command="docker exec -t $DOCKER_IMAGE systemctl is-system-running" +waiting_message="Waiting for docker container to start systemd." + +zhdun "$check_command" "$check_result" "$waiting_message" 60 2 diff --git a/core_dumps/save_stack.sh b/core_dumps/save_stack.sh new file mode 100644 index 000000000..66970e222 --- /dev/null +++ b/core_dumps/save_stack.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env sh + +set -x +set -o pipefail + + +SCRIPT_LOCATION=$(dirname "$0") +LOG_PATH=$1 + + +dump_stack () +{ + name=$1 + echo "\nStack trace of $1" + eu-stack -p `pidof $name` -n 0 | tee ${LOG_PATH}/${name}_callstacks.txt +} + +dump_stack "mariadbd" +dump_stack "workernode" +dump_stack "controllernode" +dump_stack "WriteEngineServer" +dump_stack "DDLProc" +dump_stack "DMLProc" +dump_stack "PrimProc" diff --git a/core_dumps/upgrade_data.sh b/core_dumps/upgrade_data.sh new file mode 100755 index 000000000..d4d26a0f1 --- /dev/null +++ b/core_dumps/upgrade_data.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -xeuo pipefail + +MARIADB=$(which mysql) +CPIMPORT=$(which cpimport) +SCHEMA_DIR=$(dirname "$0") +NAME1='airports' +NAME2='airlines' + + +get_data () +{ + NAME=$1 + if curl -o "${SCHEMA_DIR}/${NAME}.csv" -# "https://sample-columnstore-data.s3.us-west-2.amazonaws.com/${NAME}.csv"; then + echo -e "Downloaded '${NAME}.csv' ... done\n" + else + echo -e "Downloading '${NAME}.csv' ... failed" + exit 1 + fi +} + +import_data () +{ + NAME=$1 + echo -e "\nLoading '${NAME}.csv' with cpimport ..." + if ! $CPIMPORT -m 1 -s ',' -E '"' columnstore_bts "${NAME}" -l "${SCHEMA_DIR}/${NAME}.csv"; then + echo -e "loading '${NAME}.csv' ... fail" + exit 1 + fi +} + + +if $MARIADB <"${SCHEMA_DIR}"/columnstore_schema.sql &>/dev/null; then + echo -e "Creating 'columnstore_bts' schema..." "done" +else + echo -e "Creating 'columnstore_bts' schema..." "fail" + exit 1 +fi + +get_data "$NAME1" +get_data "$NAME2" + +import_data "$NAME1" +import_data "$NAME2" diff --git a/core_dumps/upgrade_setup_deb.sh b/core_dumps/upgrade_setup_deb.sh new file mode 100755 index 000000000..82c7b131b --- /dev/null +++ b/core_dumps/upgrade_setup_deb.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +source ./utils.sh + +set -xeuo pipefail + +VERSION="$1" +RESULT="$2" +ARCH="$3" +LINK="$4" +UPGRADE_TOKEN="$5" + +DEBIAN_FRONTEND=noninteractive +UCF_FORCE_CONFNEW=1 + +apt install --yes rsyslog +sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d +bash -c "apt update --yes && apt install -y procps wget curl" +wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup +chmod +x mariadb_es_repo_setup +bash -c "./mariadb_es_repo_setup --token=${UPGRADE_TOKEN} --apply --mariadb-server-version=${VERSION} --skip-maxscale --skip-tools" +apt update --yes +apt install --yes -oDebug::RunScripts=1 mariadb-server mariadb-client mariadb-plugin-columnstore +systemctl start mariadb +systemctl start mariadb-columnstore + +INITIAL_VERSION=$(mariadb -e "select @@version;") + +bash -c "./upgrade_data.sh" +bash -c "./upgrade_verify.sh" + +touch /etc/apt/auth.conf +cat << EOF > /etc/apt/auth.conf +machine ${LINK}${RESULT}/ +EOF + +bash -c "./setup-repo.sh" + + +# Configuration file '/etc/columnstore/Columnstore.xml' +# ==> Modified (by you or by a script) since installation. +# ==> Package distributor has shipped an updated version. +# What would you like to do about it ? Your options are: +# Y or I : install the package maintainer's version +# N or O : keep your currently-installed version +# D : show the differences between the versions +# Z : start a shell to examine the situation +# The default action is to keep your current version. + +# the -o options are used to make choise of keep your currently-installed version without interactive prompt + +apt-get --yes --with-new-pkgs -oDebug::RunScripts=1 -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade + +UPGRADED_VERSION=$(mariadb -e "select @@version;") + +if [[ "$INITIAL_VERSION" == "$UPGRADED_VERSION" ]]; then + error "The upgrade didn't happen!" + exit 1 +else + message_splitted "The upgrade from "$INITIAL_VERSION" to "$UPGRADED_VERSION" succeded!" + bash -c "./upgrade_verify.sh" +fi diff --git a/core_dumps/upgrade_setup_rpm.sh b/core_dumps/upgrade_setup_rpm.sh new file mode 100755 index 000000000..e8444bd16 --- /dev/null +++ b/core_dumps/upgrade_setup_rpm.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +source ./utils.sh + +set -xeuo pipefail + +VERSION="$1" +RESULT="$2" +ARCH="$3" +LINK="$4" +UPGRADE_TOKEN="$5" + +yum clean all +yum install -y wget which procps-ng diffutils rsyslog +wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup +chmod +x mariadb_es_repo_setup +bash -c "./mariadb_es_repo_setup --token=${UPGRADE_TOKEN} --apply --mariadb-server-version=${VERSION} --skip-maxscale --skip-tools --skip-check-installed" +yum repo-pkgs mariadb-es-main list +yum -y install MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo + +systemctl start mariadb +systemctl start mariadb-columnstore + +INITIAL_VERSION=$(mariadb -e "select @@version;") + +bash -c "./upgrade_data.sh" +bash -c "./upgrade_verify.sh" + +bash -c "./setup-repo.sh" + +yum repo-pkgs repo list +yum -y update MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo + +UPGRADED_VERSION=$(mariadb -e "select @@version;") + +if [[ "$INITIAL_VERSION" == "$UPGRADED_VERSION" ]]; then + error "The upgrade didn't happen!" + exit 1 +else + message_splitted "The upgrade from "$INITIAL_VERSION" to "$UPGRADED_VERSION" succeded!" + bash -c "./upgrade_verify.sh" +fi diff --git a/core_dumps/upgrade_verify.sh b/core_dumps/upgrade_verify.sh new file mode 100755 index 000000000..6c8d11f9a --- /dev/null +++ b/core_dumps/upgrade_verify.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -xeuo pipefail + +SCHEMA_DIR=$(dirname "$0") +NAME1='airports' +NAME2='airlines' + + +test_data () +{ + NAME=$1 + mariadb --init-command="SET sql_mode=''" -vvv -e "select * into outfile '/tmp/${NAME}.test.csv' FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' FROM columnstore_bts.${NAME};" + diff <(tail -n +2 "${NAME}.csv") <(tail -n +2 "/tmp/${NAME}.test.csv") + rm "/tmp/${NAME}.test.csv" +} + + +test_data "$NAME1" +test_data "$NAME2" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d359edb2f..e66b89a44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,40 +1,41 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) -MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) -MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") +include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}) -if (WITH_UNITTESTS) +if(COLUMNSTORE_ASAN_FOR_UNITTESTS) + my_check_and_set_compiler_flag("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) + my_check_and_set_compiler_flag("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") +endif() + +if(WITH_UNITTESTS) cmake_policy(SET CMP0054 NEW) set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external) - ExternalProject_Add(googletest + ExternalProject_Add( + googletest GIT_REPOSITORY https://github.com/google/googletest GIT_TAG release-1.12.0 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DBUILD_SHARED_LIBS=ON ) - # this is a workaround for gtest's bug still not closed - # https://github.com/google/googletest/issues/3659 - if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - MY_CHECK_AND_SET_COMPILER_FLAG("-U__cpp_char8_t") + # this is a workaround for gtest's bug still not closed https://github.com/google/googletest/issues/3659 + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + my_check_and_set_compiler_flag("-U__cpp_char8_t") endif() - include_directories(${EXTERNAL_INSTALL_LOCATION}/include) # lib64 for RPM-based distros link_directories(${EXTERNAL_INSTALL_LOCATION}/lib ${EXTERNAL_INSTALL_LOCATION}/lib64) set(GTEST_LIBRARIES gtest gtest_main pthread) include(GoogleTest) - #GoogleTest tests + # GoogleTest tests add_executable(rowgroup_tests rowgroup-tests.cpp) add_dependencies(rowgroup_tests googletest) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) gtest_add_tests(TARGET rowgroup_tests TEST_PREFIX columnstore:) - add_executable(rewritetest rewritetest.cpp) add_dependencies(rewritetest googletest) target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} messageqcpp execplan) - # gtest_add_tests(TARGET rewritetest TEST_PREFIX columnstore:) + # gtest_add_tests(TARGET rewritetest TEST_PREFIX columnstore:) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) add_dependencies(mcs_decimal_tests googletest) @@ -59,7 +60,9 @@ if (WITH_UNITTESTS) add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp) target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare) add_dependencies(column_scan_filter_tests googletest) - target_link_libraries(column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) + target_link_libraries( + column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc + ) gtest_add_tests(TARGET column_scan_filter_tests TEST_PREFIX columnstore:) add_executable(simd_processors simd_processors.cpp) @@ -96,11 +99,20 @@ if (WITH_UNITTESTS) set_tests_properties(columnstore:brm_em_standalone PROPERTIES DISABLED True) endif() # -Werror=sign-compare -if (WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")) +if(WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")) find_package(benchmark REQUIRED) add_executable(primitives_scan_bench primitives_scan_bench.cpp) - target_include_directories(primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) - target_link_libraries(primitives_scan_bench ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc benchmark::benchmark) + target_include_directories( + primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} + ) + target_link_libraries( + primitives_scan_bench + ${ENGINE_LDFLAGS} + ${ENGINE_WRITE_LIBS} + ${GTEST_LIBRARIES} + processor + dbbc + benchmark::benchmark + ) add_test(NAME columnstore_microbenchmarks:primitives_scan_bench, COMMAND primitives_scan_bench) endif() - diff --git a/tests/counting_allocator.cpp b/tests/counting_allocator.cpp index 6a7bdfbfc..4a2a76a36 100644 --- a/tests/counting_allocator.cpp +++ b/tests/counting_allocator.cpp @@ -32,7 +32,7 @@ struct TestClass { int value[1024]; - TestClass(int val) : value(val) + TestClass(int val) : value{val} { } }; diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index a708c34e4..529bde177 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -1,9 +1,7 @@ +include_directories(${ENGINE_COMMON_INCLUDES}) -include_directories( ${ENGINE_COMMON_INCLUDES} ) - - -########### next target ############### -# func_decode_oracle.cpp +# ########## next target ############### +# func_decode_oracle.cpp set(funcexp_LIB_SRCS functor.cpp @@ -144,13 +142,17 @@ set(funcexp_LIB_SRCS func_year.cpp func_yearweek.cpp jsonhelpers.cpp - sql_crypt.cpp) + sql_crypt.cpp +) add_library(funcexp SHARED ${funcexp_LIB_SRCS}) add_dependencies(funcexp loggingcpp) -target_link_libraries(funcexp ${NETSNMP_LIBRARIES}) - -install(TARGETS funcexp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) +target_link_libraries(funcexp ${NETSNMP_LIBRARIES} ${MARIADB_STRING_LIBS}) +install( + TARGETS funcexp + DESTINATION ${ENGINE_LIBDIR} + COMPONENT columnstore-engine +)