diff --git a/.drone.jsonnet b/.drone.jsonnet index 0ddea1abf..5219830d5 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,533 +1,555 @@ -local events = ['pull_request', 'cron']; +local events = ["pull_request", "cron"]; local servers = { - develop: ['10.6-enterprise'], - 'stable-23.10': ['10.6-enterprise'], + develop: ["10.6-enterprise"], + "stable-23.10": ["10.6-enterprise"], }; local platforms = { - develop: ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'], - 'stable-23.10': ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04','ubuntu:24.04'], + develop: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], + "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], }; local platforms_arm = { - develop: ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'], - 'stable-23.10': ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'], + develop: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], + "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], }; -local customBuildEnvCommandsMap = { - 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'], - 'clang-19': ['apt install -y clang-19', 'export CC=/usr/bin/clang-19', 'export CXX=/usr/bin/clang++-19'], - 'clang-20': ['apt install -y wget curl lsb-release software-properties-common gnupg', - 'wget https://apt.llvm.org/llvm.sh', - 'bash llvm.sh 20', - 'export CC=/usr/bin/clang-20', - 'export CXX=/usr/bin/clang++-20'], +local customEnvCommandsMap = { + // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'], + "clang-20": [ + "apt install -y wget curl lsb-release software-properties-common gnupg", + "wget https://apt.llvm.org/llvm.sh", + "bash llvm.sh 20", + "export CC=/usr/bin/clang", + "export CXX=/usr/bin/clang++", + ], +}; + +local customEnvCommands(envkey, builddir) = + local updateAlternatives = { + "clang-20": ["bash /mdb/" + builddir + + "/storage/columnstore/columnstore/build/update-clang-version.sh 20 100"], + }; + (if (std.objectHas(customEnvCommandsMap, envkey)) + then customEnvCommandsMap[envkey] + updateAlternatives[envkey] else []); + + +local customBootstrapParamsForExisitingPipelines(envkey) = + local customBootstrapMap = { + "ubuntu:24.04": "--custom-cmake-flags '-DCOLUMNSTORE_ASAN_FOR_UNITTESTS=YES'", + }; + (if (std.objectHas(customBootstrapMap, envkey)) + then customBootstrapMap[envkey] else ""); + +local customBootstrapParamsForAdditionalPipelinesMap = { + ASAN: "--asan", + TSAN: "--tsan", + UBSAN: "--ubsan", }; -local any_branch = '**'; +local any_branch = "**"; local platforms_custom = platforms.develop; local platforms_arm_custom = platforms_arm.develop; local platforms_mtr = platforms.develop; -local builddir = 'verylongdirnameforverystrangecpackbehavior'; +local builddir = "verylongdirnameforverystrangecpackbehavior"; -local mtr_suite_list = 'basic,bugfixes'; -local mtr_full_set = 'basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly'; +local mtr_suite_list = "basic,bugfixes"; +local mtr_full_set = "basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly"; local upgrade_test_lists = { - "rockylinux8": { - "arm64": ["10.6.4-1", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"], - "amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"] - }, - "rockylinux9": { - "arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], - "amd64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"] - }, - "debian11": { - "arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], - "amd64": ["10.6.5-2", "10.6.7-3", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"] - }, - "debian12": { - "arm64": [], - "amd64": [] + rockylinux8: { + arm64: ["10.6.4-1", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"], + amd64: ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + }, + rockylinux9: { + arm64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + amd64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + }, + + debian12: { + arm64: [], + amd64: [], }, "ubuntu20.04": { - "arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], - "amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"] - }, - "ubuntu22.04": { - "arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], - "amd64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"] - }, - "ubuntu24.04": - { - "arm64": [], - "amd64": [] + arm64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + amd64: ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], }, + "ubuntu22.04": { + arm64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + amd64: ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"], + }, + "ubuntu24.04": + { + arm64: [], + amd64: [], + }, }; local testRun(platform) = local platform_map = { - 'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', - 'rockylinux:9': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', - 'debian:11': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', - 'debian:12': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', - 'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', - 'ubuntu:22.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', - 'ubuntu:24.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure', + "rockylinux:8": "ctest3 -R columnstore: -j $(nproc) --output-on-failure", + "rockylinux:9": "ctest3 -R columnstore: -j $(nproc) --output-on-failure", + "debian:12": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", + "ubuntu:20.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", + "ubuntu:22.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", + "ubuntu:24.04": "cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure", }; platform_map[platform]; -local gcc_version = '11'; +local gcc_version = "11"; local rockylinux8_deps = "dnf install -y 'dnf-command(config-manager)' " + - '&& dnf config-manager --set-enabled powertools ' + - '&& dnf install -y gcc-toolset-' + gcc_version + ' libarchive cmake ' + - '&& . /opt/rh/gcc-toolset-' + gcc_version + '/enable '; + "&& dnf config-manager --set-enabled powertools " + + "&& dnf install -y gcc-toolset-" + gcc_version + " libarchive cmake " + + "&& . /opt/rh/gcc-toolset-" + gcc_version + "/enable "; local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " + - '&& dnf config-manager --set-enabled crb ' + - '&& dnf install -y gcc gcc-c++'; + "&& dnf config-manager --set-enabled crb " + + "&& dnf install -y gcc gcc-c++"; -local rockylinux_common_deps = ' && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel'; +local rockylinux_common_deps = " && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel"; -local deb_deps = 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev'; +local deb_deps = "apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev"; local testPreparation(platform) = local platform_map = { - 'rockylinux:8': rockylinux8_deps + rockylinux_common_deps, - 'rockylinux:9': rockylinux9_deps + rockylinux_common_deps, - 'debian:11': deb_deps, - 'debian:12': deb_deps, - 'ubuntu:20.04': deb_deps, - 'ubuntu:22.04': deb_deps, - 'ubuntu:24.04': deb_deps, - + "rockylinux:8": rockylinux8_deps + rockylinux_common_deps, + "rockylinux:9": rockylinux9_deps + rockylinux_common_deps, + "debian:12": deb_deps, + "ubuntu:20.04": deb_deps, + "ubuntu:22.04": deb_deps, + "ubuntu:24.04": deb_deps, }; platform_map[platform]; -local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise', customBuildEnvCommandsMapKey='') = { - local pkg_format = if (std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb', - local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd', - local mtr_path = if (pkg_format == 'rpm') then '/usr/share/mysql-test' else '/usr/share/mysql/mysql-test', - local cmapi_path = '/usr/share/columnstore/cmapi', - local etc_path = '/etc/columnstore', - local socket_path = if (pkg_format == 'rpm') then '/var/lib/mysql/mysql.sock' else '/run/mysqld/mysqld.sock', - local config_path_prefix = if (pkg_format == 'rpm') then '/etc/my.cnf.d/' else '/etc/mysql/mariadb.conf.d/50-', - local img = if (platform == 'rockylinux:8') then platform else 'detravi/' + std.strReplace(platform, '/', '-'), - local branch_ref = if (branch == any_branch) then 'stable-23.10' else branch, +local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = { + local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb", + local init = if (pkg_format == "rpm") then "/usr/lib/systemd/systemd" else "systemd", + local mtr_path = if (pkg_format == "rpm") then "/usr/share/mysql-test" else "/usr/share/mysql/mysql-test", + local cmapi_path = "/usr/share/columnstore/cmapi", + local etc_path = "/etc/columnstore", + local socket_path = if (pkg_format == "rpm") then "/var/lib/mysql/mysql.sock" else "/run/mysqld/mysqld.sock", + local config_path_prefix = if (pkg_format == "rpm") then "/etc/my.cnf.d/" else "/etc/mysql/mariadb.conf.d/50-", + local img = if (platform == "rockylinux:8") then platform else "detravi/" + std.strReplace(platform, "/", "-"), + local branch_ref = if (branch == any_branch) then "stable-23.10" else branch, // local regression_tests = if (std.startsWith(platform, 'debian') || std.startsWith(platform, 'ubuntu:20')) then 'test000.sh' else 'test000.sh,test001.sh', - local branchp = if (branch == '**') then '' else branch + '/', - local brancht = if (branch == '**') then '' else branch + '-', - local platformKey = std.strReplace(std.strReplace(platform, ':', ''), '/', '-'), - local result = platformKey + if customBuildEnvCommandsMapKey != '' then '_' + customBuildEnvCommandsMapKey else '', + local branchp = if (branch == "**") then "" else branch + "/", + local brancht = if (branch == "**") then "" else branch + "-", + local platformKey = std.strReplace(std.strReplace(platform, ":", ""), "/", "-"), + local result = platformKey + if customBuildEnvCommandsMapKey != "" then "_" + customBuildEnvCommandsMapKey else "", - local packages_url = 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server, + local packages_url = "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server, local publish_pkg_url = "https://cspkg.s3.amazonaws.com/index.html?prefix=" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/" + result + "/", local repo_pkg_url_no_res = "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/", - local container_tags = if (event == 'cron') then [brancht + std.strReplace(event, '_', '-') + '${DRONE_BUILD_NUMBER}', brancht] else [brancht + std.strReplace(event, '_', '-') + '${DRONE_BUILD_NUMBER}'], - local container_version = branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch, + local container_tags = if (event == "cron") then [brancht + std.strReplace(event, "_", "-") + "${DRONE_BUILD_NUMBER}", brancht] else [brancht + std.strReplace(event, "_", "-") + "${DRONE_BUILD_NUMBER}"], + local container_version = branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch, - local server_remote = if (std.endsWith(server, 'enterprise')) then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server', + local server_remote = if (std.endsWith(server, "enterprise")) then "https://github.com/mariadb-corporation/MariaDBEnterprise" else "https://github.com/MariaDB/server", - local sccache_arch = if (arch == 'amd64') then 'x86_64' else 'aarch64', - local get_sccache = 'echo getting sccache... && (apt update -y && apt install -y curl || yum install -y curl || true) ' + - '&& curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-' + sccache_arch + '-unknown-linux-musl.tar.gz ' + - '&& tar xzf sccache.tar.gz ' + - '&& install sccache*/sccache /usr/local/bin/ && echo sccache installed', + local sccache_arch = if (arch == "amd64") then "x86_64" else "aarch64", + local get_sccache = "echo getting sccache... && (apt update -y && apt install -y curl || yum install -y curl || true) " + + "&& curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz " + + "&& tar xzf sccache.tar.gz " + + "&& install sccache*/sccache /usr/local/bin/ && echo sccache installed", local pipeline = self, - publish(step_prefix='pkg', eventp=event + '/${DRONE_BUILD_NUMBER}'):: { - name: 'publish ' + step_prefix, - depends_on: [std.strReplace(step_prefix, ' latest', ''), 'createrepo'], - image: 'amazon/aws-cli', + publish(step_prefix="pkg", eventp=event + "/${DRONE_BUILD_NUMBER}"):: { + name: "publish " + step_prefix, + depends_on: [std.strReplace(step_prefix, " latest", ""), "createrepo"], + image: "amazon/aws-cli", when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, environment: { AWS_ACCESS_KEY_ID: { - from_secret: 'aws_access_key_id', + from_secret: "aws_access_key_id", }, AWS_SECRET_ACCESS_KEY: { - from_secret: 'aws_secret_access_key', + from_secret: "aws_secret_access_key", }, }, commands: [ '[ -z "$(ls -A "' + result + '")" ] && echo Nothing to publish! && exit 1', - 'aws s3 sync ' + result + ' s3://cspkg/' + branchp + eventp + '/' + server + '/' + arch + '/' + result + ' --only-show-errors', + "aws s3 sync " + result + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors", 'echo "Data uploaded to: ' + publish_pkg_url + '"', + "echo -e '\\e]8;;" + publish_pkg_url + "\\e\\\\" + publish_pkg_url + "\\e]8;;\\e\\\\'", - 'rm -rf ' + result + '/*' + "rm -rf " + result + "/*", ], }, - local regression_tests = if (event == 'cron') then [ - 'test000.sh', - 'test001.sh', - 'test005.sh', - 'test006.sh', - 'test007.sh', - 'test008.sh', - 'test009.sh', - 'test010.sh', - 'test011.sh', - 'test012.sh', - 'test013.sh', - 'test014.sh', - 'test023.sh', - 'test201.sh', - 'test202.sh', - 'test203.sh', - 'test204.sh', - 'test210.sh', - 'test211.sh', - 'test212.sh', - 'test222.sh', - 'test297.sh', - 'test299.sh', - 'test400.sh', - 'test500.sh', + local regression_tests = if (event == "cron") then [ + "test000.sh", + "test001.sh", + "test005.sh", + "test006.sh", + "test007.sh", + "test008.sh", + "test009.sh", + "test010.sh", + "test011.sh", + "test012.sh", + "test013.sh", + "test014.sh", + "test023.sh", + "test201.sh", + "test202.sh", + "test203.sh", + "test204.sh", + "test210.sh", + "test211.sh", + "test212.sh", + "test222.sh", + "test297.sh", + "test299.sh", + "test400.sh", + "test500.sh", ] else [ - 'test000.sh', - 'test001.sh', + "test000.sh", + "test001.sh", ], local mdb_server_versions = upgrade_test_lists[platformKey][arch], local indexes(arr) = std.range(0, std.length(arr) - 1), - local execInnerDocker(command, dockerImage, flags = '') = - 'docker exec ' + flags + ' -t ' + dockerImage + ' ' + command, + local execInnerDocker(command, dockerImage, flags="") = + "docker exec " + flags + " -t " + dockerImage + " " + command, - local execInnerDockerNoTTY(command, dockerImage, flags = '') = - 'docker exec ' + flags + ' ' + dockerImage + ' ' + command, + local execInnerDockerNoTTY(command, dockerImage, flags="") = + "docker exec " + flags + " " + dockerImage + " " + command, local installRpmDeb(pkg_format, rpmpackages, debpackages) = - if (pkg_format == 'rpm') - then ' bash -c "yum install -y ' + rpmpackages + '"' - else ' bash -c "apt update --yes && apt install -y ' + debpackages + '"', + if (pkg_format == "rpm") + then ' bash -c "yum install -y ' + rpmpackages + '"' + else ' bash -c "apt update --yes && apt install -y ' + debpackages + '"', local dockerImage(stepname) = stepname + "$${DRONE_BUILD_NUMBER}", local installEngine(dockerImage, pkg_format) = - if (pkg_format == 'deb') then execInnerDocker('bash -c "apt update && apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage) - else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"', dockerImage), + if (pkg_format == "deb") then execInnerDocker('bash -c "apt update && apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage) + else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"', dockerImage), local installCmapi(dockerImage, pkg_format) = - if (pkg_format == 'deb') then execInnerDocker('bash -c "apt update && apt install -y mariadb-columnstore-cmapi"', dockerImage) - else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-cmapi"', dockerImage), + if (pkg_format == "deb") then execInnerDocker('bash -c "apt update && apt install -y mariadb-columnstore-cmapi"', dockerImage) + else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-cmapi"', dockerImage), local prepareTestStage(dockerImage, pkg_format, result, do_setup) = [ - 'apk add bash && bash core_dumps/docker-awaiter.sh ' + dockerImage, - if (pkg_format == 'deb') - then execInnerDocker('sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', dockerImage), + "apk add bash && bash core_dumps/docker-awaiter.sh " + dockerImage, + if (pkg_format == "deb") + then execInnerDocker('sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', dockerImage), 'echo "Docker CGroups opts here"', - 'ls -al /sys/fs/cgroup/cgroup.controllers || true ', - 'ls -al /sys/fs/cgroup/ || true ', - 'ls -al /sys/fs/cgroup/memory || true', + "ls -al /sys/fs/cgroup/cgroup.controllers || true ", + "ls -al /sys/fs/cgroup/ || true ", + "ls -al /sys/fs/cgroup/memory || true", "docker ps --filter=name=" + dockerImage, execInnerDocker('echo "Inner Docker CGroups opts here"', dockerImage), - execInnerDocker('ls -al /sys/fs/cgroup/cgroup.controllers || true', dockerImage), - execInnerDocker('ls -al /sys/fs/cgroup/ || true', dockerImage), - execInnerDocker('ls -al /sys/fs/cgroup/memory || true', dockerImage), + execInnerDocker("ls -al /sys/fs/cgroup/cgroup.controllers || true", dockerImage), + execInnerDocker("ls -al /sys/fs/cgroup/ || true", dockerImage), + execInnerDocker("ls -al /sys/fs/cgroup/memory || true", dockerImage), - execInnerDocker('mkdir core', dockerImage), - execInnerDocker('chmod 777 core', dockerImage), - 'docker cp core_dumps/. ' + dockerImage + ':/', - 'docker cp build/utils.sh ' + dockerImage + ':/', - 'docker cp setup-repo.sh ' + dockerImage + ':/', - if (do_setup) then execInnerDocker('/setup-repo.sh', dockerImage), + execInnerDocker("mkdir core", dockerImage), + execInnerDocker("chmod 777 core", dockerImage), + "docker cp core_dumps/. " + dockerImage + ":/", + "docker cp build/utils.sh " + dockerImage + ":/", + "docker cp setup-repo.sh " + dockerImage + ":/", + if (do_setup) then execInnerDocker("/setup-repo.sh", dockerImage), execInnerDocker(installRpmDeb(pkg_format, - "cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which", - "elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget"), dockerImage), + "cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which", + "elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget"), dockerImage), execInnerDocker('sysctl -w kernel.core_pattern="/core/%E_' + result + '_core_dump.%p"', dockerImage), ], local reportTestStage(dockerImage, result, stage) = [ execInnerDocker('bash -c "/logs.sh ' + stage + '"', dockerImage), - execInnerDocker('bash -c "/core_dump_check.sh core /core/ ' + stage + '"' , dockerImage), - 'docker cp ' + dockerImage + ':/core/ /drone/src/' + result + '/', - 'docker cp ' + dockerImage + ':/unit_logs/ /drone/src/' + result + '/', - 'ls -l /drone/src/' + result, + execInnerDocker('bash -c "/core_dump_check.sh core /core/ ' + stage + '"', dockerImage), + "docker cp " + dockerImage + ":/core/ /drone/src/" + result + "/", + "docker cp " + dockerImage + ":/unit_logs/ /drone/src/" + result + "/", + "ls -l /drone/src/" + result, execInnerDocker('bash -c "/core_dump_drop.sh core"', dockerImage), - 'docker stop ' + dockerImage + ' && docker rm ' + dockerImage + ' || echo "cleanup ' + stage + ' failure"', + "docker stop " + dockerImage + " && docker rm " + dockerImage + ' || echo "cleanup ' + stage + ' failure"', ], _volumes:: { mdb: { - name: 'mdb', - path: '/mdb', + name: "mdb", + path: "/mdb", }, docker: { - name: 'docker', - path: '/var/run/docker.sock', + name: "docker", + path: "/var/run/docker.sock", }, }, smoke:: { - name: 'smoke', - depends_on: ['publish pkg'], - image: 'docker', + name: "smoke", + depends_on: ["publish pkg"], + image: "docker", volumes: [pipeline._volumes.docker], commands: [ - 'docker run --memory 3g --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] - + prepareTestStage(dockerImage("smoke"), pkg_format, result, true) + [ + "docker run --memory 3g --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target", + ] + + prepareTestStage(dockerImage("smoke"), pkg_format, result, true) + [ installEngine(dockerImage("smoke"), pkg_format), - 'sleep $${SMOKE_DELAY_SECONDS:-1s}', + "sleep $${SMOKE_DELAY_SECONDS:-1s}", // start mariadb and mariadb-columnstore services and run simple query - execInnerDocker('systemctl start mariadb', dockerImage("smoke")), + execInnerDocker("systemctl start mariadb", dockerImage("smoke")), execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("smoke")), - execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("smoke")), + execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("smoke")), execInnerDocker('mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"', dockerImage("smoke")), // restart mariadb and mariadb-columnstore services and run simple query again - execInnerDocker('systemctl restart mariadb', dockerImage("smoke")), - execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("smoke")), - 'sleep 10', + execInnerDocker("systemctl restart mariadb", dockerImage("smoke")), + execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("smoke")), + "sleep 10", execInnerDocker('mariadb -e "insert into test.t1 values (2); select * from test.t1"', dockerImage("smoke")), ], }, smokelog:: { - name: 'smokelog', - depends_on: ['smoke'], - image: 'docker', + name: "smokelog", + depends_on: ["smoke"], + image: "docker", volumes: [pipeline._volumes.docker], commands: [ 'echo "---------- start mariadb service logs ----------"', - execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage('smoke')), + execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("smoke")), 'echo "---------- end mariadb service logs ----------"', - 'echo', + "echo", 'echo "---------- start columnstore debug log ----------"', - execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage('smoke')), - 'echo "---------- end columnstore debug log ----------"' - ] + reportTestStage(dockerImage('smoke'), result, "smoke"), + execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("smoke")), + 'echo "---------- end columnstore debug log ----------"', + ] + reportTestStage(dockerImage("smoke"), result, "smoke"), when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, }, upgrade(version):: { - name: 'upgrade-test from ' + version, - depends_on: ['regressionlog'], - image: 'docker', + name: "upgrade-test from " + version, + depends_on: ["regressionlog"], + image: "docker", volumes: [pipeline._volumes.docker], environment: { UPGRADE_TOKEN: { - from_secret: 'es_token', + from_secret: "es_token", }, }, commands: [ - // why do we mount cgroups here, but miss it on other steps? - 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}' + version + ' --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] - + prepareTestStage(dockerImage('upgrade') + version, pkg_format, result, false) + [ - if (pkg_format == 'deb') - then execInnerDocker('bash -c "./upgrade_setup_deb.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res +' $${UPGRADE_TOKEN}"', - dockerImage('upgrade') + version), - if (std.split(platform, ':')[0] == 'rockylinux') - then execInnerDocker('bash -c "./upgrade_setup_rpm.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', - dockerImage('upgrade') + version), + // why do we mount cgroups here, but miss it on other steps? + "docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}" + version + " --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target", + ] + + prepareTestStage(dockerImage("upgrade") + version, pkg_format, result, false) + [ + if (pkg_format == "deb") + then execInnerDocker('bash -c "./upgrade_setup_deb.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', + dockerImage("upgrade") + version), + if (std.split(platform, ":")[0] == "rockylinux") + then execInnerDocker('bash -c "./upgrade_setup_rpm.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', + dockerImage("upgrade") + version), ], }, upgradelog:: { - name: 'upgradelog', - depends_on: std.map(function(p) 'upgrade-test from ' + p, mdb_server_versions), - image: 'docker', + name: "upgradelog", + depends_on: std.map(function(p) "upgrade-test from " + p, mdb_server_versions), + image: "docker", volumes: [pipeline._volumes.docker], commands: [ - 'echo', - ] + std.flatMap(function(ver) reportTestStage(dockerImage('upgrade') + ver, result, "upgrade_"+ver), mdb_server_versions), + "echo", + ] + std.flatMap(function(ver) reportTestStage(dockerImage("upgrade") + ver, result, "upgrade_" + ver), mdb_server_versions), when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, }, mtr:: { - name: 'mtr', - depends_on: ['smoke'], - image: 'docker:git', + name: "mtr", + depends_on: ["smoke"], + image: "docker:git", volumes: [pipeline._volumes.docker], environment: { - MTR_SUITE_LIST: '${MTR_SUITE_LIST:-' + mtr_suite_list + '}', - MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}', + MTR_SUITE_LIST: "${MTR_SUITE_LIST:-" + mtr_suite_list + "}", + MTR_FULL_SUITE: "${MTR_FULL_SUITE:-false}", }, commands: [ - 'docker run --shm-size=500m --memory 8g --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] - + prepareTestStage('mtr$${DRONE_BUILD_NUMBER}', pkg_format, result, true) + [ + "docker run --shm-size=500m --memory 8g --env MYSQL_TEST_DIR=" + mtr_path + " --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target", + ] + + prepareTestStage("mtr$${DRONE_BUILD_NUMBER}", pkg_format, result, true) + [ installEngine(dockerImage("mtr"), pkg_format), - 'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/', - execInnerDocker('chown -R mysql:mysql ' + mtr_path, dockerImage("mtr")), + "docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:" + mtr_path + "/suite/", + execInnerDocker("chown -R mysql:mysql " + mtr_path, dockerImage("mtr")), // disable systemd 'ProtectSystem' (we need to write to /usr/share/) - execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage('mtr')), - execInnerDocker('systemctl daemon-reload', dockerImage("mtr")), - execInnerDocker('systemctl start mariadb', dockerImage("mtr")), + execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage("mtr")), + execInnerDocker("systemctl daemon-reload", dockerImage("mtr")), + execInnerDocker("systemctl start mariadb", dockerImage("mtr")), // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps. execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("mtr")), execInnerDocker('mariadb -e "create database if not exists test;"', dockerImage("mtr")), - execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("mtr")), + execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("mtr")), // delay mtr for manual debugging on live instance - 'sleep $${MTR_DELAY_SECONDS:-1s}', + "sleep $${MTR_DELAY_SECONDS:-1s}", 'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")', - if (event == 'custom' || event == 'cron') then + if (event == "custom" || event == "cron") then execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"', - dockerImage('mtr')), - if (event == 'custom' || event == 'cron') then - execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"', - dockerImage('mtr')), + dockerImage("mtr")), + if (event == "custom" || event == "cron") then + execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"', + dockerImage("mtr")), - if (event == 'cron') then - execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + - ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=' - + std.join(',', std.map(function(x) 'columnstore/' + x, std.split(mtr_full_set, ','))), - dockerImage('mtr')) + '"' - else - execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + + if (event == "cron") then + execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + + " --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=" + + std.join(",", std.map(function(x) "columnstore/" + x, std.split(mtr_full_set, ","))), + dockerImage("mtr")) + '"' + else + execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"', - dockerImage('mtr')), + dockerImage("mtr")), ], }, mtrlog:: { - name: 'mtrlog', - depends_on: ['mtr'], - image: 'docker', + name: "mtrlog", + depends_on: ["mtr"], + image: "docker", volumes: [pipeline._volumes.docker], commands: [ 'echo "---------- start mariadb service logs ----------"', - execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage('mtr')), + execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("mtr")), 'echo "---------- end mariadb service logs ----------"', - 'echo', + "echo", 'echo "---------- start columnstore debug log ----------"', - execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage('mtr')), + execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("mtr")), 'echo "---------- end columnstore debug log ----------"', 'echo "---------- end columnstore debug log ----------"', - 'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/' + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"' - ] + reportTestStage(dockerImage('mtr'), result, "mtr"), + "docker cp mtr$${DRONE_BUILD_NUMBER}:" + mtr_path + "/var/log /drone/src/" + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"', + ] + reportTestStage(dockerImage("mtr"), result, "mtr"), when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, }, prepare_regression:: { - name: 'prepare regression', - depends_on: ['mtr', 'publish pkg', 'publish cmapi build'], + name: "prepare regression", + depends_on: ["mtr", "publish pkg", "publish cmapi build"], when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, - image: 'docker:git', + image: "docker:git", volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], environment: { - REGRESSION_BRANCH_REF: '${DRONE_SOURCE_BRANCH}', + REGRESSION_BRANCH_REF: "${DRONE_SOURCE_BRANCH}", REGRESSION_REF_AUX: branch_ref, }, commands: [ - // compute branch. - 'echo "$$REGRESSION_REF"', - 'echo "$$REGRESSION_BRANCH_REF"', - // if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR. - 'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}', - 'echo "$$REGRESSION_REF"', - // REGRESSION_REF can be empty if there is no appropriate branch in regression repository. - // assign what is appropriate by default. - 'export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}', - 'echo "$$REGRESSION_REF"', - // clone regression test repo - 'git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test', - // where are we now? - 'cd mariadb-columnstore-regression-test', - 'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD', - 'cd ..', - 'docker run --shm-size=500m --memory 12g --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] - + prepareTestStage(dockerImage('regression'), pkg_format, result, true) + [ + // compute branch. + 'echo "$$REGRESSION_REF"', + 'echo "$$REGRESSION_BRANCH_REF"', + // if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR. + 'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}', + 'echo "$$REGRESSION_REF"', + // REGRESSION_REF can be empty if there is no appropriate branch in regression repository. + // assign what is appropriate by default. + "export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}", + 'echo "$$REGRESSION_REF"', + // clone regression test repo + "git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test", + // where are we now? + "cd mariadb-columnstore-regression-test", + "git rev-parse --abbrev-ref HEAD && git rev-parse HEAD", + "cd ..", + "docker run --shm-size=500m --memory 12g --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target", + ] + + prepareTestStage(dockerImage("regression"), pkg_format, result, true) + [ - 'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/', + "docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/", // list storage manager binary - 'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager', - 'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/', + "ls -la /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager", + "docker cp /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/", // check storage-manager unit test binary file - execInnerDocker('ls -l /storage-manager',dockerImage('regression')), + execInnerDocker("ls -l /storage-manager", dockerImage("regression")), // copy test data for regression test suite - execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"',dockerImage('regression')), - installEngine(dockerImage('regression'), pkg_format), + execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', dockerImage("regression")), + installEngine(dockerImage("regression"), pkg_format), // set mariadb lower_case_table_names=1 config option - execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf', dockerImage('regression')), + execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + "server.cnf", dockerImage("regression")), // set default client character set to utf-8 - execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf',dockerImage('regression')), + execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + "client.cnf", dockerImage("regression")), // Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps. execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("regression")), - execInnerDocker('systemctl start mariadb',dockerImage('regression')), - execInnerDocker('systemctl restart mariadb-columnstore',dockerImage('regression')), + execInnerDocker("systemctl start mariadb", dockerImage("regression")), + execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("regression")), // delay regression for manual debugging on live instance - 'sleep $${REGRESSION_DELAY_SECONDS:-1s}', - execInnerDocker('/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester',dockerImage('regression')), + "sleep $${REGRESSION_DELAY_SECONDS:-1s}", + execInnerDocker("/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester", dockerImage("regression")), ], }, regression(name, depends_on):: { name: name, depends_on: depends_on, - image: 'docker:git', + image: "docker:git", volumes: [pipeline._volumes.docker], when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, - [if (name != 'test000.sh' && name != 'test001.sh') then 'failure']: 'ignore', + [if (name != "test000.sh" && name != "test001.sh") then "failure"]: "ignore", environment: { REGRESSION_TIMEOUT: { - from_secret: 'regression_timeout', + from_secret: "regression_timeout", }, }, commands: [ execInnerDocker("mkdir -p reg-logs", dockerImage("regression"), "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"), execInnerDocker("bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' & ", dockerImage("regresion")), - execInnerDockerNoTTY('bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + ' || ./regression_logs.sh ' + name + '"', - dockerImage("regression"), - "--env PRESERVE_LOGS=true --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"), + execInnerDockerNoTTY('bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + " || ./regression_logs.sh " + name + '"', + dockerImage("regression"), + "--env PRESERVE_LOGS=true --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"), ], }, regressionlog:: { - name: 'regressionlog', + name: "regressionlog", depends_on: [regression_tests[std.length(regression_tests) - 1]], - image: 'docker', + image: "docker", volumes: [pipeline._volumes.docker], commands: [ 'echo "---------- start columnstore regression short report ----------"', execInnerDocker('cat go.log || echo "missing go.log"', - dockerImage('regression'), - '--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'), + dockerImage("regression"), + "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"), 'echo "---------- end columnstore regression short report ----------"', - 'echo', - 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/' + result + '/', - 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"', - execInnerDocker('bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"',dockerImage('regression')), + "echo", + "docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/" + result + "/", + "docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/" + result + '/ || echo "missing testErrorLogs.tgz"', + execInnerDocker('bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"', dockerImage("regression")), execInnerDocker('bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"', - dockerImage('regression'), - '--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'), - 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"', - 'docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/' + result + '/' - ] + reportTestStage(dockerImage('regression'), result, "regression"), + dockerImage("regression"), + "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"), + "docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/" + result + '/ || echo "missing testErrorLogs.tgz"', + "docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/" + result + "/", + ] + reportTestStage(dockerImage("regression"), result, "regression"), when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, }, dockerfile:: { - name: 'dockerfile', - depends_on: ['publish pkg', 'publish cmapi build'], + name: "dockerfile", + depends_on: ["publish pkg", "publish cmapi build"], //failure: 'ignore', - image: 'alpine/git', + image: "alpine/git", environment: { - DOCKER_BRANCH_REF: '${DRONE_SOURCE_BRANCH}', + DOCKER_BRANCH_REF: "${DRONE_SOURCE_BRANCH}", DOCKER_REF_AUX: branch_ref, }, commands: [ @@ -539,249 +561,253 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise', 'echo "$$DOCKER_REF"', // DOCKER_REF can be empty if there is no appropriate branch in docker repository. // assign what is appropriate by default. - 'export DOCKER_REF=$${DOCKER_REF:-$$DOCKER_REF_AUX}', + "export DOCKER_REF=$${DOCKER_REF:-$$DOCKER_REF_AUX}", 'echo "$$DOCKER_REF"', - 'git clone --branch $$DOCKER_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker', - 'touch docker/.secrets', + "git clone --branch $$DOCKER_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker", + "touch docker/.secrets", ], }, dockerhub:: { - name: 'dockerhub', - depends_on: ['dockerfile'], + name: "dockerhub", + depends_on: ["dockerfile"], //failure: 'ignore', - image: 'plugins/docker', + image: "plugins/docker", environment: { VERSION: container_version, - MCS_REPO: 'columnstore', - DEV: 'true', + MCS_REPO: "columnstore", + DEV: "true", // branchp has slash if not empty - MCS_BASEURL: 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/', - CMAPI_REPO: 'cmapi', - CMAPI_BASEURL: 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/', + MCS_BASEURL: "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/" + result + "/", + CMAPI_REPO: "cmapi", + CMAPI_BASEURL: "https://cspkg.s3.amazonaws.com/" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/" + result + "/", }, settings: { - repo: 'mariadb/enterprise-columnstore-dev', - context: 'docker', - dockerfile: 'docker/Dockerfile', - build_args_from_env: ['VERSION', 'MCS_REPO', 'MCS_BASEURL', 'CMAPI_REPO', 'CMAPI_BASEURL', 'DEV'], + repo: "mariadb/enterprise-columnstore-dev", + context: "docker", + dockerfile: "docker/Dockerfile", + build_args_from_env: ["VERSION", "MCS_REPO", "MCS_BASEURL", "CMAPI_REPO", "CMAPI_BASEURL", "DEV"], tags: container_tags, username: { - from_secret: 'dockerhub_user', + from_secret: "dockerhub_user", }, password: { - from_secret: 'dockerhub_password', + from_secret: "dockerhub_password", }, }, }, cmapitest:: { - name: 'cmapi test', - depends_on: ['publish cmapi build'], - image: 'docker:git', + name: "cmapi test", + depends_on: ["publish cmapi build"], + image: "docker:git", volumes: [pipeline._volumes.docker], environment: { - PYTHONPATH: '/usr/share/columnstore/cmapi/deps', + PYTHONPATH: "/usr/share/columnstore/cmapi/deps", }, commands: [ - 'docker run --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env PYTHONPATH=$${PYTHONPATH} --name cmapi$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] + - prepareTestStage(dockerImage('cmapi'), pkg_format, result, true) + [ - if (platform == 'rockylinux:9') then execInnerDocker('bash -c "yum install -y libxcrypt-compat"', dockerImage('cmapi')), - installEngine(dockerImage('cmapi'), pkg_format), - installCmapi(dockerImage('cmapi'), pkg_format), - 'cd cmapi', - 'for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/$${i}/; done', - 'docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/', - execInnerDocker('systemctl start mariadb-columnstore-cmapi', dockerImage('cmapi')), + "docker run --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env PYTHONPATH=$${PYTHONPATH} --name cmapi$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target", + ] + + prepareTestStage(dockerImage("cmapi"), pkg_format, result, true) + [ + if (platform == "rockylinux:9") then execInnerDocker('bash -c "yum install -y libxcrypt-compat"', dockerImage("cmapi")), + installEngine(dockerImage("cmapi"), pkg_format), + installCmapi(dockerImage("cmapi"), pkg_format), + "cd cmapi", + "for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/$${i}/; done", + "docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/", + execInnerDocker("systemctl start mariadb-columnstore-cmapi", dockerImage("cmapi")), // set API key to /etc/columnstore/cmapi_server.conf - execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', dockerImage('cmapi')), + execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', dockerImage("cmapi")), // copy cmapi conf file for test purposes (there are api key already set inside) - execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], dockerImage('cmapi')), - execInnerDocker('systemctl stop mariadb-columnstore-cmapi', dockerImage('cmapi')), - execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', dockerImage('cmapi')), + execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], dockerImage("cmapi")), + execInnerDocker("systemctl stop mariadb-columnstore-cmapi", dockerImage("cmapi")), + execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', dockerImage("cmapi")), ], }, - cmapilog:: { - name: 'cmapilog', - depends_on: ['cmapi test'], - image: 'docker', - volumes: [pipeline._volumes.docker], - commands: [ - 'echo "---------- start mariadb service logs ----------"', - execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage('cmapi')), - 'echo "---------- end mariadb service logs ----------"', - 'echo', - 'echo "---------- start columnstore debug log ----------"', - execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage('cmapi')), - 'echo "---------- end columnstore debug log ----------"', - 'echo "---------- start cmapi log ----------"', - execInnerDocker('cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi cmapi_server.log"', dockerImage('cmapi')), - 'echo "---------- end cmapi log ----------"'] - + reportTestStage(dockerImage('cmapi'), result, "cmapi"), - when: { - status: ['success', 'failure'], - }, + cmapilog:: { + name: "cmapilog", + depends_on: ["cmapi test"], + image: "docker", + volumes: [pipeline._volumes.docker], + commands: [ + 'echo "---------- start mariadb service logs ----------"', + execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("cmapi")), + 'echo "---------- end mariadb service logs ----------"', + "echo", + 'echo "---------- start columnstore debug log ----------"', + execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("cmapi")), + 'echo "---------- end columnstore debug log ----------"', + 'echo "---------- start cmapi log ----------"', + execInnerDocker('cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi cmapi_server.log"', dockerImage("cmapi")), + 'echo "---------- end cmapi log ----------"', + ] + + reportTestStage(dockerImage("cmapi"), result, "cmapi"), + when: { + status: ["success", "failure"], }, + }, multi_node_mtr:: { - name: 'mtr', - depends_on: ['dockerhub'], + name: "mtr", + depends_on: ["dockerhub"], //failure: 'ignore', - image: 'docker', + image: "docker", volumes: [pipeline._volumes.docker], environment: { DOCKER_LOGIN: { - from_secret: 'dockerhub_user', + from_secret: "dockerhub_user", }, DOCKER_PASSWORD: { - from_secret: 'dockerhub_password', + from_secret: "dockerhub_password", }, - MCS_IMAGE_NAME: 'mariadb/enterprise-columnstore-dev:' + container_tags[0], + MCS_IMAGE_NAME: "mariadb/enterprise-columnstore-dev:" + container_tags[0], }, commands: [ - 'echo $$DOCKER_PASSWORD | docker login --username $$DOCKER_LOGIN --password-stdin', - 'cd docker', - 'cp .env_example .env', + "echo $$DOCKER_PASSWORD | docker login --username $$DOCKER_LOGIN --password-stdin", + "cd docker", + "cp .env_example .env", 'sed -i "/^MCS_IMAGE_NAME=/s/=.*/=${MCS_IMAGE_NAME}/" .env', 'sed -i "/^MAXSCALE=/s/=.*/=false/" .env', - 'docker-compose up -d', - 'docker exec mcs1 provision mcs1 mcs2 mcs3', - 'docker cp ../mysql-test/columnstore mcs1:' + mtr_path + '/suite/', - 'docker exec -t mcs1 chown mysql:mysql -R ' + mtr_path, + "docker-compose up -d", + "docker exec mcs1 provision mcs1 mcs2 mcs3", + "docker cp ../mysql-test/columnstore mcs1:" + mtr_path + "/suite/", + "docker exec -t mcs1 chown mysql:mysql -R " + mtr_path, 'docker exec -t mcs1 mariadb -e "create database if not exists test;"', // delay for manual debugging on live instance - 'sleep $${COMPOSE_DELAY_SECONDS:-1s}', - 'docker exec -t mcs1 bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"', + "sleep $${COMPOSE_DELAY_SECONDS:-1s}", + 'docker exec -t mcs1 bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"', ], }, - kind: 'pipeline', - type: 'docker', - name: std.join(' ', [branch, platform, event, arch, server, customBuildEnvCommandsMapKey]), + kind: "pipeline", + type: "docker", + name: std.join(" ", [branch, platform, event, arch, server, customBootstrapParams, customBuildEnvCommandsMapKey]), platform: { arch: arch }, // [if arch == 'arm64' then 'node']: { arch: 'arm64' }, clone: { depth: 10 }, steps: [ { - name: 'submodules', - image: 'alpine/git', + name: "submodules", + image: "alpine/git", commands: [ - 'git submodule update --init --recursive', - 'git config cmake.update-submodules no', - 'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD', + "git submodule update --init --recursive", + "git config cmake.update-submodules no", + "git rev-parse --abbrev-ref HEAD && git rev-parse HEAD", ], }, { - name: 'clone-mdb', - image: 'alpine/git', + name: "clone-mdb", + image: "alpine/git", volumes: [pipeline._volumes.mdb], environment: { - SERVER_REF: '${SERVER_REF:-' + server + '}', - SERVER_REMOTE: '${SERVER_REMOTE:-' + server_remote + '}', - SERVER_SHA: '${SERVER_SHA:-' + server + '}', + SERVER_REF: "${SERVER_REF:-" + server + "}", + SERVER_REMOTE: "${SERVER_REMOTE:-" + server_remote + "}", + SERVER_SHA: "${SERVER_SHA:-" + server + "}", }, commands: [ - 'echo $$SERVER_REF', - 'echo $$SERVER_REMOTE', - 'mkdir -p /mdb/' + builddir + ' && cd /mdb/' + builddir, + "echo $$SERVER_REF", + "echo $$SERVER_REMOTE", + "mkdir -p /mdb/" + builddir + " && cd /mdb/" + builddir, 'git config --global url."https://github.com/".insteadOf git@github.com:', 'git -c submodule."storage/rocksdb/rocksdb".update=none -c submodule."wsrep-lib".update=none -c submodule."storage/columnstore/columnstore".update=none clone --recurse-submodules --depth 200 --branch $$SERVER_REF $$SERVER_REMOTE .', - 'git reset --hard $$SERVER_SHA', - 'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD', - 'git config cmake.update-submodules no', - 'rm -rf storage/columnstore/columnstore', - 'cp -r /drone/src /mdb/' + builddir + '/storage/columnstore/columnstore', + "git reset --hard $$SERVER_SHA", + "git rev-parse --abbrev-ref HEAD && git rev-parse HEAD", + "git config cmake.update-submodules no", + "rm -rf storage/columnstore/columnstore", + "cp -r /drone/src /mdb/" + builddir + "/storage/columnstore/columnstore", ], }, { - name: 'build', - depends_on: ['clone-mdb'], + name: "build", + depends_on: ["clone-mdb"], image: img, volumes: [pipeline._volumes.mdb], environment: { - DEBIAN_FRONTEND: 'noninteractive', - DEB_BUILD_OPTIONS: 'parallel=4', - DH_BUILD_DDEBS: '1', - BUILDPACKAGE_FLAGS: '-b', // Save time and produce only binary packages, not source + DEBIAN_FRONTEND: "noninteractive", + DEB_BUILD_OPTIONS: "parallel=4", + DH_BUILD_DDEBS: "1", + BUILDPACKAGE_FLAGS: "-b", // Save time and produce only binary packages, not source AWS_ACCESS_KEY_ID: { - from_secret: 'aws_access_key_id', + from_secret: "aws_access_key_id", }, AWS_SECRET_ACCESS_KEY: { - from_secret: 'aws_secret_access_key', + from_secret: "aws_secret_access_key", }, - SCCACHE_BUCKET: 'cs-sccache', - SCCACHE_REGION: 'us-east-1', - SCCACHE_S3_USE_SSL: 'true', - SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + '${DRONE_PULL_REQUEST}', + SCCACHE_BUCKET: "cs-sccache", + SCCACHE_REGION: "us-east-1", + SCCACHE_S3_USE_SSL: "true", + SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + "${DRONE_PULL_REQUEST}", //SCCACHE_ERROR_LOG: '/tmp/sccache_log.txt', //SCCACHE_LOG: 'debug', }, commands: [ - 'export CLICOLOR_FORCE=1', - 'mkdir /mdb/' + builddir + '/' + result, - get_sccache] - + (if (std.objectHas(customBuildEnvCommandsMap, customBuildEnvCommandsMapKey)) then - customBuildEnvCommandsMap[customBuildEnvCommandsMapKey] else []) + - [ - 'bash -c "set -o pipefail && bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/bootstrap_mcs.sh ' + - '--build-type RelWithDebInfo ' + - '--distro ' + platform + ' ' + - '--build-packages --sccache ' + - '--server-version ' + server + ' | ' + - '/mdb/' + builddir + '/storage/columnstore/columnstore/build/ansi2txt.sh ' + - '/mdb/' + builddir + '/' + result + '/build.log"' , - 'sccache --show-stats', - ], + "export CLICOLOR_FORCE=1", + "mkdir /mdb/" + builddir + "/" + result, + get_sccache, + ] + + customEnvCommands(customBuildEnvCommandsMapKey, builddir) + + [ + 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + + "--build-type RelWithDebInfo " + + "--distro " + platform + " " + + "--build-packages --sccache " + + "--server-version " + server + + " " + customBootstrapParams + + " " + customBootstrapParamsForExisitingPipelines(platform) + " | " + + "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " + + "/mdb/" + builddir + "/" + result + '/build.log"', + "sccache --show-stats", + ], }, { - name: 'cmapi build', - depends_on: ['clone-mdb'], + name: "cmapi build", + depends_on: ["clone-mdb"], image: img, volumes: [pipeline._volumes.mdb], environment: { - DEBIAN_FRONTEND: 'noninteractive', + DEBIAN_FRONTEND: "noninteractive", }, commands: [ - 'bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/build_cmapi.sh --distro ' + platform + ' --arch ' + arch, + "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/build_cmapi.sh --distro " + platform + " --arch " + arch, ], - }, + }, { - name: 'createrepo', - depends_on: ['build', 'cmapi build'], + name: "createrepo", + depends_on: ["build", "cmapi build"], image: img, when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, volumes: [pipeline._volumes.mdb], commands: [ - 'bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/createrepo.sh --result ' + result, + "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/createrepo.sh --result " + result, ], }, { - name: 'unittests', - depends_on: ['createrepo'], + name: "unittests", + depends_on: ["createrepo"], image: img, volumes: [pipeline._volumes.mdb], environment: { - DEBIAN_FRONTEND: 'noninteractive', + DEBIAN_FRONTEND: "noninteractive", }, commands: [ - 'cd /mdb/' + builddir, + "cd /mdb/" + builddir, testPreparation(platform), testRun(platform), ], }, { - name: 'pkg', - depends_on: ['unittests'], - image: 'alpine/git', + name: "pkg", + depends_on: ["unittests"], + image: "alpine/git", when: { - status: ['success', 'failure'], + status: ["success", "failure"], }, volumes: [pipeline._volumes.mdb], environment: { - SERVER_REF: '${SERVER_REF:-' + server + '}', - SERVER_REMOTE: '${SERVER_REMOTE:-' + server_remote + '}', + SERVER_REF: "${SERVER_REF:-" + server + "}", + SERVER_REMOTE: "${SERVER_REMOTE:-" + server_remote + "}", }, commands: [ - 'cd /mdb/' + builddir, + "cd /mdb/" + builddir, 'echo "engine: $DRONE_COMMIT" > buildinfo.txt', 'echo "server: $$(git rev-parse HEAD)" >> buildinfo.txt', 'echo "buildNo: $DRONE_BUILD_NUMBER" >> buildinfo.txt', @@ -789,46 +815,33 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise', 'echo "serverRepo: $$SERVER_REMOTE" >> buildinfo.txt', 'echo "engineBranch: $DRONE_SOURCE_BRANCH" >> buildinfo.txt', 'echo "engineRepo: https://github.com/$DRONE_REPO" >> buildinfo.txt', - 'mv buildinfo.txt ./%s/' % result, - 'yes | cp -vr ./%s/. /drone/src/%s/' % [result, result], - 'ls -l /drone/src/' + result, + "mv buildinfo.txt ./%s/" % result, + "yes | cp -vr ./%s/. /drone/src/%s/" % [result, result], + "ls -l /drone/src/" + result, 'echo "check columnstore package:"', - 'ls -l /drone/src/%s | grep columnstore' % result, + "ls -l /drone/src/%s | grep columnstore" % result, ], }, ] + - [pipeline.publish('cmapi build')] + + [pipeline.publish("cmapi build")] + [pipeline.publish()] + - [ - { - name: 'publish pkg url', - depends_on: ['publish pkg'], - image: 'alpine/git', - commands: [ - "echo -e '\\e]8;;" + publish_pkg_url + '\\e\\\\' + publish_pkg_url + "\\e]8;;\\e\\\\'", - "echo 'for installation run:'", - "echo 'export OS="+result+"'", - "echo 'export PACKAGES_URL="+packages_url+"'", - ], - }, - ] + - (if (event == 'cron') then [pipeline.publish('pkg latest', 'latest')] else []) + + (if (event == "cron") then [pipeline.publish("pkg latest", "latest")] else []) + [pipeline.smoke] + [pipeline.smokelog] + - [pipeline.publish('smokelog')] + + [pipeline.publish("smokelog")] + [pipeline.cmapitest] + [pipeline.cmapilog] + - [pipeline.publish('cmapilog')] + - (if (platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish('mtrlog')]) + + [pipeline.publish("cmapilog")] + + (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) + [pipeline.prepare_regression] + - [pipeline.regression(regression_tests[i], [if (i == 0) then 'prepare regression' else regression_tests[i - 1]]) for i in indexes(regression_tests)] + + [pipeline.regression(regression_tests[i], [if (i == 0) then "prepare regression" else regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + - [pipeline.publish('regressionlog')] + + [pipeline.publish("regressionlog")] + [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + - (if (std.length(mdb_server_versions) == 0) then [] else [pipeline.upgradelog] + [pipeline.publish('upgradelog')]) + - (if (event == 'cron') then [pipeline.publish('regressionlog latest', 'latest')] else []), + (if (std.length(mdb_server_versions) == 0) then [] else [pipeline.upgradelog] + [pipeline.publish("upgradelog")]) + + (if (event == "cron") then [pipeline.publish("regressionlog latest", "latest")] else []), - volumes: [pipeline._volumes.mdb { temp: {} }, pipeline._volumes.docker { host: { path: '/var/run/docker.sock' } }], + volumes: [pipeline._volumes.mdb { temp: {} }, pipeline._volumes.docker { host: { path: "/var/run/docker.sock" } }], trigger: { event: [event], branch: [branch], @@ -839,18 +852,18 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise', }; local FinalPipeline(branch, event) = { - kind: 'pipeline', - name: std.join(' ', ['after', branch, event]), + kind: "pipeline", + name: std.join(" ", ["after", branch, event]), steps: [ { - name: 'notify', - image: 'plugins/slack', + name: "notify", + image: "plugins/slack", settings: { webhook: { - from_secret: 'slack_webhook', + from_secret: "slack_webhook", }, - template: '*' + event + (if event == 'pull_request' then ' ' else '') + - ' build <{{build.link}}|{{build.number}}> {{#success build.status}}succeeded{{else}}failed{{/success}}*.\n\n*Branch*: \n*Commit*: {{truncate build.message.title 100 }}\n*Author*: {{ build.author }}\n*Duration*: {{since build.started}}\n*Artifacts*: https://cspkg.s3.amazonaws.com/index.html?prefix={{build.branch}}/{{build.event}}/{{build.number}}', + template: "*" + event + (if event == "pull_request" then " " else "") + + " build <{{build.link}}|{{build.number}}> {{#success build.status}}succeeded{{else}}failed{{/success}}*.\n\n*Branch*: \n*Commit*: {{truncate build.message.title 100 }}\n*Author*: {{ build.author }}\n*Duration*: {{since build.started}}\n*Artifacts*: https://cspkg.s3.amazonaws.com/index.html?prefix={{build.branch}}/{{build.event}}/{{build.number}}", }, }, ], @@ -858,23 +871,23 @@ local FinalPipeline(branch, event) = { event: [event], branch: [branch], status: [ - 'success', - 'failure', + "success", + "failure", ], - } + (if event == 'cron' then { cron: ['nightly-' + std.strReplace(branch, '.', '-')] } else {}), - depends_on: std.map(function(p) std.join(' ', [branch, p, event, 'amd64', '10.6-enterprise', '']), platforms.develop) + - std.map(function(p) std.join(' ', [branch, p, event, 'arm64', '10.6-enterprise', '']), platforms_arm.develop), + } + (if event == "cron" then { cron: ["nightly-" + std.strReplace(branch, ".", "-")] } else {}), + depends_on: std.map(function(p) std.join(" ", [branch, p, event, "amd64", "10.6-enterprise", "", ""]), platforms.develop) + + std.map(function(p) std.join(" ", [branch, p, event, "arm64", "10.6-enterprise", "", ""]), platforms_arm.develop), }; [ - Pipeline(b, p, e, 'amd64', s) + Pipeline(b, p, e, "amd64", s) for b in std.objectFields(platforms) for p in platforms[b] for s in servers[b] for e in events ] + [ - Pipeline(b, p, e, 'arm64', s) + Pipeline(b, p, e, "arm64", s) for b in std.objectFields(platforms_arm) for p in platforms_arm[b] for s in servers[b] @@ -882,24 +895,25 @@ local FinalPipeline(branch, event) = { ] + [ - FinalPipeline(b, 'cron') + FinalPipeline(b, "cron") for b in std.objectFields(platforms) ] + [ - Pipeline(any_branch, p, 'custom', 'amd64', '10.6-enterprise') + Pipeline(any_branch, p, "custom", "amd64", "10.6-enterprise") for p in platforms_custom ] + [ - Pipeline(any_branch, p, 'custom', 'arm64', '10.6-enterprise') + Pipeline(any_branch, p, "custom", "arm64", "10.6-enterprise") for p in platforms_arm_custom ] + [ - Pipeline(b, platform, triggeringEvent, 'amd64', server, buildenv) + Pipeline(b, platform, triggeringEvent, a, server, "", buildenv) + for a in ["amd64"] for b in std.objectFields(platforms) - for platform in ['ubuntu:24.04'] - for buildenv in std.objectFields(customBuildEnvCommandsMap) + for platform in ["ubuntu:24.04"] + for buildenv in std.objectFields(customEnvCommandsMap) for triggeringEvent in events for server in servers.develop ] diff --git a/CMakeLists.txt b/CMakeLists.txt index 80f413420..311ab40e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,443 +1,86 @@ -if (CMAKE_VERSION VERSION_LESS 3.13.0) - MESSAGE_ONCE(CS_CMAKE "ColumnStore needs cmake >= 3.13") - RETURN() -ENDIF() +cmake_minimum_required(VERSION 3.13) -CMAKE_MINIMUM_REQUIRED(VERSION 3.13) +project(Columnstore) -PROJECT(Columnstore) -INCLUDE (CheckCXXSourceCompiles) - -# MariaDB server adds this flags by default for Debug builds, and this makes our external projects -# ABI incompatible with debug STL containers -STRING(REPLACE -D_GLIBCXX_DEBUG "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) -STRING(REPLACE -D_GLIBCXX_ASSERTIONS "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) - - -FOREACH(p CMP0135) - IF(POLICY ${p}) - CMAKE_POLICY(SET ${p} NEW) - ENDIF() -ENDFOREACH() - -IF(NOT INSTALL_LAYOUT) - IF(NOT CMAKE_BUILD_TYPE) - SET(CMAKE_BUILD_TYPE RELWITHDEBINFO CACHE STRING - "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) - ENDIF(NOT CMAKE_BUILD_TYPE) - SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -ENDIF() - -if(SERVER_BUILD_DIR) - if (NOT IS_ABSOLUTE ${SERVER_BUILD_DIR}) - set(SERVER_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SERVER_BUILD_DIR}) +foreach(p CMP0135 CMP0077) + if(POLICY ${p}) + cmake_policy(SET ${p} NEW) endif() - if(NOT EXISTS ${SERVER_BUILD_DIR}/CMakeCache.txt) - message(FATAL_ERROR "SERVER_BUILD_DIR parameter supplied but CMakeCache.txt not found in ${SERVER_BUILD_DIR}") - endif() - load_cache("${SERVER_BUILD_DIR}" READ_WITH_PREFIX SERVER_ MySQL_SOURCE_DIR MySQL_BINARY_DIR CMAKE_BUILD_TYPE CMAKE_INSTALL_PREFIX) +endforeach() - set(SERVER_BUILD_INCLUDE_DIR "${SERVER_MySQL_BINARY_DIR}/include" CACHE PATH "Location of server build include folder" FORCE) - set(SERVER_SOURCE_ROOT_DIR "${SERVER_MySQL_SOURCE_DIR}" CACHE PATH "Location of the server source folder" FORCE) - set(CMAKE_INSTALL_PREFIX "${SERVER_CMAKE_INSTALL_PREFIX}" CACHE PATH "Installation prefix" FORCE) - set(CMAKE_BUILD_TYPE ${SERVER_CMAKE_BUILD_TYPE} CACHE STRING "Build configuration type" FORCE) -endif() +set(PACKAGE columnstore) +set(PACKAGE_NAME columnstore) +set(PACKAGE_TARNAME columnstore) +set(PACKAGE_BUGREPORT support@mariadb.com) +set(PACKAGE_URL "") +set(PACKAGE_STRING columnstore-${PACKAGE_VERSION}) -SET_PROPERTY(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external) -LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) +set_property(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) -INCLUDE(check_compiler_flag) -INCLUDE(ExternalProject) -INCLUDE(CheckCXXSourceCompiles) +set(CMAKE_CXX_EXTENSIONS FALSE) +set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) +set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) -FIND_PACKAGE(Boost 1.81.0 COMPONENTS chrono filesystem program_options regex system thread) +# take a note, that install scripts are and services configurations are located oam/install_scripts/CMakeLists.txt -IF (NOT Boost_FOUND OR Boost_VERSION VERSION_GREATER 1.76.99) - MY_CHECK_CXX_COMPILER_FLAG("-std=c++20") -ENDIF() +set(WITH_COLUMNSTORE_LZ4 + AUTO + CACHE STRING "Build with lz4. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'" +) -IF (have_CXX__std_c__20) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20") -ELSE() - MY_CHECK_CXX_COMPILER_FLAG("-std=c++2a") - IF (have_CXX__std_c__2a) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a") - ELSE() - MESSAGE_ONCE(CS_NO_CXX20 "C++ Compiler does not understand -std=c++20") - return() - ENDIF() -ENDIF() +# the order of includes is important +include(check_compiler_flag) +include(ExternalProject) +include(CheckCXXSourceCompiles) +include(packages) +include(boost) +include(thrift) +include(dirs) +include(includes) +include(libs) +include(columnstore_version) +include(configureEngine) +include(compiler_flags) +include(misc) -# There is an inconsistency b/w default char signedness at ARM and x86. -# This flag explicitly sets char as signed. -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsigned-char") +set(COMPONENTS + dbcon/mysql + utils + oam/oamcpp + dbcon/execplan + dbcon/joblist + versioning + oam + writeengine/wrapper + writeengine/client + writeengine/xml + writeengine/redistribute + dbcon/ddlpackage + dbcon/ddlpackageproc + dbcon/dmlpackage + dbcon/dmlpackageproc + ddlproc + dmlproc + oamapps + primitives + tools + writeengine/server + writeengine/bulk + writeengine/splitter + storage-manager + datatypes + tests +) -UNSET(CMAKE_CXX_STANDARD) -SET(CMAKE_CXX_EXTENSIONS FALSE) -SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) -SET(CMAKE_POSITION_INDEPENDENT_CODE TRUE) -SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) -SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) -SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) -SET(WITH_COLUMNSTORE_LZ4 AUTO CACHE STRING "Build with lz4. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") +foreach(component ${COMPONENTS}) + add_subdirectory(${component}) +endforeach() -SET (ENGINE_SYSCONFDIR "/etc") -SET (ENGINE_DATADIR "/var/lib/columnstore") -SET (ENGINE_LOGDIR "/var/log/mariadb/columnstore") - -IF (INSTALL_LAYOUT) - # We are building from MariaDB server submodule if this is set - SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR}") - IF (INSTALL_SYSCONF2DIR) - SET(MARIADB_MYCNFDIR "${INSTALL_SYSCONF2DIR}") - ELSE () - SET(MARIADB_MYCNFDIR "/etc/mysql") - ENDIF () - SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR}") - SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR}") - SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") - SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR}") - SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR}") - SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") -ELSE () - IF (RPM) - SET(MARIADB_PLUGINDIR "/usr/lib64/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/my.cnf.d") - SET (ENGINE_LIBDIR "/usr/lib64") - SET (ENGINE_BINDIR "/usr/bin") - SET (ENGINE_INCDIR "/usr/include") - SET (ENGINE_MANDIR "/usr/share/man") - SET (ENGINE_SBINDIR "/usr/sbin") - SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") - ELSEIF (DEB) - SET(MARIADB_PLUGINDIR "/usr/lib/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/mysql/conf.d") - SET (ENGINE_LIBDIR "/usr/lib") - SET (ENGINE_BINDIR "/usr/bin") - SET (ENGINE_INCDIR "/usr/include") - SET (ENGINE_MANDIR "/usr/share/man") - SET (ENGINE_SBINDIR "/usr/sbin") - SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") - ELSE () - # TODO: prefix should probably apply here - SET(MARIADB_PLUGINDIR "/usr/local/lib/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/mysql") - SET (ENGINE_LIBDIR "/usr/local/lib") - SET (ENGINE_BINDIR "/usr/local/bin") - SET (ENGINE_INCDIR "/usr/local/include") - SET (ENGINE_MANDIR "/usr/local/man") - SET (ENGINE_SBINDIR "/usr/local/sbin") - SET (ENGINE_SUPPORTDIR "/usr/local/share/columnstore") - ENDIF () -ENDIF () - -SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - -INCLUDE(columnstore_version) -INCLUDE(misc) -INCLUDE(boost) -INCLUDE(thrift) - -FIND_PACKAGE(BISON) -IF (NOT BISON_FOUND) - MESSAGE_ONCE(CS_NO_BISON "bison not found!") - return() -ENDIF() - -check_cxx_source_compiles("#include \n void main(){}" HAS_STD_FILESYSTEM) -check_cxx_source_compiles("#include \n void main(){}" HAS_STD_EXPERIMENTAL_FILESYSTEM) - -SET (PACKAGE columnstore) -SET (PACKAGE_NAME columnstore) -SET (PACKAGE_TARNAME columnstore) -SET (PACKAGE_BUGREPORT support@mariadb.com) -SET (PACKAGE_URL "") - -SET (PACKAGE_STRING columnstore-${PACKAGE_VERSION}) - -INCLUDE (configureEngine) - -FIND_PROGRAM(LEX_EXECUTABLE flex DOC "path to the flex executable") -if(NOT LEX_EXECUTABLE) - FIND_PROGRAM(LEX_EXECUTABLE lex DOC "path to the lex executable") - if(NOT LEX_EXECUTABLE) - MESSAGE_ONCE(CS_NO_LEX "flex/lex not found!") - return() - endif() -endif() - -FIND_PACKAGE(LibXml2) - -if (NOT LIBXML2_FOUND) - MESSAGE_ONCE(CS_NO_LIBXML "Could not find a usable libxml2 development environment!") - return() -endif() - -find_package(Snappy) -if (NOT SNAPPY_FOUND) - MESSAGE_ONCE(CS_NO_SNAPPY "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") - return() -endif() - -FIND_PACKAGE(CURL) -if (NOT CURL_FOUND) - MESSAGE_ONCE(CS_NO_CURL "libcurl development headers not found") - return() -endif() - -FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") -if(NOT AWK_EXECUTABLE) - MESSAGE_ONCE(CS_NO_AWK "awk not found!") - return() -endif() - -SET(HAVE_LZ4 0 CACHE INTERNAL "") -IF (WITH_COLUMNSTORE_LZ4 STREQUAL "ON" OR WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") - FIND_PACKAGE(LZ4) - IF (NOT LZ4_FOUND) - IF (WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") - MESSAGE_ONCE(CS_LZ4 "LZ4 not found, building without LZ4") - ELSE() - MESSAGE(FATAL_ERROR "LZ4 not found.") - ENDIF() - ELSE() - MESSAGE_ONCE(CS_LZ4 "Building with LZ4") - SET(HAVE_LZ4 1 CACHE INTERNAL "") - ENDIF() -ELSE() - MESSAGE_ONCE(CS_LZ4 "Building without LZ4") -ENDIF() - -IF (NOT INSTALL_LAYOUT) - MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) - MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-DBOOST_BIND_GLOBAL_PLACEHOLDERS" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) - - # enable security hardening features, like most distributions do - # in our benchmarks that costs about ~1% of performance, depending on the load - IF(CMAKE_C_COMPILER_VERSION VERSION_LESS "4.6") - SET(security_default OFF) - ELSE() - SET(security_default ON) - ENDIF() - OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default}) - OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF) - IF(SECURITY_HARDENED AND NOT WITH_ASAN AND NOT WITH_UBSAN AND NOT WITH_TSAN AND NOT WITH_GPROF) - # security-enhancing flags - MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector --param=ssp-buffer-size=4") - MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO) - MY_CHECK_AND_SET_COMPILER_FLAG("-fexceptions") - IF(SECURITY_HARDENED_NEW) - MY_CHECK_AND_SET_COMPILER_FLAG("-mcet -fcf-protection") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector-strong") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-clash-protection") - ENDIF() - ENDIF() -ELSE () - # Remove visibility flag for now as it breaks Ubuntu 18.05 and we need to - # fix our libraries anyway - STRING(REPLACE "-fvisibility=hidden" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-D_DEBUG -O0" DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-vla" DEBUG) -ENDIF() - -IF (MASK_LONGDOUBLE) - MY_CHECK_AND_SET_COMPILER_FLAG("-DMASK_LONGDOUBLE") -ENDIF() - -if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-enum-enum-conversion -Wno-register -Wno-typedef-redefinition -Wno-cast-function-type-strict") -elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-register") -endif() - -IF (WITH_COLUMNSTORE_ASAN) - MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) - MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") -ENDIF() - -MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-copy" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) -MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-declarations" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) -MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-format-truncation" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) -MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall -Wextra") - -SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") -SET (ENGINE_DT_LIB datatypes) -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot boost_thread xml2 pthread rt ${ENGINE_DT_LIB}) -SET (ENGINE_OAM_LIBS oamcpp) -SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) -SET (PLUGIN_EXEC_LIBS execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querytele threadpool ${ENGINE_BRM_LIBS}) -SET (ENGINE_EXEC_LIBS joblist querystats libmysql_client ${PLUGIN_EXEC_LIBS}) -SET (PLUGIN_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage writeengine writeengineclient idbdatafile cacheutils) -SET (ENGINE_WRITE_LIBS ${PLUGIN_WRITE_LIBS} ${ENGINE_EXEC_LIBS}) - -IF (NOT INSTALL_LAYOUT) - SET (LIBMARIADB_BININC_DIR ${SERVER_BUILD_DIR}/libmariadb/include) - SET (LIBMARIADB_SRCINC_DIR ${SERVER_SOURCE_ROOT_DIR}/libmariadb/include) -ELSE() - SET (LIBMARIADB_BININC_DIR ${CMAKE_BINARY_DIR}/libmariadb/include) - SET (LIBMARIADB_SRCINC_DIR ${CMAKE_SOURCE_DIR}/libmariadb/include) -ENDIF() - -SET (ENGINE_COMMON_LDFLAGS "") - -IF (SERVER_BUILD_INCLUDE_DIR) - IF (NOT IS_ABSOLUTE ${SERVER_BUILD_INCLUDE_DIR}) - SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR}) - ENDIF() -ELSEIF (INSTALL_LAYOUT) - SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include) -ELSE() - SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include) -ENDIF() - -IF (SERVER_SOURCE_ROOT_DIR) - IF (NOT IS_ABSOLUTE ${SERVER_SOURCE_ROOT_DIR}) - SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR}) - ENDIF() -ELSEIF (INSTALL_LAYOUT) - SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR}) -ELSE() - SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..) -ENDIF() - -IF (NOT SERVER_BUILD_DIR) - SET (SERVER_BUILD_DIR ${SERVER_SOURCE_ROOT_DIR}) -ENDIF() - -MESSAGE_ONCE(SERVER_BUILD_INCLUDE_DIR "SERVER_BUILD_INCLUDE_DIR = ${SERVER_BUILD_INCLUDE_DIR}") -MESSAGE_ONCE(SERVER_SOURCE_ROOT_DIR "SERVER_SOURCE_ROOT_DIR = ${SERVER_SOURCE_ROOT_DIR}") - -IF (INSTALL_LAYOUT) - SET (MARIADB_CLIENT_LIBS libmariadb) -ELSE() - SET (MARIADB_CLIENT_LIBS -L${SERVER_BUILD_INCLUDE_DIR}/../libmariadb/libmariadb/ libmariadb.so) -ENDIF() - -IF (INSTALL_LAYOUT) - SET (MARIADB_STRING_LIBS dbug strings mysys) -ELSE() - SET (MARIADB_STRING_LIBS -L${SERVER_BUILD_INCLUDE_DIR}/../mysys/ libmysys.a -L${SERVER_BUILD_INCLUDE_DIR}/../strings/ libstrings.a -L${SERVER_BUILD_INCLUDE_DIR}/../dbug/ libdbug.a) -ENDIF() - -#SET (ENGINE_UTILS_BOOSTIDB_INCLUDE "{CMAKE_CURRENT_SOURCE_DIR}/utils/boost_idb") -SET (ENGINE_UTILS_MESSAGEQCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/messageqcpp") -SET (ENGINE_WE_SHARED_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/shared") -SET (ENGINE_UTILS_IDBDATAFILE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/idbdatafile") -SET (ENGINE_UTILS_LOGGINGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/loggingcpp" "${CMAKE_CURRENT_BINARY_DIR}/utils/loggingcpp") -SET (ENGINE_UTILS_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/configcpp") -SET (ENGINE_UTILS_COMPRESS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/compress") -SET (ENGINE_VERSIONING_BRM_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/versioning/BRM") -SET (ENGINE_UTILS_ROWGROUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rowgroup") -SET (ENGINE_UTILS_COMMON_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/common") -SET (ENGINE_UTILS_DATACONVERT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/dataconvert") -SET (ENGINE_UTILS_RWLOCK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rwlock") -SET (ENGINE_UTILS_FUNCEXP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/funcexp") -SET (ENGINE_UTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils") -SET (ENGINE_OAM_OAMCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/oam/oamcpp") -SET (ENGINE_DBCON_DDLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackageproc") -SET (ENGINE_DBCON_DDLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackage") -SET (ENGINE_DBCON_EXECPLAN_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/execplan") -SET (ENGINE_UTILS_STARTUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/startup") -SET (ENGINE_DBCON_JOBLIST_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/joblist") -SET (ENGINE_WE_WRAPPER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/wrapper") -SET (ENGINE_WE_SERVER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/server") -SET (ENGINE_DBCON_DMLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackage") -SET (ENGINE_WE_CLIENT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/client") -SET (ENGINE_DBCON_DMLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackageproc") -SET (ENGINE_UTILS_CACHEUTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/cacheutils") -SET (ENGINE_UTILS_MYSQLCL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/mysqlcl_idb") -SET (ENGINE_UTILS_QUERYTELE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querytele") -SET (ENGINE_UTILS_JOINER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/joiner") -SET (ENGINE_UTILS_THREADPOOL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/threadpool") -SET (ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchloader") -SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup") -SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") -SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") -SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") -SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") -SET (ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache") -SET (ENGINE_PRIMPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/primproc") -SET (ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") -SET (ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") -IF (PCRE_INCLUDES) - SET (ENGINE_SERVER_PCRE_INCLUDE "${PCRE_INCLUDES}") -ELSE () - SET (ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre") -ENDIF () -SET (ENGINE_SERVER_WSREP_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/include") -SET (ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/wsrep-API/v26/") -SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk") - -SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) - -SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) - -ADD_SUBDIRECTORY(dbcon/mysql) -IF(NOT TARGET columnstore) - RETURN() -ENDIF() - -SET (SHMEM_FILE_GLOB "MCS-shm-") - -ADD_SUBDIRECTORY(utils) -ADD_SUBDIRECTORY(oam/oamcpp) -ADD_SUBDIRECTORY(dbcon/execplan) -ADD_SUBDIRECTORY(dbcon/joblist) -ADD_SUBDIRECTORY(versioning) -ADD_SUBDIRECTORY(oam) -ADD_SUBDIRECTORY(writeengine/wrapper) -ADD_SUBDIRECTORY(writeengine/client) -ADD_SUBDIRECTORY(writeengine/xml) -ADD_SUBDIRECTORY(writeengine/redistribute) -ADD_SUBDIRECTORY(dbcon/ddlpackage) -ADD_SUBDIRECTORY(dbcon/ddlpackageproc) -ADD_SUBDIRECTORY(dbcon/dmlpackage) -ADD_SUBDIRECTORY(dbcon/dmlpackageproc) -ADD_SUBDIRECTORY(ddlproc) -ADD_SUBDIRECTORY(dmlproc) -ADD_SUBDIRECTORY(oamapps) -ADD_SUBDIRECTORY(primitives) -ADD_SUBDIRECTORY(tools) -ADD_SUBDIRECTORY(writeengine/server) -ADD_SUBDIRECTORY(writeengine/bulk) -ADD_SUBDIRECTORY(writeengine/splitter) -ADD_SUBDIRECTORY(storage-manager) -ADD_SUBDIRECTORY(datatypes) -ADD_SUBDIRECTORY(tests) - -find_package(Git QUIET) - -IF (GIT_FOUND AND EXISTS ${ENGINE_SRC_DIR}/.git) - exec_program("git" - ${CMAKE_CURRENT_SOURCE_DIR} - ARGS "describe --match=NeVeRmAtCh --always --dirty" - OUTPUT_VARIABLE GIT_VERSION) -ELSE () - SET(GIT_VERSION "source") -ENDIF () - -# releasenum is used by external scripts for various tasks. Leave it alone. -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h) - -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) - -IF (INSTALL_LAYOUT) - # Do this or when MariaDB builds us we don't have GenError which is required for these - ADD_DEPENDENCIES(udf_mysql GenError) - ADD_DEPENDENCIES(funcexp GenError) - ADD_DEPENDENCIES(oamcpp GenError) - ADD_DEPENDENCIES(regr_mysql GenError) -ELSE () - # MariaDB has its own packaging routines - INCLUDE(cpackEngineRPM) - INCLUDE(cpackEngineDEB) -ENDIF () +add_dependencies(udf_mysql GenError) +add_dependencies(funcexp GenError) +add_dependencies(oamcpp GenError) +add_dependencies(regr_mysql GenError) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index ed6239c33..fa8180b40 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -38,33 +38,33 @@ cd - >/dev/null echo "Arguments received: $@" message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" -optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE +optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true +optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default="$MDB_SOURCE_PATH"/../MariaDBBuild +optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true +optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true +optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true optparse.define short=d long=distro desc="Choose your OS: ${DISTRO_OPTIONS[*]}" variable=OS optparse.define short=D long=install-deps desc="Install dependences" variable=INSTALL_DEPS default=false value=true -optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true -optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true -optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true -optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true -optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true -optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true -optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true -optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true -optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true -optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" -optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true -optparse.define short=G long=draw-deps desc="Draw dependencies graph" variable=DRAW_DEPS default=false value=true -optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true -optparse.define short=n long=no-clean-install desc="Do not perform a clean install (keep existing db files)" variable=NO_CLEAN default=false value=true -optparse.define short=j long=parallel desc="Number of paralles for build" variable=CPUS default=$(getconf _NPROCESSORS_ONLN) -optparse.define short=F long=show-build-flags desc="Print CMake flags, while build" variable=PRINT_CMAKE_FLAGS default=false value=true -optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true +optparse.define short=F long=custom-cmake-flags desc="Add custom cmake flags" variable=CUSTOM_CMAKE_FLAGS optparse.define short=f long=do-not-freeze-revision desc="Disable revision freezing, or do not set 'update none' for columnstore submodule in MDB repository" variable=DO_NOT_FREEZE_REVISION default=false value=true -optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default="$MDB_SOURCE_PATH"/../MariaDBBuild +optparse.define short=G long=draw-deps desc="Draw dependencies graph" variable=DRAW_DEPS default=false value=true +optparse.define short=j long=parallel desc="Number of paralles for build" variable=CPUS default=$(getconf _NPROCESSORS_ONLN) +optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true +optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true +optparse.define short=n long=no-clean-install desc="Do not perform a clean install (keep existing db files)" variable=NO_CLEAN default=false value=true optparse.define short=o long=recompile-only variable=RECOMPILE_ONLY default=false value=true -optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false -optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE default=false value=true optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true +optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" +optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false optparse.define short=R long=server-version desc="MariaDB server version" variable=MARIADB_SERVER_VERSION +optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE default=false value=true +optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true +optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE +optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true +optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true +optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true +optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true +optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true source $(optparse.build) @@ -263,6 +263,7 @@ construct_cmake_flags() { -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE -DPLUGIN_GSSAPI=NO -DMYSQL_MAINTAINER_MODE=NO + -DCOLUMNSTORE_MAINTAINER=YES ) if [[ $SKIP_UNIT_TESTS = true ]]; then @@ -362,10 +363,10 @@ construct_cmake_flags() { MDB_CMAKE_FLAGS+=(-DDEB=${CODENAME}) fi - if [[ $PRINT_CMAKE_FLAGS = true ]]; then - message "Building with flags" - newline_array "${MDB_CMAKE_FLAGS[@]}" - fi + MDB_CMAKE_FLAGS+=($CUSTOM_CMAKE_FLAGS) + + message "Building with flags" + newline_array "${MDB_CMAKE_FLAGS[@]}" } init_submodules() { diff --git a/build/update-clang-version.sh b/build/update-clang-version.sh new file mode 100644 index 000000000..4528561dc --- /dev/null +++ b/build/update-clang-version.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +function register_clang_version() { + local version=$1 + local priority=$2 + + update-alternatives \ + --verbose \ + --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \ + --slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \ + --slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \ + --slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \ + --slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \ + --slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \ + --slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \ + --slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \ + --slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \ + --slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \ + --slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \ + --slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \ + --slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \ + --slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \ + --slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \ + --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \ + --slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \ + --slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \ + --slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \ + --slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \ + --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \ + --slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \ + --slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \ + --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \ + --slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \ + --slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version} + + update-alternatives \ + --verbose \ + --install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \ + --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \ + --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \ + --slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \ + --slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \ + --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \ + --slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \ + --slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \ + --slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \ + --slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \ + --slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \ + --slave /usr/bin/lld lld /usr/bin/lld-${version} \ + --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \ + --slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \ + --slave /usr/bin/llc llc /usr/bin/llc-${version} \ + --slave /usr/bin/lli lli /usr/bin/lli-${version} \ + --slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \ + --slave /usr/bin/opt opt /usr/bin/opt-${version} \ + --slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \ + --slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \ + --slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \ + --slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version} + +} + +register_clang_version $1 $2 diff --git a/cmake-format.py b/cmake-format.py new file mode 100644 index 000000000..2d232d51b --- /dev/null +++ b/cmake-format.py @@ -0,0 +1,241 @@ +# ---------------------------------- +# Options affecting listfile parsing +# ---------------------------------- +with section("parse"): + + # Specify structure for custom cmake functions + additional_commands = { 'foo': { 'flags': ['BAR', 'BAZ'], + 'kwargs': {'DEPENDS': '*', 'HEADERS': '*', 'SOURCES': '*'}}} + + # Override configurations per-command where available + override_spec = {} + + # Specify variable tags. + vartags = [] + + # Specify property tags. + proptags = [] + +# ----------------------------- +# Options affecting formatting. +# ----------------------------- +with section("format"): + + # Disable formatting entirely, making cmake-format a no-op + disable = False + + # How wide to allow formatted cmake files + line_width = 120 + + # How many spaces to tab for indent + tab_size = 4 + + # If true, lines are indented using tab characters (utf-8 0x09) instead of + # space characters (utf-8 0x20). In cases where the layout would + # require a fractional tab character, the behavior of the fractional + # indentation is governed by + use_tabchars = False + + # If is True, then the value of this variable indicates how + # fractional indentions are handled during whitespace replacement. If set to + # 'use-space', fractional indentation is left as spaces (utf-8 0x20). If set + # to `round-up` fractional indentation is replaced with a single tab character + # (utf-8 0x09) effectively shifting the column to the next tabstop + fractional_tab_policy = 'use-space' + + # If an argument group contains more than this many sub-groups (parg or kwarg + # groups) then force it to a vertical layout. + max_subgroups_hwrap = 2 + + # If a positional argument group contains more than this many arguments, then + # force it to a vertical layout. + max_pargs_hwrap = 6 + + # If a cmdline positional group consumes more than this many lines without + # nesting, then invalidate the layout (and nest) + max_rows_cmdline = 2 + + # If true, separate flow control names from their parentheses with a space + separate_ctrl_name_with_space = False + + # If true, separate function names from parentheses with a space + separate_fn_name_with_space = False + + # If a statement is wrapped to more than one line, than dangle the closing + # parenthesis on its own line. + dangle_parens = True + + # If the trailing parenthesis must be 'dangled' on its on line, then align it + # to this reference: `prefix`: the start of the statement, `prefix-indent`: + # the start of the statement, plus one indentation level, `child`: align to + # the column of the arguments + dangle_align = 'prefix' + + # If the statement spelling length (including space and parenthesis) is + # smaller than this amount, then force reject nested layouts. + min_prefix_chars = 4 + + # If the statement spelling length (including space and parenthesis) is larger + # than the tab width by more than this amount, then force reject un-nested + # layouts. + max_prefix_chars = 10 + + # If a candidate layout is wrapped horizontally but it exceeds this many + # lines, then reject the layout. + max_lines_hwrap = 2 + + # What style line endings to use in the output. + line_ending = 'unix' + + # Format command names consistently as 'lower' or 'upper' case + command_case = 'canonical' + + # Format keywords consistently as 'lower' or 'upper' case + keyword_case = 'unchanged' + + # A list of command names which should always be wrapped + always_wrap = [] + + # If true, the argument lists which are known to be sortable will be sorted + # lexicographicall + enable_sort = True + + # If true, the parsers may infer whether or not an argument list is sortable + # (without annotation). + autosort = False + + # By default, if cmake-format cannot successfully fit everything into the + # desired linewidth it will apply the last, most aggressive attempt that it + # made. If this flag is True, however, cmake-format will print error, exit + # with non-zero status code, and write-out nothing + require_valid_layout = False + + # A dictionary mapping layout nodes to a list of wrap decisions. See the + # documentation for more information. + layout_passes = {} + +# ------------------------------------------------ +# Options affecting comment reflow and formatting. +# ------------------------------------------------ +with section("markup"): + + # What character to use for bulleted lists + bullet_char = '*' + + # What character to use as punctuation after numerals in an enumerated list + enum_char = '.' + + # If comment markup is enabled, don't reflow the first comment block in each + # listfile. Use this to preserve formatting of your copyright/license + # statements. + first_comment_is_literal = False + + # If comment markup is enabled, don't reflow any comment block which matches + # this (regex) pattern. Default is `None` (disabled). + literal_comment_pattern = None + + # Regular expression to match preformat fences in comments default= + # ``r'^\s*([`~]{3}[`~]*)(.*)$'`` + fence_pattern = '^\\s*([`~]{3}[`~]*)(.*)$' + + # Regular expression to match rulers in comments default= + # ``r'^\s*[^\w\s]{3}.*[^\w\s]{3}$'`` + ruler_pattern = '^\\s*[^\\w\\s]{3}.*[^\\w\\s]{3}$' + + # If a comment line matches starts with this pattern then it is explicitly a + # trailing comment for the preceding argument. Default is '#<' + explicit_trailing_pattern = '#<' + + # If a comment line starts with at least this many consecutive hash + # characters, then don't lstrip() them off. This allows for lazy hash rulers + # where the first hash char is not separated by space + hashruler_min_length = 10 + + # If true, then insert a space between the first hash char and remaining hash + # chars in a hash ruler, and normalize its length to fill the column + canonicalize_hashrulers = True + + # enable comment markup parsing and reflow + enable_markup = True + +# ---------------------------- +# Options affecting the linter +# ---------------------------- +with section("lint"): + + # a list of lint codes to disable + disabled_codes = [] + + # regular expression pattern describing valid function names + function_pattern = '[0-9a-z_]+' + + # regular expression pattern describing valid macro names + macro_pattern = '[0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with global + # (cache) scope + global_var_pattern = '[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with global + # scope (but internal semantic) + internal_var_pattern = '_[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with local + # scope + local_var_pattern = '[a-z][a-z0-9_]+' + + # regular expression pattern describing valid names for privatedirectory + # variables + private_var_pattern = '_[0-9a-z_]+' + + # regular expression pattern describing valid names for public directory + # variables + public_var_pattern = '[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for function/macro + # arguments and loop variables. + argument_var_pattern = '[a-z][a-z0-9_]+' + + # regular expression pattern describing valid names for keywords used in + # functions or macros + keyword_pattern = '[A-Z][0-9A-Z_]+' + + # In the heuristic for C0201, how many conditionals to match within a loop in + # before considering the loop a parser. + max_conditionals_custom_parser = 2 + + # Require at least this many newlines between statements + min_statement_spacing = 1 + + # Require no more than this many newlines between statements + max_statement_spacing = 2 + max_returns = 6 + max_branches = 12 + max_arguments = 5 + max_localvars = 15 + max_statements = 50 + +# ------------------------------- +# Options affecting file encoding +# ------------------------------- +with section("encode"): + + # If true, emit the unicode byte-order mark (BOM) at the start of the file + emit_byteorder_mark = False + + # Specify the encoding of the input file. Defaults to utf-8 + input_encoding = 'utf-8' + + # Specify the encoding of the output file. Defaults to utf-8. Note that cmake + # only claims to support utf-8 so be careful when using anything else + output_encoding = 'utf-8' + +# ------------------------------------- +# Miscellaneous configurations options. +# ------------------------------------- +with section("misc"): + + # A dictionary containing any per-command configuration overrides. Currently + # only `command_case` is supported. + per_command = {} + diff --git a/cmake/boost.cmake b/cmake/boost.cmake index 31c40f19b..c166de348 100644 --- a/cmake/boost.cmake +++ b/cmake/boost.cmake @@ -1,51 +1,56 @@ -if (Boost_FOUND) - add_custom_target(external_boost) - return() +find_package(Boost 1.84.0 COMPONENTS chrono filesystem program_options regex system thread) + +if(Boost_FOUND) + add_custom_target(external_boost) + return() endif() include(ExternalProject) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(_toolset "gcc") - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0") - set(_extra "pch=off") - endif() + set(_toolset "gcc") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0") + set(_extra "pch=off") + endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") - set(_toolset "clang") + set(_toolset "clang") elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") - set(_toolset "intel-linux") + set(_toolset "intel-linux") endif() set(INSTALL_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/.boost/boost-lib) -SET(Boost_INCLUDE_DIRS "${INSTALL_LOCATION}/include") -SET(Boost_LIBRARY_DIRS "${INSTALL_LOCATION}/lib") -LINK_DIRECTORIES("${Boost_LIBRARY_DIRS}") +set(BOOST_ROOT "${INSTALL_LOCATION}") +set(Boost_INCLUDE_DIRS "${INSTALL_LOCATION}/include") +set(Boost_LIBRARY_DIRS "${INSTALL_LOCATION}/lib") +link_directories("${Boost_LIBRARY_DIRS}") set(_cxxargs "-fPIC -DBOOST_NO_AUTO_PTR -fvisibility=default") -set(_b2args cxxflags=${_cxxargs};cflags=-fPIC;threading=multi;${_extra};toolset=${_toolset} --without-python;--prefix=${INSTALL_LOCATION}) - -SET(byproducts) -FOREACH(name chrono filesystem program_options regex system thread) - SET(lib boost_${name}) - ADD_LIBRARY(${lib} STATIC IMPORTED GLOBAL) - ADD_DEPENDENCIES(${lib} external_boost) - SET (loc "${Boost_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}${lib}${CMAKE_STATIC_LIBRARY_SUFFIX}") - SET(byproducts ${byproducts} BUILD_BYPRODUCTS ${loc}) - SET_TARGET_PROPERTIES(${lib} PROPERTIES IMPORTED_LOCATION ${loc} - EXCLUDE_FROM_ALL TRUE) -ENDFOREACH() - -ExternalProject_Add(external_boost - PREFIX .boost - URL https://archives.boost.io/release/1.84.0/source/boost_1_84_0.tar.gz - URL_HASH SHA256=a5800f405508f5df8114558ca9855d2640a2de8f0445f051fa1c7c3383045724 - CONFIGURE_COMMAND ./bootstrap.sh - UPDATE_COMMAND "" - BUILD_COMMAND ./b2 -q ${_b2args} - BUILD_IN_SOURCE TRUE - INSTALL_COMMAND ./b2 -q install ${_b2args} - LOG_BUILD TRUE - LOG_INSTALL TRUE - EXCLUDE_FROM_ALL TRUE - ${byproducts} +set(_b2args cxxflags=${_cxxargs};cflags=-fPIC;threading=multi;${_extra};toolset=${_toolset} + --without-python;--prefix=${INSTALL_LOCATION} +) + +set(byproducts) +foreach(name chrono filesystem program_options regex system thread) + set(lib boost_${name}) + add_library(${lib} STATIC IMPORTED GLOBAL) + add_dependencies(${lib} external_boost) + set(loc "${Boost_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}${lib}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(byproducts ${byproducts} BUILD_BYPRODUCTS ${loc}) + set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION ${loc} EXCLUDE_FROM_ALL TRUE) +endforeach() + +ExternalProject_Add( + external_boost + PREFIX .boost + URL https://archives.boost.io/release/1.84.0/source/boost_1_84_0.tar.gz + URL_HASH SHA256=a5800f405508f5df8114558ca9855d2640a2de8f0445f051fa1c7c3383045724 + CONFIGURE_COMMAND ./bootstrap.sh + UPDATE_COMMAND "" + BUILD_COMMAND ./b2 -q ${_b2args} + BUILD_IN_SOURCE TRUE + INSTALL_COMMAND ./b2 -q install ${_b2args} + LOG_BUILD TRUE + LOG_INSTALL TRUE + EXCLUDE_FROM_ALL TRUE + ${byproducts} ) diff --git a/cmake/columnstore_version.cmake b/cmake/columnstore_version.cmake index 8162fe506..cb9f85086 100644 --- a/cmake/columnstore_version.cmake +++ b/cmake/columnstore_version.cmake @@ -40,11 +40,6 @@ IF(NOT "${CS_MAJOR_VERSION}" MATCHES "[0-9]+" OR SET(VERSION "${CS_MAJOR_VERSION}.${CS_MINOR_VERSION}.${CS_PATCH_VERSION}${CS_EXTRA_VERSION}") MESSAGE("== MariaDB-Columnstore ${VERSION}") - IF (NOT INSTALL_LAYOUT) - SET(CPACK_PACKAGE_VERSION_MAJOR ${CS_MAJOR_VERSION}) - SET(CPACK_PACKAGE_VERSION_MINOR ${CS_MINOR_VERSION}) - SET(CPACK_PACKAGE_VERSION_PATCH ${CS_PATCH_VERSION}${CS_EXTRA_VERSION}) - ENDIF () SET(PACKAGE_VERSION "${CS_MAJOR_VERSION}.${CS_MINOR_VERSION}.${CS_PATCH_VERSION}${CS_EXTRA_VERSION}") SET(PACKAGE_RELEASE "${CS_RELEASE_VERSION}") MATH(EXPR MCSVERSIONHEX "${CS_MAJOR_VERSION} * 256 + ${CS_MINOR_VERSION}" OUTPUT_FORMAT HEXADECIMAL) diff --git a/cmake/compiler_flags.cmake b/cmake/compiler_flags.cmake new file mode 100644 index 000000000..37f795ef1 --- /dev/null +++ b/cmake/compiler_flags.cmake @@ -0,0 +1,109 @@ +macro(SET_FLAGS) + foreach(F ${ARGV}) + my_check_and_set_compiler_flag(${F} DEBUG RELWITHDEBINFO MINSIZEREL) + endforeach() +endmacro() + +macro(SET_FLAGS_DEBUG) + foreach(F ${ARGV}) + my_check_and_set_compiler_flag(${F} DEBUG) + endforeach() +endmacro() + +macro(SET_FLAGS_RELEASE) + foreach(F ${ARGV}) + my_check_and_set_compiler_flag(${F} RELWITHDEBINFO) + endforeach() +endmacro() + +# C++ standard { +if(have_CXX__std_c__20) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20") +else() + my_check_cxx_compiler_flag("-std=c++2a") + if(have_CXX__std_c__2a) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a") + else() + message_once(CS_NO_CXX20 "C++ Compiler does not understand -std=c++20") + return() + endif() +endif() + +unset(CMAKE_CXX_STANDARD) +# } end C++ standard + +# Hacks to keep alive with MariaDB server { +string(REPLACE -D_GLIBCXX_DEBUG "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) +string(REPLACE -D_GLIBCXX_ASSERTIONS "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) +# } end hacks + +# Maintainer flags, works when build is done via bootstrap_mcs.sh { +set(COLUMNSTORE_MAINTAINER_FLAGS -Werror) +# } end Maintainer flags + +# Release, Debug and common flags { +set(FLAGS_ALL + -Wall + -Wextra + -fno-omit-frame-pointer + -fno-strict-aliasing + -fsigned-char + -msse4.2 + -DHAVE_CONFIG_H + -DBOOST_BIND_GLOBAL_PLACEHOLDERS +) + +set(FLAGS_RELEASE -g -O3 -DDBUG_OFF) + +set(FLAGS_DEBUG -ggdb3 -O0 -D_DEBUG) +# } end Release, Debug and common flags + +# linker flags { +set(ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") +# } end linker flags + +# compiler specific flags { +set(CLANG_FLAGS + # suppressed warnings + -Wno-cast-function-type-strict + -Wno-deprecated-copy + -Wno-deprecated-declarations + -Wno-deprecated-enum-enum-conversion + -Wno-format-truncation + -Wno-register + -Wno-typedef-redefinition +) + +set(GNU_FLAGS # suppressed warnings + -Wno-deprecated-copy -Wno-deprecated-declarations -Wno-format-truncation -Wno-register -Wno-unused-variable +) +# } end compiler specific flags + +# Sanitizers { +set(ASAN_FLAGS -U_FORTIFY_SOURCE -fsanitize=address -fsanitize-address-use-after-scope -fPIC) +# } end Sanitizers + +# configured by cmake/configureEngine.cmake { +if(MASK_LONGDOUBLE) + my_check_and_set_compiler_flag("-DMASK_LONGDOUBLE") +endif() +# } end configured by cmake/configureEngine.cmake + +# apply them all +set_flags(${FLAGS_ALL}) +set_flags_debug(${FLAGS_DEBUG}) +set_flags_release(${FLAGS_RELEASE}) + +if(COLUMNSTORE_MAINTAINER) + set_flags(${COLUMNSTORE_MAINTAINER_FLAGS}) +endif() + +if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set_flags(${CLANG_FLAGS}) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set_flags(${GNU_FLAGS}) +endif() + +if(WITH_COLUMNSTORE_ASAN) + set_flags(${ASAN_FLAGS}) +endif() diff --git a/cmake/configureEngine.cmake b/cmake/configureEngine.cmake index 4de0f2946..560166d7f 100644 --- a/cmake/configureEngine.cmake +++ b/cmake/configureEngine.cmake @@ -1,147 +1,143 @@ - - # # Tests for header files # -INCLUDE (CheckIncludeFiles) -INCLUDE (CheckIncludeFileCXX) -INCLUDE (CheckCSourceCompiles) -INCLUDE (CheckCXXSourceRuns) -INCLUDE (CheckCXXSourceCompiles) -INCLUDE (CheckStructHasMember) -INCLUDE (CheckLibraryExists) -INCLUDE (CheckFunctionExists) -INCLUDE (CheckCCompilerFlag) -INCLUDE (CheckCXXCompilerFlag) -INCLUDE (CheckCXXSourceRuns) -INCLUDE (CheckSymbolExists) -INCLUDE (CheckCXXSymbolExists) -INCLUDE (CheckTypeSize) +include(CheckIncludeFiles) +include(CheckIncludeFileCXX) +include(CheckCSourceCompiles) +include(CheckCXXSourceRuns) +include(CheckCXXSourceCompiles) +include(CheckStructHasMember) +include(CheckLibraryExists) +include(CheckFunctionExists) +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) +include(CheckCXXSourceRuns) +include(CheckSymbolExists) +include(CheckCXXSymbolExists) +include(CheckTypeSize) -CHECK_INCLUDE_FILE_CXX (alloca.h HAVE_ALLOCA_H) -CHECK_INCLUDE_FILE_CXX (arpa/inet.h HAVE_ARPA_INET_H) -CHECK_INCLUDE_FILE_CXX (dlfcn.h HAVE_DLFCN_H) -CHECK_INCLUDE_FILE_CXX (fcntl.h HAVE_FCNTL_H) -CHECK_INCLUDE_FILE_CXX (inttypes.h HAVE_INTTYPES_H) -CHECK_INCLUDE_FILE_CXX (limits.h HAVE_LIMITS_H) -CHECK_INCLUDE_FILE_CXX (malloc.h HAVE_MALLOC_H) -CHECK_INCLUDE_FILE_CXX (memory.h HAVE_MEMORY_H) -CHECK_INCLUDE_FILE_CXX (ncurses.h HAVE_NCURSES_H) -CHECK_INCLUDE_FILE_CXX (netdb.h HAVE_NETDB_H) -CHECK_INCLUDE_FILE_CXX (netinet/in.h HAVE_NETINET_IN_H) -CHECK_INCLUDE_FILE_CXX (stddef.h HAVE_STDDEF_H) -CHECK_INCLUDE_FILE_CXX (stdint.h HAVE_STDINT_H) -CHECK_INCLUDE_FILE_CXX (stdlib.h HAVE_STDLIB_H) -CHECK_INCLUDE_FILE_CXX (strings.h HAVE_STRINGS_H) -CHECK_INCLUDE_FILE_CXX (string.h HAVE_STRING_H) -CHECK_INCLUDE_FILE_CXX (syslog.h HAVE_SYSLOG_H) -CHECK_INCLUDE_FILE_CXX (sys/file.h HAVE_SYS_FILE_H) -CHECK_INCLUDE_FILE_CXX (sys/mount.h HAVE_SYS_MOUNT_H) -CHECK_INCLUDE_FILE_CXX (sys/select.h HAVE_SYS_SELECT_H) -CHECK_INCLUDE_FILE_CXX (sys/socket.h HAVE_SYS_SOCKET_H) -CHECK_INCLUDE_FILE_CXX (sys/statfs.h HAVE_SYS_STATFS_H) -CHECK_INCLUDE_FILE_CXX (sys/stat.h HAVE_SYS_STAT_H) -CHECK_INCLUDE_FILE_CXX (sys/timeb.h HAVE_SYS_TIMEB_H) -CHECK_INCLUDE_FILE_CXX (sys/time.h HAVE_SYS_TIME_H) -CHECK_INCLUDE_FILE_CXX (sys/types.h HAVE_SYS_TYPES_H) -CHECK_INCLUDE_FILE_CXX (sys/wait.h HAVE_SYS_WAIT_H) -CHECK_INCLUDE_FILE_CXX (unistd.h HAVE_UNISTD_H) -CHECK_INCLUDE_FILE_CXX (utime.h HAVE_UTIME_H) -CHECK_INCLUDE_FILE_CXX (values.h HAVE_VALUES_H) -CHECK_INCLUDE_FILE_CXX (vfork.h HAVE_VFORK_H) -CHECK_INCLUDE_FILE_CXX (wchar.h HAVE_WCHAR_H) -CHECK_INCLUDE_FILE_CXX (wctype.h HAVE_WCTYPE_H) -CHECK_INCLUDE_FILE_CXX (zlib.h HAVE_ZLIB_H) +check_include_file_cxx(alloca.h HAVE_ALLOCA_H) +check_include_file_cxx(arpa/inet.h HAVE_ARPA_INET_H) +check_include_file_cxx(dlfcn.h HAVE_DLFCN_H) +check_include_file_cxx(fcntl.h HAVE_FCNTL_H) +check_include_file_cxx(inttypes.h HAVE_INTTYPES_H) +check_include_file_cxx(limits.h HAVE_LIMITS_H) +check_include_file_cxx(malloc.h HAVE_MALLOC_H) +check_include_file_cxx(memory.h HAVE_MEMORY_H) +check_include_file_cxx(ncurses.h HAVE_NCURSES_H) +check_include_file_cxx(netdb.h HAVE_NETDB_H) +check_include_file_cxx(netinet/in.h HAVE_NETINET_IN_H) +check_include_file_cxx(stddef.h HAVE_STDDEF_H) +check_include_file_cxx(stdint.h HAVE_STDINT_H) +check_include_file_cxx(stdlib.h HAVE_STDLIB_H) +check_include_file_cxx(strings.h HAVE_STRINGS_H) +check_include_file_cxx(string.h HAVE_STRING_H) +check_include_file_cxx(syslog.h HAVE_SYSLOG_H) +check_include_file_cxx(sys/file.h HAVE_SYS_FILE_H) +check_include_file_cxx(sys/mount.h HAVE_SYS_MOUNT_H) +check_include_file_cxx(sys/select.h HAVE_SYS_SELECT_H) +check_include_file_cxx(sys/socket.h HAVE_SYS_SOCKET_H) +check_include_file_cxx(sys/statfs.h HAVE_SYS_STATFS_H) +check_include_file_cxx(sys/stat.h HAVE_SYS_STAT_H) +check_include_file_cxx(sys/timeb.h HAVE_SYS_TIMEB_H) +check_include_file_cxx(sys/time.h HAVE_SYS_TIME_H) +check_include_file_cxx(sys/types.h HAVE_SYS_TYPES_H) +check_include_file_cxx(sys/wait.h HAVE_SYS_WAIT_H) +check_include_file_cxx(unistd.h HAVE_UNISTD_H) +check_include_file_cxx(utime.h HAVE_UTIME_H) +check_include_file_cxx(values.h HAVE_VALUES_H) +check_include_file_cxx(vfork.h HAVE_VFORK_H) +check_include_file_cxx(wchar.h HAVE_WCHAR_H) +check_include_file_cxx(wctype.h HAVE_WCTYPE_H) +check_include_file_cxx(zlib.h HAVE_ZLIB_H) -CHECK_FUNCTION_EXISTS (_getb67 GETB1) -CHECK_FUNCTION_EXISTS (GETB67 GETB2) -CHECK_FUNCTION_EXISTS (getb67 GETB3) +check_function_exists(_getb67 GETB1) +check_function_exists(GETB67 GETB2) +check_function_exists(getb67 GETB3) -IF(GETB1) - SET (CRAY_STACKSEG_END 1) -ELSEIF(GETB2) - SET (CRAY_STACKSEG_END 1) -ELSEIF(GETB3) - SET (CRAY_STACKSEG_END 1) -ENDIF() +if(GETB1) + set(CRAY_STACKSEG_END 1) +elseif(GETB2) + set(CRAY_STACKSEG_END 1) +elseif(GETB3) + set(CRAY_STACKSEG_END 1) +endif() -CHECK_FUNCTION_EXISTS (alarm HAVE_ALARM) -CHECK_FUNCTION_EXISTS (btowc HAVE_BTOWC) -CHECK_FUNCTION_EXISTS (dup2 HAVE_DUP2) -CHECK_FUNCTION_EXISTS (error_at_line HAVE_ERROR_AT_LINE) -CHECK_FUNCTION_EXISTS (floor HAVE_FLOOR) -CHECK_FUNCTION_EXISTS (fork HAVE_FORK) -CHECK_FUNCTION_EXISTS (ftime HAVE_FTIME) -CHECK_FUNCTION_EXISTS (ftruncate HAVE_FTRUNCATE) -CHECK_FUNCTION_EXISTS (getenv HAVE_DECL_GETENV) -CHECK_FUNCTION_EXISTS (gethostbyname HAVE_GETHOSTBYNAME) -CHECK_FUNCTION_EXISTS (getpagesize HAVE_GETPAGESIZE) -CHECK_FUNCTION_EXISTS (gettimeofday HAVE_GETTIMEOFDAY) -CHECK_FUNCTION_EXISTS (inet_ntoa HAVE_INET_NTOA) -CHECK_FUNCTION_EXISTS (isascii HAVE_ISASCII) -CHECK_FUNCTION_EXISTS (localtime_r HAVE_LOCALTIME_R) -CHECK_FUNCTION_EXISTS (malloc HAVE_MALLOC) -CHECK_FUNCTION_EXISTS (mbsrtowcs HAVE_MBSRTOWCS) -CHECK_FUNCTION_EXISTS (memchr HAVE_MEMCHR) -CHECK_FUNCTION_EXISTS (memmove HAVE_MEMMOVE) -CHECK_FUNCTION_EXISTS (mempcpy HAVE_MEMPCPY) -CHECK_FUNCTION_EXISTS (memset HAVE_MEMSET) -CHECK_FUNCTION_EXISTS (mkdir HAVE_MKDIR) -CHECK_FUNCTION_EXISTS (mktime HAVE_MKTIME) -CHECK_FUNCTION_EXISTS (pow HAVE_POW) -CHECK_FUNCTION_EXISTS (regcomp HAVE_REGCOMP) -CHECK_FUNCTION_EXISTS (rmdir HAVE_RMDIR) -CHECK_FUNCTION_EXISTS (select HAVE_SELECT) -CHECK_FUNCTION_EXISTS (setenv HAVE_SETENV) -CHECK_FUNCTION_EXISTS (setlocale HAVE_SETLOCALE) -CHECK_FUNCTION_EXISTS (socket HAVE_SOCKET) -CHECK_FUNCTION_EXISTS (stat HAVE_STAT) -CHECK_FUNCTION_EXISTS (strcasecmp HAVE_STRCASECMP) -CHECK_FUNCTION_EXISTS (strchr HAVE_STRCHR) -CHECK_FUNCTION_EXISTS (strcspn HAVE_STRCSPN) -CHECK_FUNCTION_EXISTS (strdup HAVE_STRDUP) -CHECK_FUNCTION_EXISTS (strerror HAVE_STRERROR) -CHECK_FUNCTION_EXISTS (strerror_r HAVE_STRERROR_R) -CHECK_FUNCTION_EXISTS (strftime HAVE_STRFTIME) -CHECK_FUNCTION_EXISTS (strrchr HAVE_STRRCHR) -CHECK_FUNCTION_EXISTS (strspn HAVE_STRSPN) -CHECK_FUNCTION_EXISTS (strstr HAVE_STRSTR) -CHECK_FUNCTION_EXISTS (strtod HAVE_STRTOD) -CHECK_FUNCTION_EXISTS (strtol HAVE_STRTOL) -CHECK_FUNCTION_EXISTS (strtoul HAVE_STRTOUL) -CHECK_FUNCTION_EXISTS (strtoull HAVE_STRTOULL) -CHECK_FUNCTION_EXISTS (utime HAVE_UTIME) -CHECK_FUNCTION_EXISTS (vfork HAVE_VFORK) -CHECK_FUNCTION_EXISTS (wmempcpy HAVE_WMEMPCPY) +check_function_exists(alarm HAVE_ALARM) +check_function_exists(btowc HAVE_BTOWC) +check_function_exists(dup2 HAVE_DUP2) +check_function_exists(error_at_line HAVE_ERROR_AT_LINE) +check_function_exists(floor HAVE_FLOOR) +check_function_exists(fork HAVE_FORK) +check_function_exists(ftime HAVE_FTIME) +check_function_exists(ftruncate HAVE_FTRUNCATE) +check_function_exists(getenv HAVE_DECL_GETENV) +check_function_exists(gethostbyname HAVE_GETHOSTBYNAME) +check_function_exists(getpagesize HAVE_GETPAGESIZE) +check_function_exists(gettimeofday HAVE_GETTIMEOFDAY) +check_function_exists(inet_ntoa HAVE_INET_NTOA) +check_function_exists(isascii HAVE_ISASCII) +check_function_exists(localtime_r HAVE_LOCALTIME_R) +check_function_exists(malloc HAVE_MALLOC) +check_function_exists(mbsrtowcs HAVE_MBSRTOWCS) +check_function_exists(memchr HAVE_MEMCHR) +check_function_exists(memmove HAVE_MEMMOVE) +check_function_exists(mempcpy HAVE_MEMPCPY) +check_function_exists(memset HAVE_MEMSET) +check_function_exists(mkdir HAVE_MKDIR) +check_function_exists(mktime HAVE_MKTIME) +check_function_exists(pow HAVE_POW) +check_function_exists(regcomp HAVE_REGCOMP) +check_function_exists(rmdir HAVE_RMDIR) +check_function_exists(select HAVE_SELECT) +check_function_exists(setenv HAVE_SETENV) +check_function_exists(setlocale HAVE_SETLOCALE) +check_function_exists(socket HAVE_SOCKET) +check_function_exists(stat HAVE_STAT) +check_function_exists(strcasecmp HAVE_STRCASECMP) +check_function_exists(strchr HAVE_STRCHR) +check_function_exists(strcspn HAVE_STRCSPN) +check_function_exists(strdup HAVE_STRDUP) +check_function_exists(strerror HAVE_STRERROR) +check_function_exists(strerror_r HAVE_STRERROR_R) +check_function_exists(strftime HAVE_STRFTIME) +check_function_exists(strrchr HAVE_STRRCHR) +check_function_exists(strspn HAVE_STRSPN) +check_function_exists(strstr HAVE_STRSTR) +check_function_exists(strtod HAVE_STRTOD) +check_function_exists(strtol HAVE_STRTOL) +check_function_exists(strtoul HAVE_STRTOUL) +check_function_exists(strtoull HAVE_STRTOULL) +check_function_exists(utime HAVE_UTIME) +check_function_exists(vfork HAVE_VFORK) +check_function_exists(wmempcpy HAVE_WMEMPCPY) -CHECK_CXX_SYMBOL_EXISTS (alloca alloca.h HAVE_ALLOCA) -CHECK_CXX_SYMBOL_EXISTS (strerror_r string.h HAVE_DECL_STRERROR_R) -CHECK_CXX_SYMBOL_EXISTS (tm sys/time.h TM_IN_SYS_TIME) +check_cxx_symbol_exists(alloca alloca.h HAVE_ALLOCA) +check_cxx_symbol_exists(strerror_r string.h HAVE_DECL_STRERROR_R) +check_cxx_symbol_exists(tm sys/time.h TM_IN_SYS_TIME) -#AC_TYPE_SIGNAL -CHECK_TYPE_SIZE (ptrdiff_t PTRDIFF_T) -CHECK_TYPE_SIZE (_Bool __BOOL) -CHECK_TYPE_SIZE (mode_t mode_t_test) -IF(NOT HAVE_mode_t_test) -SET (mode_t int) -ENDIF() -CHECK_TYPE_SIZE(off_t off_t_test) -IF(NOT HAVE_off_t_test) -SET (off_t long int) -ENDIF() -CHECK_TYPE_SIZE(pid_t pid_t_test) -IF(NOT HAVE_pid_t_test) -SET (pid_t int) -ENDIF() -CHECK_TYPE_SIZE(size_t size_t_test) -IF(NOT HAVE_size_t_test) -SET (size_t unsigned int) -ENDIF() +# AC_TYPE_SIGNAL +check_type_size(ptrdiff_t PTRDIFF_T) +check_type_size(_Bool __BOOL) +check_type_size(mode_t mode_t_test) +if(NOT HAVE_mode_t_test) + set(mode_t int) +endif() +check_type_size(off_t off_t_test) +if(NOT HAVE_off_t_test) + set(off_t long int) +endif() +check_type_size(pid_t pid_t_test) +if(NOT HAVE_pid_t_test) + set(pid_t int) +endif() +check_type_size(size_t size_t_test) +if(NOT HAVE_size_t_test) + set(size_t unsigned int) +endif() - - -CHECK_CXX_SOURCE_COMPILES( -"#include +check_cxx_source_compiles( + "#include #include #include #include @@ -152,71 +148,72 @@ main () ; return 0; -}" STDC_HEADERS) +}" + STDC_HEADERS +) +set(TEST_INCLUDES " +#include " +) -SET (TEST_INCLUDES -" -#include ") - -IF (HAVE_SYS_TYPES_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_SYS_STAT_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (STDC_HEADERS) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include - # include ") -ELSE() - IF() - SET ( TEST_INCLUDES +if(HAVE_SYS_TYPES_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_SYS_STAT_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(STDC_HEADERS) + set(TEST_INCLUDES "${TEST_INCLUDES} - # include ") - ENDIF() -ENDIF() -IF (HAVE_STRING_H) - IF(NOT STDC_HEADERS) - IF (HAVE_MEMORY_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") - ENDIF() - ENDIF() - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_STRINGS_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_INTTYPES_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_STDINT_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() -IF (HAVE_UNISTD_H) - SET ( TEST_INCLUDES - "${TEST_INCLUDES} - # include ") -ENDIF() + # include + # include " + ) +else() + if() + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) + endif() +endif() +if(HAVE_STRING_H) + if(NOT STDC_HEADERS) + if(HAVE_MEMORY_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) + endif() + endif() + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_STRINGS_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_INTTYPES_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_STDINT_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() +if(HAVE_UNISTD_H) + set(TEST_INCLUDES "${TEST_INCLUDES} + # include " + ) +endif() - -CHECK_CXX_SOURCE_COMPILES( -" +check_cxx_source_compiles( + " ${TEST_INCLUDES} # include int @@ -225,11 +222,12 @@ main () mbstate_t x; return sizeof x; ; return 0; -}" HAVE_MBSTATE_T) +}" + HAVE_MBSTATE_T +) - -CHECK_CXX_SOURCE_RUNS( -" +check_cxx_source_runs( + " ${TEST_INCLUDES} int main () @@ -238,14 +236,15 @@ struct stat sbuf; return stat (\"\", &sbuf) == 0; ; return 0; -}" STAT_EMPTY_STRING_BUG) -IF (NOT STAT_EMPTY_STRING_BUG) -SET (HAVE_STAT_EMPTY_STRING_BUG 1) -ENDIF() +}" + STAT_EMPTY_STRING_BUG +) +if(NOT STAT_EMPTY_STRING_BUG) + set(HAVE_STAT_EMPTY_STRING_BUG 1) +endif() - -CHECK_CXX_SOURCE_COMPILES( - " +check_cxx_source_compiles( + " ${TEST_INCLUDES} #include #ifndef bool @@ -304,11 +303,13 @@ main () ; return 0; -}" HAVE_STDBOOL_H) +}" + HAVE_STDBOOL_H +) -IF (HAVE_UTIME_H) -CHECK_CXX_SOURCE_COMPILES( -"${TEST_INCLUDES} +if(HAVE_UTIME_H) + check_cxx_source_compiles( + "${TEST_INCLUDES} # include int main () @@ -321,11 +322,13 @@ struct stat s, t; && t.st_mtime - s.st_mtime < 120); ; return 0; -}" HAVE_UTIME_NULL) -ENDIF() +}" + HAVE_UTIME_NULL + ) +endif() -CHECK_CXX_SOURCE_COMPILES( -" +check_cxx_source_compiles( + " ${TEST_INCLUDES} int main () @@ -334,10 +337,12 @@ main () return fork () < 0; ; return 0; -}" HAVE_WORKING_FORK) +}" + HAVE_WORKING_FORK +) -CHECK_CXX_SOURCE_COMPILES( -"${TEST_INCLUDES} +check_cxx_source_compiles( + "${TEST_INCLUDES} #include #ifdef HAVE_VFORK_H include @@ -424,14 +429,16 @@ main () || fstat(fileno(stdout), &st) != 0 ); } -}" HAVE_WORKING_VFORK) +}" + HAVE_WORKING_VFORK +) -IF (NOT HAVE_WORKING_VFORK) -SET (VFORK fork) -ENDIF() +if(NOT HAVE_WORKING_VFORK) + set(VFORK fork) +endif() -CHECK_CXX_SOURCE_COMPILES( -" +check_cxx_source_compiles( + " #include #include @@ -441,22 +448,24 @@ main () return *(signal (0, 0)) (0) == 1; ; return 0; -}" RET_SIGNAL_TYPES) -IF (RET_SIGNAL_TYPES) -SET (RETSIGTYPE int) -ELSE() -SET (RETSIGTYPE void) -ENDIF() +}" + RET_SIGNAL_TYPES +) +if(RET_SIGNAL_TYPES) + set(RETSIGTYPE int) +else() + set(RETSIGTYPE void) +endif() -#IF(NOT LSTAT_FOLLOWS_SLASHED_SYMLINK) -EXECUTE_PROCESS( +# IF(NOT LSTAT_FOLLOWS_SLASHED_SYMLINK) +execute_process( COMMAND rm -f conftest.sym conftest.file COMMAND touch conftest.file COMMAND ln -s conftest.file conftest.sym WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) -CHECK_CXX_SOURCE_RUNS( -" +) +check_cxx_source_runs( + " ${TEST_INCLUDES} int main () @@ -468,28 +477,27 @@ struct stat sbuf; return lstat (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.sym/\", &sbuf) == 0; ; return 0; -}" LSTAT_FOLLOWS_SLASHED_SYMLINK) +}" + LSTAT_FOLLOWS_SLASHED_SYMLINK +) +set(SELECT_INCLUDES ${TEST_INCLUDES}) +if(HAVE_SYS_SELECT_H) + set(SELECT_INCULDES "${SELECT_INCLUDES} +# include " + ) +endif() +if(HAVE_SYS_SOCKET_H) + set(SELECT_INCULDES "${SELECT_INCLUDES} +# include " + ) +endif() - -SET (SELECT_INCLUDES ${TEST_INCLUDES}) -IF (HAVE_SYS_SELECT_H) -SET (SELECT_INCULDES -"${SELECT_INCLUDES} -# include ") -ENDIF() -IF (HAVE_SYS_SOCKET_H) -SET (SELECT_INCULDES -"${SELECT_INCLUDES} -# include ") -ENDIF() - - -FOREACH( ARG234 "fd_set *" "int *" "void *") -FOREACH( ARG1 "int" "size_t" "unsigned long int" "unsigned int") -FOREACH( ARG5 "struct timeval *" "const struct timeval *") -CHECK_CXX_SOURCE_COMPILES( -" +foreach(ARG234 "fd_set *" "int *" "void *") + foreach(ARG1 "int" "size_t" "unsigned long int" "unsigned int") + foreach(ARG5 "struct timeval *" "const struct timeval *") + check_cxx_source_compiles( + " ${SELECT_INCLUDES} #ifdef HAVE_SYS_SELECT_H # include @@ -507,26 +515,27 @@ extern int select (${ARG1}, ; return 0; } -" SELECT_ARGS) -IF(SELECT_ARGS) -SET (SELECT_TYPE_ARG1 ${ARG1}) -SET (SELECT_TYPE_ARG234 ${ARG234}) -SET (SELECT_TYPE_ARG5 ${ARG5}) -BREAK() -ENDIF() -ENDFOREACH() -IF(SELECT_ARGS) -BREAK() -ENDIF() -ENDFOREACH() -IF(SELECT_ARGS) -BREAK() -ENDIF() -ENDFOREACH() - - -CHECK_CXX_SOURCE_COMPILES( " + SELECT_ARGS + ) + if(SELECT_ARGS) + set(SELECT_TYPE_ARG1 ${ARG1}) + set(SELECT_TYPE_ARG234 ${ARG234}) + set(SELECT_TYPE_ARG5 ${ARG5}) + break() + endif() + endforeach() + if(SELECT_ARGS) + break() + endif() + endforeach() + if(SELECT_ARGS) + break() + endif() +endforeach() + +check_cxx_source_compiles( + " #include #include @@ -549,13 +558,15 @@ int main() { return 0; } -" STATS_MACROS_CHECK) -IF (NOT STATS_MACROS_CHECK) -SET (STAT_MACROS_BROKEN 1) -ENDIF() - -CHECK_CXX_SOURCE_COMPILES( " + STATS_MACROS_CHECK +) +if(NOT STATS_MACROS_CHECK) + set(STAT_MACROS_BROKEN 1) +endif() + +check_cxx_source_compiles( + " ${TEST_INCLUDES} int main () @@ -569,10 +580,12 @@ main () ; return 0; } -" STRERROR_R_CHAR_P) - -CHECK_CXX_SOURCE_COMPILES( " + STRERROR_R_CHAR_P +) + +check_cxx_source_compiles( + " #include #include #include @@ -585,11 +598,12 @@ return 0; ; return 0; } -" TIME_WITH_SYS_TIME) - - -CHECK_CXX_SOURCE_COMPILES( " + TIME_WITH_SYS_TIME +) + +check_cxx_source_compiles( + " int main () { @@ -647,13 +661,15 @@ main () ; return 0; } -" CONST_CONFORM_CHECK) -IF (NOT CONST_CONFORM_CHECK) -SET (const "") -ENDIF() - -CHECK_CXX_SOURCE_COMPILES( " + CONST_CONFORM_CHECK +) +if(NOT CONST_CONFORM_CHECK) + set(const "") +endif() + +check_cxx_source_compiles( + " int main () { @@ -664,15 +680,17 @@ return !x && !y; ; return 0; } -" WORKING_VOLATILE) -IF (NOT WORKING_VOLATILE) -SET (volatile "") -ENDIF() - -FOREACH (RESTRICT_KW __restrict __restrict__ _Restrict restrict) - -CHECK_CXX_SOURCE_COMPILES( " + WORKING_VOLATILE +) +if(NOT WORKING_VOLATILE) + set(volatile "") +endif() + +foreach(RESTRICT_KW __restrict __restrict__ _Restrict restrict) + + check_cxx_source_compiles( + " typedef int * int_ptr; int foo (int_ptr ${RESTRICT_KW} ip) { return ip[0]; @@ -687,53 +705,84 @@ int s[1]; ; return 0; } -" RESTRICT_CHECK) -IF (RESTRICT_CHECK) -SET (restrict ${RESTRICT_KW}) -BREAK() -ENDIF() -ENDFOREACH() - - -FOREACH(INLINE_KW inline __inline__ __inline) -CHECK_CXX_SOURCE_COMPILES( " + RESTRICT_CHECK + ) + if(RESTRICT_CHECK) + set(restrict ${RESTRICT_KW}) + break() + endif() +endforeach() + +foreach(INLINE_KW inline __inline__ __inline) + check_cxx_source_compiles( + " #ifndef __cplusplus typedef int foo_t; static ${INLINE_KW} foo_t static_foo () {return 0; } ${INLINE_KW} foo_t foo () {return 0; } int main (){return 0;} #endif -" INLINE) - -IF (INLINE) -SET (inline ${INLINE_KW}) -BREAK() -ENDIF() -ENDFOREACH() - -IF (NOT INLINE) -SET (inline "") -ENDIF() - -EXECUTE_PROCESS( - COMMAND rm -f conftest.data conftest.file conftest.sym - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +" + INLINE ) - -CHECK_CXX_SOURCE_RUNS(" + + if(INLINE) + set(inline ${INLINE_KW}) + break() + endif() +endforeach() + +if(NOT INLINE) + set(inline "") +endif() + +execute_process(COMMAND rm -f conftest.data conftest.file conftest.sym WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +check_cxx_source_runs( + " #include int main() { - // If long double is 16 bytes and digits and exponent are 64 and 16384 respectively, then we need to mask out the + // If long double is 16 bytes and digits and exponent are 64 and 16384 respectively, then we need to mask out the // unused bits, as they contain garbage. There are times we test for equality by memcmp of a buffer containing, // in part, the long double set here. Garbage bytes will adversly affect that compare. // Note: There may be compilers that store 80 bit floats in 12 bytes. We do not account for that here. I don't believe // there are any modern Linux compilers that do that as a default. Windows uses 64 bits, so no masking is needed. - if (std::numeric_limits::digits == 64 - && std::numeric_limits::max_exponent == 16384 + if (std::numeric_limits::digits == 64 + && std::numeric_limits::max_exponent == 16384 && sizeof(long double) == 16) return 0; - return 1; + return 1; }" -MASK_LONGDOUBLE) + MASK_LONGDOUBLE +) + +find_package(Git QUIET) + +if(GIT_FOUND AND EXISTS ${ENGINE_SRC_DIR}/.git) + exec_program( + "git" ${CMAKE_CURRENT_SOURCE_DIR} + ARGS "describe --match=NeVeRmAtCh --always --dirty" + OUTPUT_VARIABLE GIT_VERSION + ) +else() + set(GIT_VERSION "source") +endif() + +# releasenum is used by external scripts for various tasks. Leave it alone. +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) +install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum + DESTINATION ${ENGINE_SUPPORTDIR} + COMPONENT columnstore-engine +) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) +install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine + DESTINATION ${ENGINE_SUPPORTDIR} + COMPONENT columnstore-engine +) diff --git a/cmake/cpackEngineDEB.cmake b/cmake/cpackEngineDEB.cmake deleted file mode 100644 index a7070805a..000000000 --- a/cmake/cpackEngineDEB.cmake +++ /dev/null @@ -1,65 +0,0 @@ -IF(DEB) - -CMAKE_MINIMUM_REQUIRED(VERSION 3.10) - -SET(CMAKE_INSTALL_PREFIX ${INSTALL_ENGINE}) - -SET(CPACK_GENERATOR "DEB") -SET(CPACK_DEBIAN_PACKAGE_DEBUG 1) -SET(CPACK_PACKAGING_INSTALL_PREFIX ${INSTALL_ENGINE}) - -# Note that this variable is DEB not DEBIAN! http://public.kitware.com/pipermail/cmake/2014-July/058030.html -SET(CPACK_DEB_COMPONENT_INSTALL ON) - -SET(CPACK_COMPONENTS_ALL columnstore-engine) - -SET(CPACK_PACKAGE_NAME "MariaDB") -SET(ENGINE_ARCH "amd64") - -IF (NOT CPACK_DEBIAN_PACKAGE_VERSION) - SET (CPACK_DEBIAN_PACKAGE_VERSION ${PACKAGE_VERSION}) -ENDIF() -IF (NOT CPACK_DEBIAN_PACKAGE_RELEASE) - SET (CPACK_DEBIAN_PACKAGE_RELEASE ${PACKAGE_RELEASE}) -ENDIF() - -SET(CPACK_DEBIAN_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) -SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_DEBIAN_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_RELEASE}-${ENGINE_ARCH}") - -SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MariaDB ColumnStore: A Scale out Columnar storage engine for MariaDB") -SET(CPACK_PACKAGE_URL "http://mariadb.org") -SET(CPACK_PACKAGE_CONTACT "MariaDB Corporation Ab") -SET(CPACK_PACKAGE_SUMMARY "MariaDB ColumnStore: A Scale out Columnar storage engine for MariaDB") -SET(CPACK_PACKAGE_VENDOR "MariaDB Corporation Ab") -SET(CPACK_PACKAGE_LICENSE "Copyright (c) 2016 MariaDB Corporation Ab., all rights reserved; redistributable under the terms of the GPL, see the file COPYING for details.") - - -SET(CPACK_DEBIAN_PACKAGE_LICENSE "GPLv2") -SET(CPACK_DEBIAN_PACKAGE_RELOCATABLE FALSE) -SET(CPACK_PACKAGE_RELOCATABLE FALSE) -SET(CPACK_DEBIAN_PACKAGE_URL ${CPACK_PACKAGE_URL}) -SET(CPACK_DEBIAN_PACKAGE_SUMMARY ${CPACK_PACKAGE_SUMMARY}) -SET(CPACK_DEBIAN_PACKAGE_VENDOR ${CPACK_PACKAGE_VENDOR}) -SET(CPACK_DEBIAN_PACKAGE_LICENSE ${CPACK_PACKAGE_LICENSE}) - -SET(CPACK_DEBIAN_PACKAGE_DESCRIPTION ${CPACK_PACKAGE_DESCRIPTION_SUMMARY}) - -SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DESCRIPTION "MariaDB Columnstore connector binary files") -SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_SUMMARY "MariaDB ColumnStore: A Scale out Columnar storage engine for MariaDB") - -SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_PROVIDES "MariaDB-columnstore-engine") - -set(DEBIAN_VERSION_NUMBER OFF) -if (EXISTS "/etc/debian_version") - file (READ "/etc/debian_version" DEBIAN_VERSION) - string(REGEX MATCH "([0-9]+).[0-9]+" DEBIAN "${DEBIAN_VERSION}") - set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") -endif () - -SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "openssl, file, libdbi-perl, rsync, libsnappy1v5, net-tools, MariaDB-server, python3, procps") - -set(CPACK_DEBIAN_COLUMNSTORE-ENGINE_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/build/debian/storageEngine/postinst;${CMAKE_CURRENT_SOURCE_DIR}/build/debian/storageEngine/prerm;") - -INCLUDE (CPack) - -ENDIF() diff --git a/cmake/cpackEngineRPM.cmake b/cmake/cpackEngineRPM.cmake deleted file mode 100644 index ac69f7bf9..000000000 --- a/cmake/cpackEngineRPM.cmake +++ /dev/null @@ -1,113 +0,0 @@ -IF(RPM) - -SET(CMAKE_INSTALL_PREFIX ${INSTALL_ENGINE}) - -SET(CPACK_GENERATOR "RPM") -SET(CPACK_RPM_PACKAGE_DEBUG 1) -SET(CPACK_PACKAGING_INSTALL_PREFIX ${INSTALL_ENGINE}) - -SET(CPACK_RPM_COMPONENT_INSTALL ON) - -SET(CPACK_COMPONENTS_ALL columnstore-engine) - -SET(CPACK_PACKAGE_NAME "MariaDB") -SET(ENGINE_ARCH "x86_64") - -IF (NOT CPACK_RPM_PACKAGE_VERSION) -SET (CPACK_RPM_PACKAGE_VERSION ${PACKAGE_VERSION}) -ENDIF() -IF (NOT CPACK_RPM_PACKAGE_RELEASE) -SET (CPACK_RPM_PACKAGE_RELEASE ${PACKAGE_RELEASE}) -ENDIF() - -SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) -SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_RPM_PACKAGE_VERSION}-${CPACK_RPM_PACKAGE_RELEASE}-${ENGINE_ARCH}-${RPM}") - -SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MariaDB ColumnStore: a scale out columnar storage engine for MariaDB") -SET(CPACK_PACKAGE_URL "http://mariadb.org") - -SET(CPACK_PACKAGE_SUMMARY "MariaDB ColumnStore: a scale out columnar storage engine for MariaDB") -SET(CPACK_PACKAGE_VENDOR "MariaDB Corporation Ab") -SET(CPACK_PACKAGE_LICENSE "Copyright (c) 2016 MariaDB Corporation Ab., all rights reserved; redistributable under the terms of the GPL, see the file COPYING for details.") - - -SET(CPACK_RPM_PACKAGE_LICENSE "GPLv2") -SET(CPACK_RPM_PACKAGE_RELOCATABLE FALSE) -SET(CPACK_PACKAGE_RELOCATABLE FALSE) -SET(CPACK_RPM_PACKAGE_GROUP "Applications/Databases") -SET(CPACK_RPM_PACKAGE_URL ${CPACK_PACKAGE_URL}) -SET(CPACK_RPM_PACKAGE_SUMMARY ${CPACK_PACKAGE_SUMMARY}) -SET(CPACK_RPM_PACKAGE_VENDOR ${CPACK_PACKAGE_VENDOR}) -SET(CPACK_RPM_PACKAGE_LICENSE ${CPACK_PACKAGE_LICENSE}) - -SET(CPACK_RPM_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY} - -It is GPL v2 licensed, which means you can use the it free of charge under the -conditions of the GNU General Public License Version 2 (http://www.gnu.org/licenses/). - -MariaDB documentation can be found at https://mariadb.com/kb -MariaDB bug reports should be submitted through https://jira.mariadb.org - -") - -SET(CPACK_RPM_columnstore-engine_PACKAGE_DESCRIPTION "MariaDB Columnstore connector binary files") -SET(CPACK_RPM_columnstore-engine_PACKAGE_SUMMARY "MariaDB ColumnStore: a scale out columnar storage engine for MariaDB") -SET(CPACK_RPM_columnstore-engine_PACKAGE_GROUP "Applications") - -# "set/append array" - append a set of strings, separated by a space -MACRO(SETA var) - FOREACH(v ${ARGN}) - SET(${var} "${${var}} ${v}") - ENDFOREACH() -ENDMACRO(SETA) - -SETA(CPACK_RPM_columnstore-engine_PACKAGE_PROVIDES "MariaDB-columnstore-engine") - -#boost is a source build in CentOS 6 so don't require it as a package -SET(REDHAT_VERSION_NUMBER OFF) -SET(SUSE_VERSION_NUMBER OFF) -IF (EXISTS "/etc/redhat-release") - file (READ "/etc/redhat-release" REDHAT_VERSION) - string(REGEX MATCH "release ([0-9]+)" CENTOS "${REDHAT_VERSION}") - set(REDHAT_VERSION_NUMBER "${CMAKE_MATCH_1}") -ENDIF () - -IF (EXISTS "/etc/SuSE-release") - file (READ "/etc/SuSE-release" SUSE_VERSION) - string(REGEX MATCH "VERSION = ([0-9]+)" SUSE "${SUSE_VERSION}") - set(SUSE_VERSION_NUMBER "${CMAKE_MATCH_1}") -ENDIF () - -SETA(CPACK_RPM_columnstore-engine_PACKAGE_REQUIRES "snappy" "net-tools" "MariaDB-server" "python3" "jemalloc" "procps-ng") - -SET(CPACK_RPM_columnstore-engine_PRE_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/preInstall_storage_engine.sh) -SET(CPACK_RPM_columnstore-engine_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/postInstall_storage_engine.sh) -SET(CPACK_RPM_columnstore-engine_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/preUn_storage_engine.sh) -SET(CPACK_RPM_columnstore-engine_POST_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/postUn_storage_engine.sh) - -SET(CPACK_RPM_SPEC_MORE_DEFINE "${CPACK_RPM_SPEC_MORE_DEFINE} -%define ignore \# -") - -SET(ignored - "%ignore /usr" - "%ignore /usr/local" - "%ignore /bin" - "%ignore /lib" - "%ignore /usr/sbin" - "%ignore /usr/lib64/mysql" - "%ignore /usr/lib64/mysql/plugin" - "%ignore /etc/my.cnf.d" - "%ignore /var/lib" - "%ignore /var" -) - -#SET(CPACK_RPM_SPEC_MORE_DEFINE " -#%define _prefix ${CMAKE_INSTALL_PREFIX} -#") - -SET(CPACK_RPM_columnstore-engine_USER_FILELIST ${ignored}) - -INCLUDE (CPack) - -ENDIF() diff --git a/cmake/dirs.cmake b/cmake/dirs.cmake new file mode 100644 index 000000000..6cd876736 --- /dev/null +++ b/cmake/dirs.cmake @@ -0,0 +1,20 @@ +set(ENGINE_SYSCONFDIR "/etc") +set(ENGINE_DATADIR "/var/lib/columnstore") +set(ENGINE_LOGDIR "/var/log/mariadb/columnstore") +set(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR}") +set(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR}") +set(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR}") +set(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") +set(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR}") +set(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR}") +set(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") +set(ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +if(INSTALL_SYSCONF2DIR) + set(MARIADB_MYCNFDIR "${INSTALL_SYSCONF2DIR}") +else() + set(MARIADB_MYCNFDIR "/etc/mysql") +endif() + +set(SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR}) +set(SERVER_BUILD_DIR ${CMAKE_BINARY_DIR}) diff --git a/cmake/includes.cmake b/cmake/includes.cmake new file mode 100644 index 000000000..88edb345a --- /dev/null +++ b/cmake/includes.cmake @@ -0,0 +1,108 @@ +message_once(SERVER_BUILD_INCLUDE_DIR "SERVER_BUILD_INCLUDE_DIR = ${SERVER_BUILD_INCLUDE_DIR}") +message_once(SERVER_SOURCE_ROOT_DIR "SERVER_SOURCE_ROOT_DIR = ${SERVER_SOURCE_ROOT_DIR}") + +set(ENGINE_UTILS_MESSAGEQCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/messageqcpp") +set(ENGINE_WE_SHARED_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/shared") +set(ENGINE_UTILS_IDBDATAFILE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/idbdatafile") +set(ENGINE_UTILS_LOGGINGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/loggingcpp" + "${CMAKE_CURRENT_BINARY_DIR}/utils/loggingcpp" +) +set(ENGINE_UTILS_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/configcpp") +set(ENGINE_UTILS_COMPRESS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/compress") +set(ENGINE_VERSIONING_BRM_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/versioning/BRM") +set(ENGINE_UTILS_ROWGROUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rowgroup") +set(ENGINE_UTILS_COMMON_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/common") +set(ENGINE_UTILS_DATACONVERT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/dataconvert") +set(ENGINE_UTILS_RWLOCK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rwlock") +set(ENGINE_UTILS_FUNCEXP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/funcexp") +set(ENGINE_UTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils") +set(ENGINE_OAM_OAMCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/oam/oamcpp") +set(ENGINE_DBCON_DDLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackageproc") +set(ENGINE_DBCON_DDLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackage") +set(ENGINE_DBCON_EXECPLAN_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/execplan") +set(ENGINE_UTILS_STARTUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/startup") +set(ENGINE_DBCON_JOBLIST_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/joblist") +set(ENGINE_WE_WRAPPER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/wrapper") +set(ENGINE_WE_SERVER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/server") +set(ENGINE_DBCON_DMLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackage") +set(ENGINE_WE_CLIENT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/client") +set(ENGINE_DBCON_DMLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackageproc") +set(ENGINE_UTILS_CACHEUTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/cacheutils") +set(ENGINE_UTILS_MYSQLCL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/mysqlcl_idb") +set(ENGINE_UTILS_QUERYTELE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querytele") +set(ENGINE_UTILS_JOINER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/joiner") +set(ENGINE_UTILS_THREADPOOL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/threadpool") +set(ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchloader") +set(ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup") +set(ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") +set(ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") +set(ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") +set(ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") +set(ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache") +set(ENGINE_PRIMPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/primproc") +set(ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") +set(ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") +if(PCRE_INCLUDES) + set(ENGINE_SERVER_PCRE_INCLUDE "${PCRE_INCLUDES}") +else() + set(ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre") +endif() +set(ENGINE_SERVER_WSREP_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/include") +set(ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/wsrep-API/v26/") +set(ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk") + +set(ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) + +set(ENGINE_COMMON_INCLUDES + ${ENGINE_DEFAULT_INCLUDES} + ${Boost_INCLUDE_DIRS} + ${LIBXML2_INCLUDE_DIR} + ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} + ${ENGINE_WE_SHARED_INCLUDE} + ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} + ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} + ${ENGINE_UTILS_CONFIGCPP_INCLUDE} + ${ENGINE_UTILS_COMPRESS_INCLUDE} + ${ENGINE_VERSIONING_BRM_INCLUDE} + ${ENGINE_UTILS_ROWGROUP_INCLUDE} + ${ENGINE_UTILS_COMMON_INCLUDE} + ${ENGINE_UTILS_DATACONVERT_INCLUDE} + ${ENGINE_UTILS_RWLOCK_INCLUDE} + ${ENGINE_UTILS_FUNCEXP_INCLUDE} + ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} + ${ENGINE_UTILS_INCLUDE} + ${ENGINE_OAM_OAMCPP_INCLUDE} + ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} + ${ENGINE_DBCON_DDLPKG_INCLUDE} + ${ENGINE_DBCON_EXECPLAN_INCLUDE} + ${ENGINE_UTILS_STARTUP_INCLUDE} + ${ENGINE_DBCON_JOBLIST_INCLUDE} + ${ENGINE_WE_WRAPPER_INCLUDE} + ${ENGINE_WE_SERVER_INCLUDE} + ${ENGINE_DBCON_DMLPKG_INCLUDE} + ${ENGINE_WE_CLIENT_INCLUDE} + ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} + ${ENGINE_UTILS_CACHEUTILS_INCLUDE} + ${ENGINE_UTILS_MYSQLCL_INCLUDE} + ${ENGINE_UTILS_QUERYTELE_INCLUDE} + ${ENGINE_UTILS_THRIFT_INCLUDE} + ${ENGINE_UTILS_JOINER_INCLUDE} + ${ENGINE_UTILS_THREADPOOL_INCLUDE} + ${ENGINE_UTILS_BATCHLDR_INCLUDE} + ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} + ${ENGINE_UTILS_QUERYSTATS_INCLUDE} + ${ENGINE_WE_CONFIGCPP_INCLUDE} + ${ENGINE_SERVER_SQL_INCLUDE} + ${ENGINE_SERVER_INCLUDE_INCLUDE} + ${ENGINE_SERVER_PCRE_INCLUDE} + ${ENGINE_SERVER_WSREP_API_INCLUDE} + ${ENGINE_SERVER_WSREP_INCLUDE} + ${ENGINE_UTILS_UDFSDK_INCLUDE} + ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} + ${ENGINE_DATATYPES_INCLUDE} +) + +set(LIBMARIADB_BININC_DIR ${CMAKE_BINARY_DIR}/libmariadb/include) +set(LIBMARIADB_SRCINC_DIR ${CMAKE_SOURCE_DIR}/libmariadb/include) +set(SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include) +set(SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR}) diff --git a/cmake/libs.cmake b/cmake/libs.cmake new file mode 100644 index 000000000..687cac600 --- /dev/null +++ b/cmake/libs.cmake @@ -0,0 +1,44 @@ +set(ENGINE_DT_LIB datatypes) +set(ENGINE_COMMON_LIBS + boost_thread + configcpp + idbboot + loggingcpp + messageqcpp + pthread + rt + xml2 + ${ENGINE_DT_LIB} +) +set(ENGINE_OAM_LIBS oamcpp) +set(ENGINE_BRM_LIBS brm cacheutils idbdatafile rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) + +set(PLUGIN_EXEC_LIBS + common + compress + dataconvert + execplan + funcexp + joiner + querytele + regr + rowgroup + threadpool + udfsdk + windowfunction + ${ENGINE_BRM_LIBS} +) +set(ENGINE_EXEC_LIBS joblist querystats libmysql_client ${PLUGIN_EXEC_LIBS}) +set(PLUGIN_WRITE_LIBS + cacheutils + ddlpackage + ddlpackageproc + dmlpackage + dmlpackageproc + idbdatafile + writeengine + writeengineclient +) +set(ENGINE_WRITE_LIBS ${PLUGIN_WRITE_LIBS} ${ENGINE_EXEC_LIBS}) +set(MARIADB_CLIENT_LIBS libmariadb) +set(MARIADB_STRING_LIBS dbug strings mysys) diff --git a/cmake/packages.cmake b/cmake/packages.cmake new file mode 100644 index 000000000..70b001604 --- /dev/null +++ b/cmake/packages.cmake @@ -0,0 +1,65 @@ +find_package(BISON) +if(NOT BISON_FOUND) + message_once(CS_NO_BISON "bison not found!") + return() +endif() + +find_program(LEX_EXECUTABLE flex DOC "path to the flex executable") +if(NOT LEX_EXECUTABLE) + find_program(LEX_EXECUTABLE lex DOC "path to the lex executable") + if(NOT LEX_EXECUTABLE) + message_once(CS_NO_LEX "flex/lex not found!") + return() + endif() +endif() + +find_package(LibXml2) + +if(NOT LIBXML2_FOUND) + message_once(CS_NO_LIBXML "Could not find a usable libxml2 development environment!") + return() +endif() + +find_package(Snappy) +if(NOT SNAPPY_FOUND) + message_once( + CS_NO_SNAPPY + "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian" + ) + return() +endif() + +find_package(CURL) +if(NOT CURL_FOUND) + message_once(CS_NO_CURL "libcurl development headers not found") + return() +endif() + +find_program(AWK_EXECUTABLE awk DOC "path to the awk executable") +if(NOT AWK_EXECUTABLE) + message_once(CS_NO_AWK "awk not found!") + return() +endif() + +set(HAVE_LZ4 + 0 + CACHE INTERNAL "" +) +if(WITH_COLUMNSTORE_LZ4 STREQUAL "ON" OR WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") + find_package(LZ4) + if(NOT LZ4_FOUND) + if(WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") + message_once(CS_LZ4 "LZ4 not found, building without LZ4") + else() + message(FATAL_ERROR "LZ4 not found.") + endif() + else() + message_once(CS_LZ4 "Building with LZ4") + set(HAVE_LZ4 + 1 + CACHE INTERNAL "" + ) + endif() +else() + message_once(CS_LZ4 "Building without LZ4") +endif() diff --git a/cmake/thrift.cmake b/cmake/thrift.cmake index 26f08bb43..11e1533d6 100644 --- a/cmake/thrift.cmake +++ b/cmake/thrift.cmake @@ -1,45 +1,46 @@ -set(WITH_THRIFT "auto" CACHE STRING - "Which Thrift to use (possible values are 'bundled', 'system', or 'auto')") +set(WITH_THRIFT + "bundled" + CACHE STRING "Which Thrift to use (possible values are 'bundled', 'system', or 'auto')" +) if(WITH_THRIFT STREQUAL "system" OR WITH_THRIFT STREQUAL "auto") - FIND_PACKAGE(Thrift) + find_package(Thrift) - if (Thrift_FOUND) - add_custom_target(external_thrift) - set(THRIFT_INCLUDE_DIR "${THRIFT_INCLUDE_DIR}") - set(THRIFT_LIBRARY "${THRIFT_LIBRARIES}") - return() - elseif(WITH_THRIFT STREQUAL "system") - message(FATAL_ERROR "System Thrift requested but not found!") - endif() + if(Thrift_FOUND) + add_custom_target(external_thrift) + set(THRIFT_INCLUDE_DIR "${THRIFT_INCLUDE_DIR}") + set(THRIFT_LIBRARY "${THRIFT_LIBRARIES}") + return() + elseif(WITH_THRIFT STREQUAL "system") + message(FATAL_ERROR "System Thrift requested but not found!") + endif() endif() include(ExternalProject) set(INSTALL_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/thrift) -SET(THRIFT_INCLUDE_DIRS "${INSTALL_LOCATION}/include") -SET(THRIFT_LIBRARY_DIRS "${INSTALL_LOCATION}/lib") +set(THRIFT_INCLUDE_DIRS "${INSTALL_LOCATION}/include") +set(THRIFT_LIBRARY_DIRS "${INSTALL_LOCATION}/lib") set(THRIFT_LIBRARY ${THRIFT_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}thrift${CMAKE_STATIC_LIBRARY_SUFFIX}) - -ExternalProject_Add(external_thrift +ExternalProject_Add( + external_thrift URL https://github.com/apache/thrift/archive/refs/tags/v0.17.0.tar.gz URL_HASH SHA256=f5888bcd3b8de40c2c2ab86896867ad9b18510deb412cba3e5da76fb4c604c29 PREFIX ${INSTALL_LOCATION} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_LOCATION} - -DBUILD_COMPILER=YES - -DBUILD_CPP=YES - -DBUILD_C_GLIB=YES - -DBUILD_JAVA=NO - -DBUILD_JAVASCRIPT=NO - -DBUILD_KOTLIN=NO - -DBUILD_NODEJS=NO - -DBUILD_PYTHON=NO - -DBUILD_TESTING=NO - -DBUILD_SHARED_LIBS=NO - -DCMAKE_CXX_FLAGS:STRING="-fPIC" - -DBOOST_INCLUDEDIR=${Boost_INCLUDE_DIRS} - -DBOOST_LIBRARYDIR=${Boost_LIBRARY_DIRS} + -DBUILD_COMPILER=YES + -DBUILD_CPP=YES + -DBUILD_C_GLIB=YES + -DBUILD_JAVA=NO + -DBUILD_JAVASCRIPT=NO + -DBUILD_KOTLIN=NO + -DBUILD_NODEJS=NO + -DBUILD_PYTHON=NO + -DBUILD_TESTING=NO + -DBUILD_SHARED_LIBS=NO + -DCMAKE_CXX_FLAGS:STRING="-fPIC" + -DBOOST_ROOT=${BOOST_ROOT} BUILD_BYPRODUCTS "${THRIFT_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}thrift${CMAKE_STATIC_LIBRARY_SUFFIX}" EXCLUDE_FROM_ALL TRUE ) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 7af81922f..7979669bd 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -1,15 +1,17 @@ -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/install_mcs_mysql.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/install_mcs_mysql.sh" @ONLY) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/install_mcs_mysql.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/install_mcs_mysql.sh" @ONLY +) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/ha_mcs_version.h.in" "${CMAKE_CURRENT_SOURCE_DIR}/ha_mcs_version.h") -include_directories( ${ENGINE_COMMON_INCLUDES} - /usr/include/libxml2 - ${SERVER_SOURCE_ROOT_DIR}/storage/maria ) +include_directories(${ENGINE_COMMON_INCLUDES} /usr/include/libxml2 ${SERVER_SOURCE_ROOT_DIR}/storage/maria) -SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../) -SET(S3API_DEPS marias3 curl CACHE INTERNAL "S3API_DEPS") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../) +set(S3API_DEPS + marias3 curl + CACHE INTERNAL "S3API_DEPS" +) - -SET ( libcalmysql_SRCS +set(libcalmysql_SRCS ../../datatypes/mcs_datatype.cpp ../../utils/libmysql_client/libmysql_client.cpp ../../utils/querystats/querystats.cpp @@ -27,7 +29,8 @@ SET ( libcalmysql_SRCS ha_exists_sub.cpp ha_from_sub.cpp ha_select_sub.cpp - ha_view.cpp sm.cpp + ha_view.cpp + sm.cpp ha_window_function.cpp ha_mcs_partition.cpp ha_pseudocolumn.cpp @@ -36,45 +39,82 @@ SET ( libcalmysql_SRCS is_columnstore_columns.cpp is_columnstore_files.cpp is_columnstore_extents.cpp - columnstore_dataload.cpp) - + columnstore_dataload.cpp +) set_source_files_properties(ha_mcs.cpp PROPERTIES COMPILE_FLAGS "-fno-implicit-templates") -if (COMMAND mysql_add_plugin) - IF(NOT(RPM OR DEB)) - SET(disabled DISABLED) - ENDIF() - - mysql_add_plugin(columnstore ${libcalmysql_SRCS} STORAGE_ENGINE MODULE_ONLY ${disabled} - LINK_LIBRARIES ${ENGINE_LDFLAGS} ${PLUGIN_EXEC_LIBS} ${PLUGIN_WRITE_LIBS} joblist_server ${NETSNMP_LIBRARIES} ${S3API_DEPS} threadpool - VERSION ${PACKAGE_VERSION} - COMPONENT columnstore-engine CONFIG columnstore.cnf) -else () +if(COMMAND mysql_add_plugin) + if(NOT (RPM OR DEB)) + set(disabled DISABLED) + endif() + message("Adding plugin via MariaDB plugin API") + mysql_add_plugin( + columnstore + ${libcalmysql_SRCS} + STORAGE_ENGINE + MODULE_ONLY + ${disabled} + LINK_LIBRARIES + ${ENGINE_LDFLAGS} + ${PLUGIN_EXEC_LIBS} + ${PLUGIN_WRITE_LIBS} + joblist_server + ${NETSNMP_LIBRARIES} + ${MARIADB_CLIENT_LIBS} + ${S3API_DEPS} + threadpool + VERSION + ${PACKAGE_VERSION} + COMPONENT + columnstore-engine + CONFIG + columnstore.cnf + ) +else() + message("Adding plugin via CMake") add_library(ha_columnstore SHARED ${libcalmysql_SRCS}) - SET_TARGET_PROPERTIES(ha_columnstore PROPERTIES PREFIX "") + set_target_properties(ha_columnstore PROPERTIES PREFIX "") add_dependencies(ha_columnstore loggingcpp) add_dependencies(ha_columnstore marias3) add_definitions(-DMYSQL_DYNAMIC_PLUGIN -DPLUGIN_COLUMNSTORE_VERSION="${PACKAGE_VERSION}") - target_link_libraries(ha_columnstore ${S3API_DEPS} ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) + target_link_libraries( + ha_columnstore + ${S3API_DEPS} + ${ENGINE_LDFLAGS} + ${ENGINE_WRITE_LIBS} + ${NETSNMP_LIBRARIES} + ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a + threadpool + ) - install(TARGETS ha_columnstore DESTINATION ${MARIADB_PLUGINDIR} COMPONENT columnstore-engine) + install( + TARGETS ha_columnstore + DESTINATION ${MARIADB_PLUGINDIR} + COMPONENT columnstore-engine + ) # define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist) add_custom_target(columnstore DEPENDS ha_columnstore) - install(FILES columnstore.cnf DESTINATION ${MARIADB_MYCNFDIR} COMPONENT columnstore-engine) -endif () - -if (TARGET columnstore) - install(FILES syscatalog_mysql.sql - dumpcat_mysql.sql - calsetuserpriority.sql - calremoveuserpriority.sql - calshowprocesslist.sql - columnstore_info.sql - DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) - install(PROGRAMS install_mcs_mysql.sh - DESTINATION ${ENGINE_SBINDIR} COMPONENT columnstore-engine) + install( + FILES columnstore.cnf + DESTINATION ${MARIADB_MYCNFDIR} + COMPONENT columnstore-engine + ) +endif() + +if(TARGET columnstore) + install( + FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql calremoveuserpriority.sql + calshowprocesslist.sql columnstore_info.sql + DESTINATION ${ENGINE_SUPPORTDIR} + COMPONENT columnstore-engine + ) + install( + PROGRAMS install_mcs_mysql.sh + DESTINATION ${ENGINE_SBINDIR} + COMPONENT columnstore-engine + ) endif() diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index 78aa255b9..068dad945 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -1,132 +1,181 @@ -SET (DEFAULT_USER "mysql") -SET (DEFAULT_GROUP "mysql") -SET(CORE_DUMPS "0") -OPTION(WITH_COREDUMPS "dump cores" OFF) +set(DEFAULT_USER "mysql") +set(DEFAULT_GROUP "mysql") +set(CORE_DUMPS "0") +option(WITH_COREDUMPS "dump cores" OFF) -if (WITH_COREDUMPS) - SET(CORE_DUMPS "infinity") -endif (WITH_COREDUMPS) +if(WITH_COREDUMPS) + set(CORE_DUMPS "infinity") +endif(WITH_COREDUMPS) +set(LD_PRELOAD_STRING "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}')") +set(ALLOC_CONFIG "MALLOC_CONF=''") +set(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) +set(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) +set(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) +set(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG}) +set(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) +set(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) +set(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG}) -SET(LD_PRELOAD_STRING "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}')") -SET(ALLOC_CONFIG "MALLOC_CONF=''") -SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) -SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) -SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) -SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG}) -SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) -SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) -SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG}) +if(WITH_COLUMNSTORE_ASAN) + set(ASAN_PATH "/tmp/asan") + if(WITH_COLUMNSTORE_REPORT_PATH) + set(ASAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/asan") + endif(WITH_COLUMNSTORE_REPORT_PATH) - -if (WITH_COLUMNSTORE_ASAN) - SET(ASAN_PATH "/tmp/asan") - if (WITH_COLUMNSTORE_REPORT_PATH) - SET(ASAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/asan") - endif (WITH_COLUMNSTORE_REPORT_PATH) - - - SET(LD_PRELOAD_STRING "") - SET(ALLOC_CONFIG "ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false") - SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.primproc) - SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.dmlproc) - SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.ddlproc) - SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.writeengine) - SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.controllernode) - SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.workernode) - SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.storagemanager) + set(LD_PRELOAD_STRING "") + set(ALLOC_CONFIG + "ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false" + ) + set(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.primproc) + set(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.dmlproc) + set(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.ddlproc) + set(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.writeengine) + set(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.controllernode) + set(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.workernode) + set(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.storagemanager) endif() -if (WITH_UBSAN) - SET(UBSAN_PATH "/tmp/asan") - if (WITH_COLUMNSTORE_REPORT_PATH) - SET(UBSAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/ubsan") - endif (WITH_COLUMNSTORE_REPORT_PATH) +if(WITH_UBSAN) + set(UBSAN_PATH "/tmp/asan") + if(WITH_COLUMNSTORE_REPORT_PATH) + set(UBSAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/ubsan") + endif(WITH_COLUMNSTORE_REPORT_PATH) - SET(LD_PRELOAD_STRING "") - SET(ALLOC_CONFIG "UBSAN_OPTIONS=abort_on_error=0,print_stacktrace=true") - SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.primproc) - SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.dmlproc) - SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.ddlproc) - SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.writeengine) - SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.controllernode) - SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.workernode) - SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.storagemanager) + set(LD_PRELOAD_STRING "") + set(ALLOC_CONFIG "UBSAN_OPTIONS=abort_on_error=0,print_stacktrace=true") + set(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.primproc) + set(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.dmlproc) + set(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.ddlproc) + set(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.writeengine) + set(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.controllernode) + set(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.workernode) + set(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.storagemanager) endif() -if (WITH_TSAN) - SET(TSAN_PATH "/tmp/tsan") - if (WITH_COLUMNSTORE_REPORT_PATH) - SET(TSAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/tsan") - endif (WITH_COLUMNSTORE_REPORT_PATH) +if(WITH_TSAN) + set(TSAN_PATH "/tmp/tsan") + if(WITH_COLUMNSTORE_REPORT_PATH) + set(TSAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/tsan") + endif(WITH_COLUMNSTORE_REPORT_PATH) - SET(LD_PRELOAD_STRING "") - SET(ALLOC_CONFIG "TSAN_OPTIONS=abort_on_error=0:log_path=${TSAN_PATH}") - SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.primproc) - SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.dmlproc) - SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.ddlproc) - SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.writeengine) - SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.controllernode) - SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.workernode) - SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.storagemanager) + set(LD_PRELOAD_STRING "") + set(ALLOC_CONFIG "TSAN_OPTIONS=abort_on_error=0:log_path=${TSAN_PATH}") + set(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.primproc) + set(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.dmlproc) + set(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.ddlproc) + set(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.writeengine) + set(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.controllernode) + set(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.workernode) + set(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.storagemanager) endif() +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh" + @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install" + @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall" + @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreLogRotate.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreLogRotate" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore.service.in" + "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore-start.sh.in" + "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore-start.sh" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-workernode.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-workernode.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-controllernode.service.in" + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-controllernode.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-primproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-primproc.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-writeengineserver.service.in" + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-writeengineserver.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-dmlproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-dmlproc.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-ddlproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-ddlproc.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-storagemanager.service.in" + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-storagemanager.service" @ONLY +) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-stop-controllernode.sh.in" + "${CMAKE_CURRENT_SOURCE_DIR}/mcs-stop-controllernode.sh" @ONLY +) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreLogRotate.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreLogRotate" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore-start.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore-start.sh" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-workernode.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-workernode.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-controllernode.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-controllernode.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-primproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-primproc.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-writeengineserver.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-writeengineserver.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-dmlproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-dmlproc.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-ddlproc.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-ddlproc.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-storagemanager.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-storagemanager.service" @ONLY) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-stop-controllernode.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-stop-controllernode.sh" @ONLY) +set(SHMEM_FILE_GLOB "MCS-shm-") configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.py" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcssavebrm.py" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY) -install(PROGRAMS columnstore-post-install - columnstore-pre-uninstall - columnstore_run.sh - post-mysql-install - post-mysqld-install - columnstoreSyslogSetup.sh - mcs-stop-controllernode.sh - mcs-loadbrm.py - mcs-savebrm.py - mariadb-columnstore-start.sh - mariadb-columnstore-stop.sh - loop_process_starter.sh - DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +install( + PROGRAMS columnstore-post-install + columnstore-pre-uninstall + columnstore_run.sh + post-mysql-install + post-mysqld-install + columnstoreSyslogSetup.sh + mcs-stop-controllernode.sh + mcs-loadbrm.py + mcs-savebrm.py + mariadb-columnstore-start.sh + mariadb-columnstore-stop.sh + loop_process_starter.sh + DESTINATION ${ENGINE_BINDIR} + COMPONENT columnstore-engine +) -install(FILES mariadb-columnstore.service - columnstoreAlias - columnstoreSyslog - columnstoreSyslog7 - columnstoreSyslog-ng - columnstoreLogRotate - myCnf-include-args.text - myCnf-exclude-args.text - mcs-workernode.service - mcs-controllernode.service - mcs-primproc.service - mcs-writeengineserver.service - mcs-dmlproc.service - mcs-ddlproc.service - mcs-loadbrm.service - mcs-storagemanager.service - DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) +install( + FILES mariadb-columnstore.service + columnstoreAlias + columnstoreSyslog + columnstoreSyslog7 + columnstoreSyslog-ng + columnstoreLogRotate + myCnf-include-args.text + myCnf-exclude-args.text + mcs-workernode.service + mcs-controllernode.service + mcs-primproc.service + mcs-writeengineserver.service + mcs-dmlproc.service + mcs-ddlproc.service + mcs-loadbrm.service + mcs-storagemanager.service + DESTINATION ${ENGINE_SUPPORTDIR} + COMPONENT columnstore-engine +) -install(FILES module DESTINATION ${ENGINE_DATADIR}/local COMPONENT columnstore-engine) +install( + FILES module + DESTINATION ${ENGINE_DATADIR}/local + COMPONENT columnstore-engine +) -find_package (Python3 COMPONENTS Interpreter) +find_package(Python3 COMPONENTS Interpreter) if(Python3_FOUND) - add_test(NAME PythonUnitTests COMMAND ${Python3_EXECUTABLE} -m unittest test_mcs-savebrm.py) + add_test(NAME PythonUnitTests COMMAND ${Python3_EXECUTABLE} -m unittest test_mcs-savebrm.py) endif() diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 3e668cd57..8c4350122 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -43,32 +43,42 @@ set(storagemanager_SRCS option(SM_TRACE "Enable some tracing output" OFF) option(SM_LOG_TO_STDERR "Make the logging system also print to stderr" OFF) -if (SM_TRACE) +if(SM_TRACE) add_definitions(-DSM_TRACE) endif() -if (CMAKE_BUILD_TYPE STREQUAL Debug) +if(CMAKE_BUILD_TYPE STREQUAL Debug) add_definitions(-DDEBUG -Werror -Wall) set(S3_CONFIGURE_OPT --enable-debug) endif() -if (SM_LOG_TO_STDERR) +if(SM_LOG_TO_STDERR) add_definitions(-DDEBUG) endif() -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/storagemanager.cnf.in" "${CMAKE_CURRENT_SOURCE_DIR}/storagemanager.cnf" @ONLY) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/storagemanager.cnf.in" "${CMAKE_CURRENT_SOURCE_DIR}/storagemanager.cnf" @ONLY +) # get linkage right link_directories(${CMAKE_BINARY_DIR}/lib) set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) - - add_library(storagemanager SHARED ${storagemanager_SRCS}) add_dependencies(storagemanager marias3 external_boost) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) +target_link_libraries( + storagemanager + boost_chrono + boost_system + boost_thread + boost_filesystem + boost_regex + pthread + ${S3API_DEPS} +) +target_include_directories(storagemanager PRIVATE ${Boost_INCLUDE_DIRS}) add_executable(StorageManager src/main.cpp) target_link_libraries(StorageManager storagemanager) @@ -88,73 +98,59 @@ target_link_libraries(testS3Connection storagemanager) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${TMPDIR}) add_custom_command( - TARGET unit_tests PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/test_data + TARGET unit_tests + PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test_data ) add_custom_command( - TARGET unit_tests POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory - ${CMAKE_CURRENT_SOURCE_DIR}/test_data - ${CMAKE_CURRENT_BINARY_DIR}/test_data + TARGET unit_tests + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/test_data + ${CMAKE_CURRENT_BINARY_DIR}/test_data ) -# The includes and lib linkages required to link against cloudio ... -# pretty crazy. When lib dependencies are eventually config'd right, -# change this to only include and link against cloudio. +# The includes and lib linkages required to link against cloudio ... pretty crazy. When lib dependencies are eventually +# config'd right, change this to only include and link against cloudio. include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES}) add_executable(smcat src/smcat.cpp) -target_link_libraries(smcat storagemanager cloudio - ${ENGINE_LDFLAGS} - ${ENGINE_EXEC_LIBS} -) +target_link_libraries(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) add_executable(smput src/smput.cpp) -target_link_libraries(smput storagemanager cloudio - ${ENGINE_LDFLAGS} - ${ENGINE_EXEC_LIBS} -) +target_link_libraries(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) add_executable(smls src/smls.cpp) -target_link_libraries(smls storagemanager cloudio - ${ENGINE_LDFLAGS} - ${ENGINE_EXEC_LIBS} -) +target_link_libraries(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) add_executable(smrm src/smrm.cpp) -target_link_libraries(smrm storagemanager cloudio - ${ENGINE_LDFLAGS} - ${ENGINE_EXEC_LIBS} -) +target_link_libraries(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) add_executable(smps src/smps.cpp) -target_link_libraries(smps storagemanager cloudio - ${ENGINE_LDFLAGS} - ${ENGINE_EXEC_LIBS} -) - +target_link_libraries(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) add_executable(smkill src/smkill.cpp) -target_link_libraries(smkill storagemanager cloudio - ${ENGINE_LDFLAGS} - ${ENGINE_EXEC_LIBS} -) -install(TARGETS storagemanager - LIBRARY DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine +target_link_libraries(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +install(TARGETS storagemanager LIBRARY DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) + +install( + TARGETS StorageManager + smcat + smkill + smps + smput + smls + smrm + testS3Connection + RUNTIME DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine ) -install(TARGETS StorageManager smcat smkill smps smput smls smrm testS3Connection - RUNTIME DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) - -install(FILES storagemanager.cnf +install( + FILES storagemanager.cnf DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT columnstore-engine ) -install(FILES storagemanager.cnf +install( + FILES storagemanager.cnf RENAME storagemanager.cnf.example DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT columnstore-engine diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 05eda6456..caaa047a8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,33 +1,32 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) +include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}) -IF(NOT WITH_UBSAN AND NOT WITH_TSAN) - MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) - MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") -ENDIF() +if(COLUMNSTORE_ASAN_FOR_UNITTESTS) + my_check_and_set_compiler_flag("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) + my_check_and_set_compiler_flag("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") +endif() -if (WITH_UNITTESTS) +if(WITH_UNITTESTS) cmake_policy(SET CMP0054 NEW) set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external) - ExternalProject_Add(googletest + ExternalProject_Add( + googletest GIT_REPOSITORY https://github.com/google/googletest GIT_TAG release-1.12.0 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DBUILD_SHARED_LIBS=ON ) - # this is a workaround for gtest's bug still not closed - # https://github.com/google/googletest/issues/3659 - if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - MY_CHECK_AND_SET_COMPILER_FLAG("-U__cpp_char8_t") + # this is a workaround for gtest's bug still not closed https://github.com/google/googletest/issues/3659 + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + my_check_and_set_compiler_flag("-U__cpp_char8_t") endif() - include_directories(${EXTERNAL_INSTALL_LOCATION}/include) # lib64 for RPM-based distros link_directories(${EXTERNAL_INSTALL_LOCATION}/lib ${EXTERNAL_INSTALL_LOCATION}/lib64) set(GTEST_LIBRARIES gtest gtest_main pthread) include(GoogleTest) - #GoogleTest tests + # GoogleTest tests add_executable(rowgroup_tests rowgroup-tests.cpp) target_compile_options(rowgroup_tests PRIVATE -Wno-sign-compare) add_dependencies(rowgroup_tests googletest) @@ -62,7 +61,9 @@ if (WITH_UNITTESTS) add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp) target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare) add_dependencies(column_scan_filter_tests googletest) - target_link_libraries(column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) + target_link_libraries( + column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc + ) gtest_add_tests(TARGET column_scan_filter_tests TEST_PREFIX columnstore:) add_executable(simd_processors simd_processors.cpp) @@ -109,10 +110,20 @@ if (WITH_UNITTESTS) set_tests_properties(columnstore:brm_em_standalone PROPERTIES DISABLED True) endif() -if (WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")) +if(WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")) find_package(benchmark REQUIRED) add_executable(primitives_scan_bench primitives_scan_bench.cpp) - target_include_directories(primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) - target_link_libraries(primitives_scan_bench ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc benchmark::benchmark) + target_include_directories( + primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} + ) + target_link_libraries( + primitives_scan_bench + ${ENGINE_LDFLAGS} + ${ENGINE_WRITE_LIBS} + ${GTEST_LIBRARIES} + processor + dbbc + benchmark::benchmark + ) add_test(NAME columnstore_microbenchmarks:primitives_scan_bench, COMMAND primitives_scan_bench) endif() diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index 59a33bd67..85c702676 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -1,9 +1,7 @@ +include_directories(${ENGINE_COMMON_INCLUDES}) -include_directories( ${ENGINE_COMMON_INCLUDES} ) - - -########### next target ############### -# func_decode_oracle.cpp +# ########## next target ############### +# func_decode_oracle.cpp set(funcexp_LIB_SRCS functor.cpp @@ -144,13 +142,17 @@ set(funcexp_LIB_SRCS func_year.cpp func_yearweek.cpp jsonhelpers.cpp - sql_crypt.cpp) + sql_crypt.cpp +) add_library(funcexp SHARED ${funcexp_LIB_SRCS}) add_dependencies(funcexp loggingcpp) -target_link_libraries(funcexp ${NETSNMP_LIBRARIES} pron pcre2-8) - -install(TARGETS funcexp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) +target_link_libraries(funcexp ${NETSNMP_LIBRARIES} pron pcre2-8 ${MARIADB_STRING_LIBS}) +install( + TARGETS funcexp + DESTINATION ${ENGINE_LIBDIR} + COMPONENT columnstore-engine +) diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index 6f07e1fb9..dcdcd4766 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -10,7 +10,7 @@ set(rowgroup_LIB_SRCS rowaggregation.cpp rowgroup.cpp rowstorage.cpp) add_library(rowgroup SHARED ${rowgroup_LIB_SRCS}) -add_dependencies(rowgroup loggingcpp) +add_dependencies(rowgroup loggingcpp external_boost) target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp) diff --git a/utils/udfsdk/CMakeLists.txt b/utils/udfsdk/CMakeLists.txt index 8b55883c8..baac65ddd 100755 --- a/utils/udfsdk/CMakeLists.txt +++ b/utils/udfsdk/CMakeLists.txt @@ -1,10 +1,15 @@ +include_directories(${ENGINE_COMMON_INCLUDES} ../../dbcon/mysql) +# ########## next target ############### -include_directories( ${ENGINE_COMMON_INCLUDES} - ../../dbcon/mysql ) - -########### next target ############### - -set(udfsdk_LIB_SRCS udfsdk.cpp mcsv1_udaf.cpp allnull.cpp ssq.cpp avg_mode.cpp avgx.cpp distinct_count.cpp) +set(udfsdk_LIB_SRCS + udfsdk.cpp + mcsv1_udaf.cpp + allnull.cpp + ssq.cpp + avg_mode.cpp + avgx.cpp + distinct_count.cpp +) add_definitions(-DMYSQL_DYNAMIC_PLUGIN) @@ -12,11 +17,18 @@ add_library(udfsdk SHARED ${udfsdk_LIB_SRCS}) add_dependencies(udfsdk loggingcpp) -install(TARGETS udfsdk DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) +install( + TARGETS udfsdk + DESTINATION ${ENGINE_LIBDIR} + COMPONENT columnstore-engine +) set(udf_mysql_LIB_SRCS udfmysql.cpp) add_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS}) -install(TARGETS udf_mysql DESTINATION ${MARIADB_PLUGINDIR} COMPONENT columnstore-engine) - +install( + TARGETS udf_mysql + DESTINATION ${MARIADB_PLUGINDIR} + COMPONENT columnstore-engine +) diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index 22e98611e..91be178c8 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -38,7 +38,11 @@ #include "we_tableinfo.h" #include "brmtypes.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #include "boost/ptr_container/ptr_vector.hpp" +#pragma GCC diagnostic pop + #include #include #include @@ -187,19 +191,19 @@ class BulkLoad : public FileOp int fNumOfParser; // total number of parser char fColDelim; // delimits col values within a row - int fNoOfBuffers; // Number of read buffers - int fBufferSize; // Read buffer size - int fFileVbufSize; // Internal file system buffer size - long long fMaxErrors; // Max allowable errors per job - std::string fAlternateImportDir; // Alternate bulk import directory - std::string fErrorDir; // Opt. where error records record - std::string fProcessName; // Application process name + int fNoOfBuffers; // Number of read buffers + int fBufferSize; // Read buffer size + int fFileVbufSize; // Internal file system buffer size + long long fMaxErrors; // Max allowable errors per job + std::string fAlternateImportDir; // Alternate bulk import directory + std::string fErrorDir; // Opt. where error records record + std::string fProcessName; // Application process name static std::vector> fTableInfo; // Vector of Table information - int fNoOfParseThreads; // Number of parse threads - int fNoOfReadThreads; // Number of read threads - boost::thread_group fReadThreads; // Read thread group - boost::thread_group fParseThreads; // Parse thread group - boost::mutex fReadMutex; // Manages table selection by each + int fNoOfParseThreads; // Number of parse threads + int fNoOfReadThreads; // Number of read threads + boost::thread_group fReadThreads; // Read thread group + boost::thread_group fParseThreads; // Parse thread group + boost::mutex fReadMutex; // Manages table selection by each // read thread boost::mutex fParseMutex; // Manages table/buffer/column // selection by each parsing thread diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index 2abc40e63..509c725c7 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -26,7 +26,10 @@ #include "string" #include "vector" #include "boost/thread/mutex.hpp" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #include "boost/ptr_container/ptr_vector.hpp" +#pragma GCC diagnostic pop #include "we_columninfo.h" #include "calpontsystemcatalog.h" #include "dataconvert.h" diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index 8d03cb85d..996661cf9 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -27,7 +27,10 @@ #include #include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #include +#pragma GCC diagnostic pop #include #include