database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

ci: new builds grid, parallel steps

Browse Source
This commit is contained in:
mariadb-RomanNavrotskiy
2022-05-11 12:24:52 +02:00
parent dc82c080f6
commit 194f0e9d64
8 changed files with 179 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

248
.drone.jsonnet
View File

@ -1,9 +1,13 @@
local events = ['pull_request', 'cron'];
local platforms = {
develop: ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'],
'develop-6': ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'],
'develop-5': ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'],
develop: ['centos:7', 'rockylinux:8', 'debian:10', 'debian:11', 'ubuntu:20.04', 'ubuntu:22.04'],
'develop-6': ['centos:7', 'rockylinux:8', 'debian:10', 'debian:11', 'ubuntu:20.04', 'ubuntu:22.04'],
};
local servers = {
develop: ['10.9', '10.6-enterprise'],
'develop-6': ['10.6-enterprise'],
};
local platforms_arm = {
@ -12,80 +16,85 @@ local platforms_arm = {
};
local any_branch = '**';
local platforms_custom = ['centos:7', 'rockylinux:8', 'debian:10', 'ubuntu:20.04'];
local platforms_custom = ['centos:7', 'rockylinux:8', 'debian:10', 'debian:11', 'ubuntu:20.04', 'ubuntu:22.04'];
local platforms_arm_custom = ['rockylinux:8'];
local platforms_mtr = ['centos:7', 'rockylinux:8', 'ubuntu:20.04'];
local server_ref_map = {
develop: '10.8',
'develop-6': '10.6-enterprise',
'develop-5': '10.5',
'**': '10.8',
};
local builddir = 'verylongdirnameforverystrangecpackbehavior';
local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_XPAND=NO -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO ' +
'-DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' +
'-DWITH_EMBEDDED_SERVER=OFF -DWITH_WSREP=OFF ' +
'-DBUILD_CONFIG=mysql_release -DWITH_UNITTESTS=YES -DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=PRE_TEST';
local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CONFIG=mysql_release ' +
'-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ' +
'-DPLUGIN_COLUMNSTORE=YES -DWITH_UNITTESTS=NO ' +
'-DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO ' +
'-DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' +
'-DPLUGIN_GSSAPI=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' +
'-DWITH_EMBEDDED_SERVER=NO -DWITH_WSREP=NO';
local gcc_update_alternatives = 'update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 ';
local clang12_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 ';
local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo ';
local centos7_build_deps = 'yum install -y epel-release centos-release-scl && yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel && ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable ';
local centos8_build_deps = 'dnf install -y gcc-toolset-10 libarchive cmake lz4-devel && . /opt/rh/gcc-toolset-10/enable ';
local rockylinux8_powertools = "dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools ";
local ubuntu18_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives;
local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison ' +
'snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool ' +
'policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel ' +
'expect createrepo ';
local centos7_build_deps = 'yum install -y epel-release centos-release-scl ' +
'&& yum install -y pcre2-devel devtoolset-10 devtoolset-10-gcc cmake3 lz4-devel ' +
'&& ln -s /usr/bin/cmake3 /usr/bin/cmake && . /opt/rh/devtoolset-10/enable ';
local rockylinux8_build_deps = "dnf install -y 'dnf-command(config-manager)' " +
'&& dnf config-manager --set-enabled powertools ' +
'&& dnf install -y gcc-toolset-10 libarchive cmake lz4-devel ' +
'&& . /opt/rh/gcc-toolset-10/enable ';
local debian10_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-12 main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-12 &&' + clang12_update_alternatives;
local opensuse_build_deps = 'zypper install -y clang12 liblz4-devel cmake libboost_system-devel pcre2-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran gcc10 gcc10-c++ && ' + clang12_update_alternatives;
local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata dh-systemd && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i ';
local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata libssl-dev && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i ';
local ubuntu20_04_deps = 'apt update --yes && apt install -y g++-10 git && ' + gcc_update_alternatives;
local platformMap(platform, arch) =
local clang_force = if (arch == 'arm64') then ' && export CXX=/usr/bin/clang++ && export CC=/usr/bin/clang ' else '';
local clang_force = if (arch == 'arm64') then ' && export CXX=/usr/bin/clang++ && export CC=/usr/bin/clang ';
local platform_map = {
'opensuse/leap:15': opensuse_build_deps + ' && zypper ' + rpm_build_deps + clang_force + ' && cmake ' + cmakeflags + ' -DRPM=sles15 && make -j$(nproc) package',
'centos:7': centos7_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos7 && make -j$(nproc) package',
'centos:8': centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos8 && make -j$(nproc) package',
'rockylinux:8': rockylinux8_powertools + ' && ' + centos8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && make -j$(nproc) package',
'debian:10': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=buster' debian/autobake-deb.sh",
'ubuntu:20.04': ubuntu20_04_deps + ' && ' + deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh",
'centos:7': centos7_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos7 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package',
'rockylinux:8': rockylinux8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package',
'debian:10': deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=buster' debian/autobake-deb.sh",
'debian:11': deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=bullseye' debian/autobake-deb.sh",
'ubuntu:20.04': ubuntu20_04_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh",
'ubuntu:22.04': deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=jammy' debian/autobake-deb.sh",
};
platform_map[platform];
local testRun(platform) =
local platform_map = {
'opensuse/leap:15': 'ctest -R columnstore: -j $(nproc) --output-on-failure',
'centos:7': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
'centos:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
'debian:10': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
'debian:11': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
'ubuntu:22.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
};
platform_map[platform];
local testPreparation(platform) =
local platform_map = {
'opensuse/leap:15': 'zypper install -y git boost-devel libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel cppunit-devel snappy-devel cmake',
'centos:7': 'yum -y install epel-release && yum install -y git cppunit-devel cmake3 boost-devel snappy-devel',
'centos:8': 'dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel',
'rockylinux:8': rockylinux8_powertools + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel',
'rockylinux:8': rockylinux8_build_deps + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel',
'debian:10': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake',
'ubuntu:18.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake g++',
'debian:11': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake',
'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake',
'ubuntu:22.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake',
};
platform_map[platform];
local Pipeline(branch, platform, event, arch='amd64') = {
local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'opensuse/leap' || std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb',
local Pipeline(branch, platform, event, arch='amd64', server='10.9') = {
local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb',
local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd',
local mtr_path = if (pkg_format == 'rpm') then '/usr/share/mysql-test' else '/usr/share/mysql/mysql-test',
local socket_path = if (pkg_format == 'rpm') then '/var/lib/mysql/mysql.sock' else '/run/mysqld/mysqld.sock',
@ -94,16 +103,23 @@ local Pipeline(branch, platform, event, arch='amd64') = {
local regression_ref = if (std.split(branch, '-')[0] == 'develop') then branch else 'develop-6',
local regression_tests = if (branch == 'develop') then 'test000.sh' else 'test000.sh,test001.sh',
local branchp = if (branch == '**') then '' else branch,
local result = std.strReplace(std.strReplace(platform, ':', ''), '/', '-'),
local container_tags = if (event == 'cron') then [branch, branch + '-' + std.strReplace(event, '_', '-') + '-${DRONE_BUILD_NUMBER}'] else [branch + '-' + std.strReplace(event, '_', '-') + '-${DRONE_BUILD_NUMBER}'],
local container_version = branch + '/' + event + '/${DRONE_BUILD_NUMBER}/' + arch,
local container_version = branch + '/' + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch,
local server_remote = if (std.split(branch, '-')[0] == 'columnstore' || branch == 'develop-6') then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server',
local server_remote = if (std.endsWith(server, 'enterprise')) then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server',
local sccache_arch = if (arch == 'amd64') then 'x86_64' else 'aarch64',
local get_sccache = 'curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.0/sccache-v0.3.0-' + sccache_arch + '-unknown-linux-musl.tar.gz ' +
'&& tar xzf sccache.tar.gz ' +
'&& install sccache*/sccache /usr/local/bin/',
local pipeline = self,
publish(step_prefix='pkg', eventp=event + '/${DRONE_BUILD_NUMBER}'):: {
name: 'publish ' + step_prefix,
depends_on: [std.strReplace(step_prefix, 'latest', '')],
image: 'plugins/s3-sync',
when: {
status: ['success', 'failure'],
@ -116,8 +132,8 @@ local Pipeline(branch, platform, event, arch='amd64') = {
secret_key: {
from_secret: 'aws_secret_access_key',
},
source: 'result',
target: branchp + '/' + eventp + '/' + arch + '/' + std.strReplace(std.strReplace(platform, ':', ''), '/', '-'),
source: result,
target: branchp + '/' + eventp + '/' + server + '/' + arch + '/' + result,
delete: 'true',
},
},
@ -134,15 +150,16 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
smoke:: {
name: 'smoke',
depends_on: ['pkg'],
image: 'docker',
volumes: [pipeline._volumes.docker],
commands: [
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
'docker cp result smoke$${DRONE_BUILD_NUMBER}:/',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release which rsyslog hostname procps-ng && yum install -y /result/*.' + pkg_format + '"' else '',
'docker cp ' + result + ' smoke$${DRONE_BUILD_NUMBER}:/',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release which rsyslog hostname procps-ng && yum install -y /' + result + '/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname && apt install -y -f /result/*.' + pkg_format + '"' else '',
if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname && apt install -y -f /' + result + '/*.' + pkg_format + '"',
'sleep $${SMOKE_DELAY_SECONDS:-1s}',
// start mariadb and mariadb-columnstore services and run simple query
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb-columnstore',
@ -156,15 +173,15 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
mtr:: {
name: 'mtr',
depends_on: ['smoke'],
image: 'docker:git',
volumes: [pipeline._volumes.docker],
commands: [
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env MYSQL_TEST_DIR=' + mtr_path + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
'docker cp result mtr$${DRONE_BUILD_NUMBER}:/',
if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog patch perl-Data-Dumper-Concise perl-Memoize-ExpireLRU && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils which rsyslog hostname patch perl-Data-Dumper perl-Getopt-Long perl-Memoize perl-Time-HiRes cracklib-dicts procps-ng && yum install -y /result/*.' + pkg_format + '"' else '',
'docker cp ' + result + ' mtr$${DRONE_BUILD_NUMBER}:/',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils which rsyslog hostname patch perl-Data-Dumper perl-Getopt-Long perl-Memoize perl-Time-HiRes cracklib-dicts procps-ng && yum install -y /' + result + '/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname patch && apt install -y -f /result/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y rsyslog hostname patch && apt install -y -f /' + result + '/*.' + pkg_format + '"' else '',
'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} chown -R mysql:mysql ' + mtr_path,
// disable systemd 'ProtectSystem' (we need to write to /usr/share/)
@ -172,6 +189,9 @@ local Pipeline(branch, platform, event, arch='amd64') = {
'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl daemon-reload',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl start mariadb',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} mariadb -e "create database if not exists test;"',
// set memory allowonce for hashjoin operation
'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 60%"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl restart mariadb-columnstore',
// delay mtr for manual debugging on live instance
'sleep $${MTR_DELAY_SECONDS:-1s}',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --max-test-fail=0 --suite=columnstore/basic,columnstore/bugfixes"',
@ -179,6 +199,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
mtrlog:: {
name: 'mtrlog',
depends_on: ['mtr'],
image: 'docker',
volumes: [pipeline._volumes.docker],
commands: [
@ -190,7 +211,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
'docker exec -t mtr$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"',
'echo "---------- end columnstore debug log ----------"',
'echo "---------- end columnstore debug log ----------"',
'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/result/mtr-logs || echo "missing ' + mtr_path + '/var/log"',
'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/' + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"',
'docker stop mtr$${DRONE_BUILD_NUMBER} && docker rm mtr$${DRONE_BUILD_NUMBER} || echo "cleanup mtr failure"',
],
when: {
@ -199,6 +220,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
regression:: {
name: 'regression',
depends_on: ['smoke'],
image: 'docker:git',
[if event == 'cron' then 'failure']: 'ignore',
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
@ -218,15 +240,14 @@ local Pipeline(branch, platform, event, arch='amd64') = {
'cd ..',
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
// copy packages, regresssion test suite and storage manager unit test binary to the instance
'docker cp result regression$${DRONE_BUILD_NUMBER}:/',
'docker cp ' + result + ' regression$${DRONE_BUILD_NUMBER}:/',
'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/',
'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/',
// check storage-manager unit test binary file
'docker exec -t regression$${DRONE_BUILD_NUMBER} ls -l /storage-manager',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils tar lz4 wget which rsyslog hostname procps-ng && yum install -y /result/*.' + pkg_format + '"' else '',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release diffutils tar lz4 wget which rsyslog hostname procps-ng && yum install -y /' + result + '/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y tar liblz4-tool wget rsyslog hostname && apt install -y -f /result/*.' + pkg_format + '"' else '',
if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "zypper install -y gzip tar lz4 wget which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '',
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y tar liblz4-tool wget rsyslog hostname && apt install -y -f /' + result + '/*.' + pkg_format + '"' else '',
// copy test data for regression test suite
'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"',
// set mariadb lower_case_table_names=1 config option
@ -247,6 +268,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
smokelog:: {
name: 'smokelog',
depends_on: ['smoke'],
image: 'docker',
volumes: [pipeline._volumes.docker],
commands: [
@ -265,6 +287,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
regressionlog:: {
name: 'regressionlog',
depends_on: ['regression'],
image: 'docker',
volumes: [pipeline._volumes.docker],
commands: [
@ -272,9 +295,9 @@ local Pipeline(branch, platform, event, arch='amd64') = {
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} cat go.log || echo "missing go.log"',
'echo "---------- end columnstore regression short report ----------"',
'echo',
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/result/ || echo "missing testErrorLogs.tgz"',
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"',
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"',
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/result/ || echo "missing testErrorLogs.tgz"',
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"',
'docker stop regression$${DRONE_BUILD_NUMBER} && docker rm regression$${DRONE_BUILD_NUMBER} || echo "cleanup regression failure"',
],
when: {
@ -283,6 +306,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
dockerfile:: {
name: 'dockerfile',
depends_on: ['publish pkg'],
image: 'alpine/git',
commands: [
'git clone --depth 1 https://github.com/mariadb-corporation/mariadb-skysql-columnstore-docker docker',
@ -291,6 +315,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
dockerhub:: {
name: 'dockerhub',
depends_on: ['dockerfile'],
image: 'plugins/docker',
environment: {
VERSION: container_version,
@ -313,7 +338,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
kind: 'pipeline',
type: 'docker',
name: std.join(' ', [branch, platform, event, arch]),
name: std.join(' ', [branch, platform, event, arch, server]),
platform: { arch: arch },
// [if arch == 'arm64' then 'node']: { arch: 'arm64' },
clone: { depth: 10 },
@ -332,9 +357,9 @@ local Pipeline(branch, platform, event, arch='amd64') = {
image: 'alpine/git',
volumes: [pipeline._volumes.mdb],
environment: {
SERVER_REF: '${SERVER_REF:-' + server_ref_map[branch] + '}',
SERVER_REF: '${SERVER_REF:-' + server + '}',
SERVER_REMOTE: '${SERVER_REMOTE:-' + server_remote + '}',
SERVER_SHA: '${SERVER_SHA:-' + server_ref_map[branch] + '}',
SERVER_SHA: '${SERVER_SHA:-' + server + '}',
},
commands: [
'echo $$SERVER_REF',
@ -351,46 +376,56 @@ local Pipeline(branch, platform, event, arch='amd64') = {
},
{
name: 'build',
depends_on: ['clone-mdb'],
image: img,
volumes: [pipeline._volumes.mdb],
environment: {
DEBIAN_FRONTEND: 'noninteractive',
DEB_BUILD_OPTIONS: 'parallel=4',
BUILDPACKAGE_FLAGS: '-b', // Save time and produce only binary packages, not source
AWS_ACCESS_KEY_ID: {
from_secret: 'aws_access_key_id',
},
AWS_SECRET_ACCESS_KEY: {
from_secret: 'aws_secret_access_key',
},
SCCACHE_BUCKET: 'cs-sccache',
SCCACHE_S3_USE_SSL: 'true',
SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + '${DRONE_PULL_REQUEST}',
//SCCACHE_ERROR_LOG: '/tmp/sccache_log.txt',
//SCCACHE_LOG: 'debug',
},
commands: [
'cd /mdb/' + builddir,
"sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh",
if (std.startsWith(server, '10.6')) then "sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control",
// Remove Debian build flags that could prevent ColumnStore from building
"sed '/-DPLUGIN_COLUMNSTORE=NO/d' -i debian/rules",
"sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules",
// Disable dh_missing strict check for missing files
'sed -i s/--fail-missing/--list-missing/ debian/rules',
// Tweak debian packaging stuff
"sed -i -e '/Package: mariadb-backup/,/^$/d' debian/control",
"sed -i -e '/Package: mariadb-plugin-connect/,/^$/d' debian/control",
"sed -i -e '/Package: mariadb-plugin-gssapi*/,/^$/d' debian/control",
"sed -i -e '/Package: mariadb-plugin-xpand*/,/^$/d' debian/control",
'sed "/Package: mariadb-plugin-mroonga/,/^$/d" -i debian/control',
'sed "/Package: mariadb-plugin-rocksdb/,/^$/d" -i debian/control',
'sed "/Package: mariadb-plugin-spider/,/^$/d" -i debian/control',
'sed "/Package: mariadb-plugin-oqgraph/,/^$/d" -i debian/control',
'sed "/ha_sphinx.so/d" -i debian/mariadb-server-*.install',
'sed "/Package: libmariadbd19/,/^$/d" -i debian/control',
'sed "/Package: libmariadbd-dev/,/^$/d" -i debian/control',
// Disable Galera
"sed -i -e 's/Depends: galera.*/Depends:/' debian/control",
"sed -i -e 's/\"galera-enterprise-4\"//' cmake/cpack_rpm.cmake",
"sed -i -e '/wsrep/d' debian/mariadb-server-*.install",
// Leave test package for mtr
"sed -i '/(mariadb|mysql)-test/d;/-test/d' debian/autobake-deb.sh",
"sed -i '/test-embedded/d' debian/mariadb-test.install",
// Deb dependencies from server scripts
if (pkg_format == 'deb') then "apt-cache madison liburing-dev | grep liburing-dev || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules && apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules && sed '/libfmt-dev/d' -i debian/control" else '',
'for i in mariadb-backup mariadb-plugin libmariadbd; do sed -i "/Package: $i.*/,/^$/d" debian/control; done',
"sed -i 's/Depends: galera.*/Depends:/' debian/control",
'for i in wsrep ha_sphinx embedded; do sed -i /$i/d debian/*.install; done',
// Install build dependencies for deb
if (pkg_format == 'deb') then "apt-cache madison liburing-dev | grep liburing-dev || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules && apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules && sed '/libfmt-dev/d' -i debian/control",
// Change plugin_maturity level
// "sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt",
if (pkg_format == 'deb') then 'apt update -y && apt install -y curl' else 'yum install -y curl',
get_sccache,
testPreparation(platform),
// disable LTO for 22.04 for now
if (platform == 'ubuntu:22.04') then 'apt install -y lto-disabled-list && for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do echo "$i any" >> /usr/share/lto-disabled-list/lto-disabled-list; done && grep mariadb /usr/share/lto-disabled-list/lto-disabled-list',
platformMap(platform, arch),
if (pkg_format == 'rpm') then 'createrepo .' else 'dpkg-scanpackages ../ | gzip > ../Packages.gz',
'sccache --show-stats',
'mkdir ' + result,
if (pkg_format == 'rpm') then 'mv *.' + pkg_format + ' ' + result + '/' else 'mv ../*.' + pkg_format + ' ' + result + '/',
if (pkg_format == 'rpm') then 'createrepo ' + result else 'dpkg-scanpackages ' + result + ' | gzip > ' + result + '/Packages.gz',
],
},
{
name: 'unittests',
depends_on: ['build'],
image: img,
volumes: [pipeline._volumes.mdb],
environment: {
@ -403,25 +438,26 @@ local Pipeline(branch, platform, event, arch='amd64') = {
],
},
{
name: 'list pkgs',
name: 'pkg',
depends_on: ['unittests'],
image: 'docker:git',
volumes: [pipeline._volumes.mdb],
commands: [
'cd /mdb/' + builddir,
'mkdir -p /drone/src/result',
'echo "engine: $DRONE_COMMIT" > buildinfo.txt',
'echo "server: $$(git rev-parse HEAD)" >> buildinfo.txt',
'echo "buildNo: $DRONE_BUILD_NUMBER" >> buildinfo.txt',
'cp -r ' + (if (pkg_format == 'deb') then '../Packages.gz ../' else 'repodata ') + '*.' + pkg_format + ' buildinfo.txt /drone/src/result/',
'ls -l /drone/src/result',
'mv buildinfo.txt ' + result + '/',
'mv ' + result + ' /drone/src/',
'ls -l /drone/src/' + result,
'echo "check columnstore package:"',
'ls -l /drone/src/result | grep columnstore',
'ls -l /drone/src/' + result + ' | grep columnstore',
],
},
] +
[pipeline.publish()] +
(if (event == 'cron') || (event == 'push') then [pipeline.publish('pkg latest', 'latest')] else []) +
(if (event != 'custom') && (platform == 'rockylinux:8') && (arch == 'amd64') && (branch != 'develop-5') then [pipeline.dockerfile] + [pipeline.dockerhub] else []) +
(if (event != 'custom') && (platform == 'rockylinux:8') && (arch == 'amd64') && (server == '10.6-enterprise') then [pipeline.dockerfile] + [pipeline.dockerhub] else []) +
[pipeline.smoke] +
[pipeline.smokelog] +
(if (std.member(platforms_mtr, platform)) then [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish('mtr')] else []) +
@ -439,7 +475,7 @@ local Pipeline(branch, platform, event, arch='amd64') = {
} else {}),
};
local FinalPipeline(branch, event, arch='amd64') = {
local FinalPipeline(branch, event) = {
kind: 'pipeline',
name: std.join(' ', ['after', branch, event]),
steps: [
@ -463,29 +499,45 @@ local FinalPipeline(branch, event, arch='amd64') = {
'failure',
],
} + (if event == 'cron' then { cron: ['nightly-' + std.strReplace(branch, '.', '-')] } else {}),
depends_on: std.map(function(p) std.join(' ', [branch, p, event, arch]), platforms[branch]) +
(if branch == 'develop' then std.map(function(p) std.join(' ', [branch, p, event, 'arm64']), platforms_arm[branch]) else []),
depends_on: std.map(function(p) std.join(' ', ['develop', p, event, 'amd64', '10.9']), platforms.develop) +
std.map(function(p) std.join(' ', ['develop', p, event, 'amd64', '10.6-enterprise']), platforms.develop) +
std.map(function(p) std.join(' ', ['develop', p, event, 'arm64', '10.9']), platforms_arm.develop) +
std.map(function(p) std.join(' ', ['develop', p, event, 'arm64', '10.6-enterprise']), platforms_arm.develop) +
std.map(function(p) std.join(' ', ['develop-6', p, event, 'amd64', '10.6-enterprise']), platforms['develop-6']) +
std.map(function(p) std.join(' ', ['develop-6', p, event, 'amd64', '10.9']), platforms['develop-6']) +
std.map(function(p) std.join(' ', ['develop-6', p, event, 'arm64', '10.6-enterprise']), platforms_arm['develop-6']),
};
[
Pipeline(b, p, e)
Pipeline(b, p, e, 'amd64', s)
for b in std.objectFields(platforms)
for p in platforms[b]
for s in servers[b]
for e in events
] +
[
Pipeline(b, p, e, 'arm64')
Pipeline('develop', 'rockylinux:8', e, 'amd64', '10.6')
for e in events
] +
[
Pipeline('develop-6', 'ubuntu:22.04', e, 'amd64', '10.9')
for e in events
] +
[
Pipeline(b, p, e, 'arm64', s)
for b in std.objectFields(platforms_arm)
for p in platforms_arm[b]
for s in servers[b]
for e in events
] +
[
FinalPipeline(b, e)
for b in std.objectFields(platforms)
for e in events
]
+
//[
// FinalPipeline(b, 'cron')
// for b in std.objectFields(platforms)
//] +
[
Pipeline(any_branch, p, 'custom')

2
debian/control vendored
View File

@ -4,7 +4,7 @@ Depends: binutils,
libjemalloc1 | libjemalloc2,
libsnappy1 | libsnappy1v5,
liblz4-1,
mariadb-server-10.8 (= ${server:Version}),
mariadb-server (= ${server:Version}),
net-tools,
python3,
${misc:Depends},

2
mysql-test/columnstore/basic/disabled.def
View File

@ -2,3 +2,5 @@ mcs80_set_operations : BUG#MCOL-4273 2020-08-27 susil.behera@mariadb.com
mcs211_idbExtentId_function : 2021-07-12 david.halla@mariadb.com
mcs212_idbExtentMax_function : 2020-11-30 bharath.bokka@mariadb.com
mcs213_idbExtentMin_function : 2020-11-30 bharath.bokka@mariadb.com
mcs118_charset_negative : mcs error code has changed 2022-07-07 roman.navrotskiy@mariadb.com
mcs16_functions_define_call_drop : 2022-07-08 roman.navrotskiy@mariadb.com

4
mysql-test/columnstore/basic/r/mcol641-create.result
View File

@ -62,11 +62,11 @@ test_mcol641_create cs4 d3 decimal 8 2 NULL 18 18
CREATE TABLE cs5(d1 DECIMAL(39)) ENGINE=columnstore;
ERROR HY000: Internal error: CAL0009: (3)Create table failed due to Syntax error: The maximum precision (total number of digits) that can be specified is 38
CREATE TABLE cs5(d1 DECIMAL(38,39)) ENGINE=columnstore;
ERROR 42000: Too big scale 39 specified for 'd1'. Maximum is 38
Got one of the listed errors
ALTER TABLE cs1 ADD COLUMN d10 DECIMAL(40);
ERROR HY000: Internal error: CAL0001: Alter table Failed: AlterTableProcessor::addColumn Syntax error: The maximum precision (total number of digits) that can be specified is 38
ALTER TABLE cs1 ADD COLUMN d11 DECIMAL(38,40);
ERROR 42000: Too big scale 40 specified for 'd11'. Maximum is 38
Got one of the listed errors
DROP TABLE cs1;
DROP TABLE cs2;
DROP TABLE cs3;

4
mysql-test/columnstore/basic/r/mcs69_cast_data_types.result
View File

@ -33,7 +33,7 @@ NULL NULL NULL NULL NULL
SELECT CAST(t1_int AS DECIMAL(4,5)) FROM t1;
ERROR 42000: For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column '')
SELECT CAST(t1_int AS DECIMAL(66,6)) FROM t1;
ERROR 42000: Too big precision 66 specified for 't1_int'. Maximum is 65
Got one of the listed errors
SELECT CAST(t1_int AS DECIMAL(64,63)) FROM t1;
ERROR 42000: Too big scale 63 specified for 't1_int'. Maximum is 38
Got one of the listed errors
DROP DATABASE mcs69_db;

4
mysql-test/columnstore/basic/t/mcol641-create.test
View File

@ -39,11 +39,11 @@ SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITI
# This must return an error
--error 1815
CREATE TABLE cs5(d1 DECIMAL(39)) ENGINE=columnstore;
--error 1425
--error 1425,1425
CREATE TABLE cs5(d1 DECIMAL(38,39)) ENGINE=columnstore;
--error 1815
ALTER TABLE cs1 ADD COLUMN d10 DECIMAL(40);
--error 1425
--error 1425,1425
ALTER TABLE cs1 ADD COLUMN d11 DECIMAL(38,40);
DROP TABLE cs1;

4
mysql-test/columnstore/basic/t/mcs69_cast_data_types.test
View File

@ -24,9 +24,9 @@ SELECT CAST(t1_int AS DECIMAL(6,5)), CAST(t1_decimal AS DECIMAL(6,5)), CAST(t1_c
--error ER_M_BIGGER_THAN_D
SELECT CAST(t1_int AS DECIMAL(4,5)) FROM t1;
--error ER_TOO_BIG_PRECISION
--error ER_TOO_BIG_PRECISION,ER_TOO_BIG_PRECISION
SELECT CAST(t1_int AS DECIMAL(66,6)) FROM t1;
--error ER_TOO_BIG_SCALE
--error ER_TOO_BIG_SCALE,ER_TOO_BIG_SCALE
SELECT CAST(t1_int AS DECIMAL(64,63)) FROM t1;
# Clean UP

27
utils/common/vlarray.h
View File

@ -16,13 +16,18 @@
MA 02110-1301, USA. */
#pragma once
#include <cstddef>
#include <stdexcept>
#include <type_traits>
namespace utils
{
template <typename T, size_t SIZE = 64>
template <typename T, int SIZE = 64>
class VLArray
{
public:
VLArray(size_t sz) : sz(sz), stack_storage(NULL), dyn_storage(NULL), ptr(NULL)
VLArray(int sz) : sz(sz), stack_storage(NULL), dyn_storage(NULL), ptr(NULL)
{
if (sz > SIZE)
{
@ -36,9 +41,9 @@ class VLArray
}
}
VLArray(size_t sz, const T& initval) : VLArray(sz)
VLArray(int sz, const T& initval) : VLArray(sz)
{
for (size_t i = 0; i < sz; ++i)
for (int i = 0; i < sz; ++i)
ptr[i] = initval;
}
@ -58,13 +63,13 @@ class VLArray
// we cannot use `delete [] stack_storage` here so call d-tors explicitly
if (!std::is_trivially_destructible<T>::value)
{
for (size_t i = 0; i < sz; ++i)
for (int i = 0; i < sz; ++i)
stack_storage[i].~T();
}
}
}
size_t size() const
int size() const
{
return sz;
}
@ -77,16 +82,16 @@ class VLArray
return ptr;
}
const T& operator[](size_t i) const
const T& operator[](int i) const
{
return ptr[i];
}
T& operator[](size_t i)
T& operator[](int i)
{
return ptr[i];
}
const T& at(size_t i) const
const T& at(int i) const
{
if (i >= sz)
{
@ -95,7 +100,7 @@ class VLArray
return ptr[i];
}
T& at(size_t i)
T& at(int i)
{
if (i >= sz)
{
@ -114,7 +119,7 @@ class VLArray
}
private:
const size_t sz;
const int sz;
alignas(T) char stack[SIZE * sizeof(T)];
T* stack_storage;
T* dyn_storage;