You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-30 19:23:07 +03:00
MCOL-5496: Fix CI adding CMAPI steps.
[fix] deb packages deps commands [add] several additional local variables [fix] packages url [fix] smoke step [fix] mtr step [fix] regression step [add] cmapipython, cmapibuild, cmapitest and cmapilog steps [fix] dockerfile step [fix] build step to include cmapi package on repodata creating [fix] multi_node_mtr step [fix] pkg step [add] cmapi steps to pipelines [fix] cmapi/CMakeLists.txt to prevent cmake byte-compile .py files for rpm packages [add] setup-repo.sh file [fix] now use packages from the repos in tests steps
This commit is contained in:
mariadb-RomanNavrotskiy
committed by
Alan Mologorsky
Alan Mologorsky
parent
a079a2c944
commit
1eca477485
177
.drone.jsonnet
177
.drone.jsonnet
@ -57,7 +57,7 @@ local rockylinux9_build_deps = "dnf install -y 'dnf-command(config-manager)' " +
|
||||
'&& dnf install -y pcre2-devel lz4-devel gcc gcc-c++';
|
||||
|
||||
local debian11_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' && ' + clang_update_alternatives;
|
||||
local ubuntu20_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' &&' + clang_update_alternatives;
|
||||
local ubuntu20_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' && ' + clang_update_alternatives;
|
||||
|
||||
local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata libssl-dev && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i ';
|
||||
local turnon_clang = 'export CC=/usr/bin/clang; export CXX=/usr/bin/clang++ ';
|
||||
@ -76,9 +76,7 @@ local platformMap(platform, arch) =
|
||||
'ubuntu:22.04': bootstrap_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=jammy' debian/autobake-deb.sh",
|
||||
};
|
||||
local result = std.strReplace(std.strReplace(platform, ':', ''), '/', '-');
|
||||
"export CLICOLOR_FORCE=1; " + platform_map[platform] + " | storage/columnstore/columnstore/build/ansi2txt.sh " + result + "/build.log";
|
||||
|
||||
|
||||
'export CLICOLOR_FORCE=1; ' + platform_map[platform] + ' | storage/columnstore/columnstore/build/ansi2txt.sh ' + result + '/build.log';
|
||||
|
||||
|
||||
local testRun(platform) =
|
||||
@ -109,6 +107,8 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb',
|
||||
local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd',
|
||||
local mtr_path = if (pkg_format == 'rpm') then '/usr/share/mysql-test' else '/usr/share/mysql/mysql-test',
|
||||
local cmapi_path = '/usr/share/columnstore/cmapi',
|
||||
local etc_path = '/etc/columnstore',
|
||||
local socket_path = if (pkg_format == 'rpm') then '/var/lib/mysql/mysql.sock' else '/run/mysqld/mysqld.sock',
|
||||
local config_path_prefix = if (pkg_format == 'rpm') then '/etc/my.cnf.d/' else '/etc/mysql/mariadb.conf.d/50-',
|
||||
local img = if (platform == 'centos:7' || platform == 'rockylinux:8') then platform else 'romcheck/' + std.strReplace(platform, '/', '-'),
|
||||
@ -119,7 +119,9 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
local brancht = if (branch == '**') then '' else branch + '-',
|
||||
local result = std.strReplace(std.strReplace(platform, ':', ''), '/', '-'),
|
||||
|
||||
local publish_pkg_url = "https://cspkg.s3.amazonaws.com/index.html?prefix=" + branchp + event + "/${DRONE_BUILD_NUMBER}/" + server + "/" + arch + "/" + result + "/",
|
||||
local publish_pkg_url = 'https://cspkg.s3.amazonaws.com/index.html?prefix=' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/',
|
||||
|
||||
local packages_url = 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server,
|
||||
|
||||
local container_tags = if (event == 'cron') then [brancht + std.strReplace(event, '_', '-') + '${DRONE_BUILD_NUMBER}', brancht] else [brancht + std.strReplace(event, '_', '-') + '${DRONE_BUILD_NUMBER}'],
|
||||
local container_version = branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch,
|
||||
@ -208,7 +210,6 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} cat go.log || echo "missing go.log"',
|
||||
],
|
||||
},
|
||||
|
||||
_volumes:: {
|
||||
mdb: {
|
||||
name: 'mdb',
|
||||
@ -221,21 +222,20 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
},
|
||||
smoke:: {
|
||||
name: 'smoke',
|
||||
depends_on: ['pkg'],
|
||||
depends_on: ['publish pkg'],
|
||||
image: 'docker',
|
||||
volumes: [pipeline._volumes.docker],
|
||||
commands: [
|
||||
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
|
||||
'docker cp ' + result + ' smoke$${DRONE_BUILD_NUMBER}:/',
|
||||
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget procps-ng"',
|
||||
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
|
||||
if (pkg_format == 'rpm') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gdb gawk epel-release which rsyslog hostname procps-ng"' else 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y gdb gawk rsyslog hostname procps wget"',
|
||||
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
|
||||
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y procps wget"',
|
||||
'docker exec -t smoke$${DRONE_BUILD_NUMBER} mkdir core',
|
||||
'docker exec -t smoke$${DRONE_BUILD_NUMBER} chmod 777 core',
|
||||
'docker exec -t smoke$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_smoke_core_dump.%p"',
|
||||
'docker cp core_dumps/. smoke$${DRONE_BUILD_NUMBER}:/',
|
||||
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y gdb gawk epel-release which rsyslog hostname procps-ng && yum install -y /' + result + '/*.' + pkg_format + '"' else '',
|
||||
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y gdb gawk rsyslog hostname && apt install -y -f /' + result + '/*.' + pkg_format + '"',
|
||||
'docker cp setup-repo.sh smoke$${DRONE_BUILD_NUMBER}:/',
|
||||
'docker exec -t smoke$${DRONE_BUILD_NUMBER} /setup-repo.sh',
|
||||
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore"' else 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine"',
|
||||
'sleep $${SMOKE_DELAY_SECONDS:-1s}',
|
||||
// start mariadb and mariadb-columnstore services and run simple query
|
||||
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb',
|
||||
@ -258,8 +258,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}',
|
||||
},
|
||||
commands: [
|
||||
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --shm-size=500m --env MYSQL_TEST_DIR=' + mtr_path + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
|
||||
'docker cp ' + result + ' mtr$${DRONE_BUILD_NUMBER}:/',
|
||||
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --shm-size=500m --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
|
||||
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget tar lz4 procps-ng"',
|
||||
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
|
||||
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y procps wget tar liblz4-tool"',
|
||||
@ -267,9 +266,12 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
'docker exec -t mtr$${DRONE_BUILD_NUMBER} chmod 777 core',
|
||||
'docker exec -t mtr$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_mtr_core_dump.%p"',
|
||||
'docker cp core_dumps/. mtr$${DRONE_BUILD_NUMBER}:/',
|
||||
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb epel-release diffutils which rsyslog hostname patch perl cracklib-dicts procps-ng && yum install -y /' + result + '/*.' + pkg_format + '"' else '',
|
||||
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb epel-release diffutils which rsyslog hostname patch perl cracklib-dicts procps-ng"' else '',
|
||||
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
|
||||
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget gawk gdb rsyslog hostname patch && apt install -y -f /' + result + '/*.' + pkg_format + '"' else '',
|
||||
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget gawk gdb rsyslog hostname patch"' else '',
|
||||
'docker cp setup-repo.sh mtr$${DRONE_BUILD_NUMBER}:/',
|
||||
'docker exec -t mtr$${DRONE_BUILD_NUMBER} /setup-repo.sh',
|
||||
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore mariadb-test"' else 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"',
|
||||
'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/',
|
||||
'docker exec -t mtr$${DRONE_BUILD_NUMBER} chown -R mysql:mysql ' + mtr_path,
|
||||
// disable systemd 'ProtectSystem' (we need to write to /usr/share/)
|
||||
@ -320,7 +322,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
},
|
||||
prepare_regression:: {
|
||||
name: 'prepare regression',
|
||||
depends_on: ['mtr'],
|
||||
depends_on: ['mtr', 'publish pkg', 'publish cmapi build'],
|
||||
when: {
|
||||
status: ['success', 'failure'],
|
||||
},
|
||||
@ -347,12 +349,9 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
'cd mariadb-columnstore-regression-test',
|
||||
'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD',
|
||||
'cd ..',
|
||||
'docker run --shm-size=500m --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --ulimit core=-1 --name regression$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
|
||||
// copy packages, regresssion test suite and storage manager unit test binary to the instance
|
||||
'docker cp ' + result + ' regression$${DRONE_BUILD_NUMBER}:/',
|
||||
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y procps-ng wget elfutils"',
|
||||
'docker run --shm-size=500m --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
|
||||
if (pkg_format == 'rpm') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb gcc-c++ epel-release diffutils tar findutils lz4 wget which rsyslog hostname procps-ng elfutils"' else 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget tar liblz4-tool procps wget findutils gawk gdb rsyslog hostname g++"',
|
||||
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
|
||||
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y procps wget elfutils"',
|
||||
'docker exec -t regression$${DRONE_BUILD_NUMBER} mkdir core',
|
||||
'docker exec -t regression$${DRONE_BUILD_NUMBER} chmod 777 core',
|
||||
'docker exec -t regression$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_regression_core_dump.%p"',
|
||||
@ -363,11 +362,11 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/',
|
||||
// check storage-manager unit test binary file
|
||||
'docker exec -t regression$${DRONE_BUILD_NUMBER} ls -l /storage-manager',
|
||||
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb gcc-c++ epel-release diffutils tar findutils lz4 wget which rsyslog hostname procps-ng && yum install -y /' + result + '/*.' + pkg_format + '"' else '',
|
||||
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
|
||||
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget tar liblz4-tool wget findutils gawk gdb rsyslog hostname && apt install -y -f g++ /' + result + '/*.' + pkg_format + '"' else '',
|
||||
// copy test data for regression test suite
|
||||
'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"',
|
||||
'docker cp setup-repo.sh regression$${DRONE_BUILD_NUMBER}:/',
|
||||
'docker exec -t regression$${DRONE_BUILD_NUMBER} /setup-repo.sh',
|
||||
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore"' else 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine"',
|
||||
// set mariadb lower_case_table_names=1 config option
|
||||
'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf',
|
||||
// set default client character set to utf-8
|
||||
@ -387,7 +386,6 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
'docker exec -t regression$${DRONE_BUILD_NUMBER} /usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester',
|
||||
],
|
||||
},
|
||||
|
||||
smokelog:: {
|
||||
name: 'smokelog',
|
||||
depends_on: ['smoke'],
|
||||
@ -413,6 +411,29 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
status: ['success', 'failure'],
|
||||
},
|
||||
},
|
||||
cmapilog:: {
|
||||
name: 'cmapi log',
|
||||
depends_on: ['cmapi test'],
|
||||
image: 'docker',
|
||||
volumes: [pipeline._volumes.docker],
|
||||
commands: [
|
||||
'echo "---------- start mariadb service logs ----------"',
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"',
|
||||
'echo "---------- end mariadb service logs ----------"',
|
||||
'echo',
|
||||
'echo "---------- start columnstore debug log ----------"',
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"',
|
||||
'echo "---------- end columnstore debug log ----------"',
|
||||
'echo "---------- start cmapi log ----------"',
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi cmapi_server.log"',
|
||||
'echo "---------- end cmapi log ----------"',
|
||||
'ls -l /drone/src/' + result,
|
||||
'docker stop cmapi$${DRONE_BUILD_NUMBER} && docker rm cmapi$${DRONE_BUILD_NUMBER} || echo "cleanup cmapi failure"',
|
||||
],
|
||||
when: {
|
||||
status: ['success', 'failure'],
|
||||
},
|
||||
},
|
||||
regressionlog:: {
|
||||
name: 'regressionlog',
|
||||
depends_on: [regression_tests[std.length(regression_tests) - 1]],
|
||||
@ -443,7 +464,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
},
|
||||
dockerfile:: {
|
||||
name: 'dockerfile',
|
||||
depends_on: ['publish pkg'],
|
||||
depends_on: ['publish pkg', 'publish cmapi build'],
|
||||
//failure: 'ignore',
|
||||
image: 'alpine/git',
|
||||
commands: [
|
||||
@ -463,7 +484,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
// branchp has slash if not empty
|
||||
MCS_BASEURL: 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/',
|
||||
CMAPI_REPO: 'cmapi',
|
||||
CMAPI_BASEURL: 'https://cspkg.s3.amazonaws.com/cmapi/develop/latest/' + arch + '/',
|
||||
CMAPI_BASEURL: 'https://cspkg.s3.amazonaws.com/' + branchp + event + '/${DRONE_BUILD_NUMBER}/' + server + '/' + arch + '/' + result + '/',
|
||||
},
|
||||
settings: {
|
||||
repo: 'mariadb/enterprise-columnstore-dev',
|
||||
@ -479,6 +500,76 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
},
|
||||
},
|
||||
},
|
||||
cmapipython:: {
|
||||
name: 'cmapi python',
|
||||
image: img,
|
||||
volumes: [pipeline._volumes.mdb],
|
||||
environment: {
|
||||
PYTHON_URL_AMD64: 'https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst',
|
||||
PYTHON_URL_ARM64: 'https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst',
|
||||
},
|
||||
commands: [
|
||||
'cd cmapi',
|
||||
'%s install -y wget zstd findutils gcc' % if (pkg_format == 'rpm') then 'yum install -y epel-release && yum makecache && yum ' else 'apt update && apt',
|
||||
'wget -qO- $${PYTHON_URL_' + std.asciiUpper(arch) + '} | tar --use-compress-program=unzstd -xf - -C ./',
|
||||
'mv python pp && mv pp/install python',
|
||||
'chown -R root:root python',
|
||||
if (platform == 'rockylinux:9') then 'yum install -y libxcrypt-compat',
|
||||
if (arch == 'arm64') then 'export CC=gcc',
|
||||
'python/bin/pip3 install -t deps --only-binary :all -r requirements.txt',
|
||||
'./cleanup.sh',
|
||||
'cp cmapi_server/cmapi_server.conf cmapi_server/cmapi_server.conf.default',
|
||||
],
|
||||
},
|
||||
cmapibuild:: {
|
||||
name: 'cmapi build',
|
||||
depends_on: ['cmapi python'],
|
||||
image: img,
|
||||
volumes: [pipeline._volumes.mdb],
|
||||
environment: {
|
||||
DEBIAN_FRONTEND: 'noninteractive',
|
||||
},
|
||||
commands: [
|
||||
'cd cmapi',
|
||||
if (pkg_format == 'rpm') then 'yum install -y cmake make rpm-build libarchive createrepo findutils' else 'apt update && apt install --no-install-recommends -y cmake make dpkg-dev',
|
||||
if (platform == 'centos:7') then 'yum install -y epel-release && yum install -y cmake3 && ln -sf /usr/bin/cmake3 /usr/bin/cmake',
|
||||
'./cleanup.sh',
|
||||
'cmake -D' + std.asciiUpper(pkg_format) + '=1 . && make package',
|
||||
'mkdir ./' + result,
|
||||
'mv -v *.%s ./%s/' % [pkg_format, result],
|
||||
if (pkg_format == 'rpm') then 'createrepo ./' + result else 'dpkg-scanpackages %s | gzip > ./%s/Packages.gz' % [result, result],
|
||||
'mkdir /drone/src/' + result,
|
||||
'yes | cp -vr ./%s /drone/src/' % result,
|
||||
],
|
||||
},
|
||||
cmapitest:: {
|
||||
name: 'cmapi test',
|
||||
depends_on: ['publish cmapi build', 'smoke'],
|
||||
image: 'docker:git',
|
||||
volumes: [pipeline._volumes.docker],
|
||||
environment: {
|
||||
PYTHONPATH: '/usr/share/columnstore/cmapi/deps',
|
||||
},
|
||||
commands: [
|
||||
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env PYTHONPATH=$${PYTHONPATH} --name cmapi$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target',
|
||||
if (pkg_format == 'rpm') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y iproute sudo epel-release which rsyslog hostname procps-ng"' else 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y iproute2 rsyslog hostname procps sudo"',
|
||||
if (pkg_format == 'deb') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
|
||||
if (platform == 'rockylinux:9') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y libxcrypt-compat"',
|
||||
'docker cp setup-repo.sh cmapi$${DRONE_BUILD_NUMBER}:/',
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} /setup-repo.sh',
|
||||
if (pkg_format == 'deb') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore mariadb-columnstore-cmapi"' else 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine MariaDB-columnstore-cmapi"',
|
||||
'cd cmapi',
|
||||
'for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/$${i}/; done',
|
||||
'docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/',
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} systemctl start mariadb-columnstore-cmapi',
|
||||
// set API key to /etc/columnstore/cmapi_server.conf
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "mcs cluster set api-key --key somekey123"',
|
||||
// copy cmapi conf file for test purposes (there are api key already set inside)
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path],
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} systemctl stop mariadb-columnstore-cmapi',
|
||||
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"',
|
||||
],
|
||||
},
|
||||
multi_node_mtr:: {
|
||||
name: 'mtr',
|
||||
depends_on: ['dockerhub'],
|
||||
@ -595,8 +686,9 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
if (platform == 'ubuntu:22.04') then 'apt install -y lto-disabled-list && for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do echo "$i any" >> /usr/share/lto-disabled-list/lto-disabled-list; done && grep mariadb /usr/share/lto-disabled-list/lto-disabled-list',
|
||||
platformMap(platform, arch),
|
||||
'sccache --show-stats',
|
||||
if (pkg_format == 'rpm') then 'mv *.' + pkg_format + ' ' + result + '/' else 'mv ../*.' + pkg_format + ' ' + result + '/',
|
||||
if (pkg_format == 'rpm') then 'createrepo ' + result else 'dpkg-scanpackages ' + result + ' | gzip > ' + result + '/Packages.gz',
|
||||
// move engine and cmapi packages to one dir to make a repo
|
||||
'mv -v -t ./%s/ %s/*.%s /drone/src/cmapi/%s/*.%s ' % [result, if (pkg_format == 'rpm') then '.' else '..', pkg_format, result, pkg_format],
|
||||
if (pkg_format == 'rpm') then 'createrepo ./' + result else 'dpkg-scanpackages %s | gzip > ./%s/Packages.gz' % [result, result],
|
||||
// list storage manager binary
|
||||
'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager',
|
||||
],
|
||||
@ -618,7 +710,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
{
|
||||
name: 'pkg',
|
||||
depends_on: ['unittests'],
|
||||
image: 'docker:git',
|
||||
image: 'alpine/git',
|
||||
when: {
|
||||
status: ['success', 'failure'],
|
||||
},
|
||||
@ -628,29 +720,33 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
|
||||
'echo "engine: $DRONE_COMMIT" > buildinfo.txt',
|
||||
'echo "server: $$(git rev-parse HEAD)" >> buildinfo.txt',
|
||||
'echo "buildNo: $DRONE_BUILD_NUMBER" >> buildinfo.txt',
|
||||
'mv buildinfo.txt ' + result + '/',
|
||||
'mv ' + result + ' /drone/src/',
|
||||
'mv buildinfo.txt ./%s/' % result,
|
||||
'yes | cp -vr ./%s/. /drone/src/%s/' % [result, result],
|
||||
'ls -l /drone/src/' + result,
|
||||
'echo "check columnstore package:"',
|
||||
'ls -l /drone/src/' + result + ' | grep columnstore',
|
||||
'ls -l /drone/src/%s | grep columnstore' % result,
|
||||
],
|
||||
},
|
||||
] +
|
||||
[pipeline.cmapipython] + [pipeline.cmapibuild] +
|
||||
[pipeline.publish('cmapi build')] +
|
||||
[pipeline.publish()] +
|
||||
[
|
||||
{
|
||||
{
|
||||
name: 'publish pkg url',
|
||||
depends_on: ['publish pkg'],
|
||||
image: 'alpine/git',
|
||||
commands: [
|
||||
"echo -e '\\e]8;;" + publish_pkg_url + "\\e\\\\" + publish_pkg_url + "\\e]8;;\\e\\\\'"
|
||||
]
|
||||
}
|
||||
] +
|
||||
"echo -e '\\e]8;;" + publish_pkg_url + '\\e\\\\' + publish_pkg_url + "\\e]8;;\\e\\\\'",
|
||||
],
|
||||
},
|
||||
] +
|
||||
(if (event == 'cron') then [pipeline.publish('pkg latest', 'latest')] else []) +
|
||||
[pipeline.smoke] +
|
||||
[pipeline.smokelog] +
|
||||
[pipeline.publish('smokelog')] +
|
||||
[pipeline.cmapitest] +
|
||||
[pipeline.cmapilog] +
|
||||
(if (platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish('mtr')] + [pipeline.mtrlog] + [pipeline.publish('mtrlog')]) +
|
||||
(if (event == 'cron' && platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.publish('mtr latest', 'latest')] else []) +
|
||||
[pipeline.prepare_regression] +
|
||||
@ -696,7 +792,6 @@ local FinalPipeline(branch, event) = {
|
||||
std.map(function(p) std.join(' ', [branch, p, event, 'arm64', '10.6-enterprise']), platforms_arm.develop),
|
||||
};
|
||||
|
||||
|
||||
[
|
||||
Pipeline(b, p, e, 'amd64', s)
|
||||
for b in std.objectFields(platforms)
|
||||
|
||||
@ -57,6 +57,27 @@ INSTALL(FILES mcs
|
||||
OPTION(RPM "Build an RPM" OFF)
|
||||
IF(RPM)
|
||||
SET(CPACK_GENERATOR "RPM")
|
||||
# cmake wants to byte-compile all .py files.
|
||||
# this line below will prevent it of doing such
|
||||
# trying to byte-compile could produce some issues with not existing
|
||||
# python3 on some systems like centos7
|
||||
# more info:
|
||||
# https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation#Status_quo
|
||||
# https://stackoverflow.com/questions/69988093/cmake-brp-python-bytecompile-and-python3
|
||||
SET(CPACK_RPM_SPEC_INSTALL_POST "%global _python_bytecompile_extra 0")
|
||||
SET(CPACK_RPM_SPEC_MORE_DEFINE "%global _python_bytecompile_extra 0")
|
||||
# Turn off the brp-python-bytecompile script
|
||||
# for every release including EPEL
|
||||
# more info here:
|
||||
# https://pagure.io/packaging-committee/issue/755
|
||||
SET(CPACK_RPM_SPEC_INSTALL_POST "%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')")
|
||||
SET(CPACK_RPM_SPEC_MORE_DEFINE "%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')")
|
||||
# otherwise it could be solved to install python3 in centos7 and adding this line:
|
||||
# SET(CPACK_RPM_SPEC_MORE_DEFINE "%define __python %{__python3}")
|
||||
# example here:
|
||||
# https://github.com/irods/irods/pull/6347/files
|
||||
# but it's doesn't work because of some CPACK versions don't add definitions
|
||||
# to the spec file using CPACK_RPM_SPEC_MORE_DEFINE
|
||||
|
||||
SET(CPACK_RPM_PACKAGE_VERSION ${PACKAGE_VERSION})
|
||||
SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME})
|
||||
|
||||
31
setup-repo.sh
Executable file
31
setup-repo.sh
Executable file
@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
set -e
|
||||
|
||||
. /etc/os-release
|
||||
|
||||
ARCH=$(expr "$(uname -m)" : "arm64\|aarch64" > /dev/null && echo "arm64" || echo "amd64")
|
||||
|
||||
case "$ID" in
|
||||
ubuntu|debian)
|
||||
apt update -y
|
||||
apt install -y ca-certificates
|
||||
echo "deb [trusted=yes] ${PACKAGES_URL}/${ARCH}/ ${OS}/" > /etc/apt/sources.list.d/repo.list
|
||||
cat /etc/apt/sources.list.d/repo.list
|
||||
apt update -y
|
||||
;;
|
||||
rocky|centos)
|
||||
cat << EOF > /etc/yum.repos.d/repo.repo
|
||||
[repo]
|
||||
name=repo
|
||||
baseurl=${PACKAGES_URL}/${ARCH}/${OS}
|
||||
enabled=1
|
||||
gpgcheck=0
|
||||
module_hotfixes=1
|
||||
EOF
|
||||
;;
|
||||
*)
|
||||
echo "$ID is unknown!"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user