database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

chore(ci) refactor reports from tests stage, add it to upgrade

Browse Source
This commit is contained in:
Leonid Fedorov
2023-09-29 11:19:38 +00:00
committed by Leonid Fedorov
parent 79ad7fd01f
commit 2ae171f96c
5 changed files with 246 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

310
.drone.jsonnet
View File

@ -66,6 +66,36 @@ local bootstrap_deps = 'apt-get -y update && apt-get -y install build-essential
local mtr_suite_list = 'basic,bugfixes'; local mtr_suite_list = 'basic,bugfixes';
local mtr_full_set = 'basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly'; local mtr_full_set = 'basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly';
local upgrade_test_lists = {
"centos7": {
"amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"]
},
"rockylinux8": {
"arm64": ["10.6.4-1", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"],
"amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"]
},
"rockylinux9": {
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"],
"amd64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"]
},
"debian11": {
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"],
"amd64": ["10.6.5-2", "10.6.7-3", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"]
},
"debian12": {
"arm64": ["10.6.15-10"],
"amd64": ["10.6.15-10"]
},
"ubuntu20.04": {
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"],
"amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"]
},
"ubuntu22.04": {
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"],
"amd64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"]
},
};
local platformMap(platform, arch) = local platformMap(platform, arch) =
local platform_map = { local platform_map = {
'centos:7': centos7_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos7 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package', 'centos:7': centos7_build_deps + ' && yum ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=centos7 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package',
@ -105,7 +135,6 @@ local testPreparation(platform) =
}; };
platform_map[platform]; platform_map[platform];
local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') = { local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') = {
local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb', local pkg_format = if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'rpm' else 'deb',
local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd', local init = if (pkg_format == 'rpm') then '/usr/lib/systemd/systemd' else 'systemd',
@ -191,10 +220,54 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
'test001.sh', 'test001.sh',
], ],
local mdb_server_versions = ['10.6.4-1', '10.6.5-2', '10.6.7-3', '10.6.8-4', '10.6.9-5', '10.6.11-6', '10.6.12-7'], local mdb_server_versions = upgrade_test_lists[result][arch],
local indexes(arr) = std.range(0, std.length(arr) - 1), local indexes(arr) = std.range(0, std.length(arr) - 1),
local execInnerDocker(command, dockerImage, flags = '') =
'docker exec ' + flags + ' -t ' + dockerImage + ' ' + command,
local installRpmDeb(pkg_format, rpmpackages, debpackages) =
if (pkg_format == 'rpm')
then ' bash -c "yum install -y ' + rpmpackages + '"'
else ' bash -c "apt update --yes && apt install -y ' + debpackages + '"',
local dockerImage(stepname) = stepname + "$${DRONE_BUILD_NUMBER}",
local installEngine(dockerImage, pkg_format) =
if (pkg_format == 'deb') then execInnerDocker('bash -c "apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage)
else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"', dockerImage),
local installCmapi(dockerImage, pkg_format) =
if (pkg_format == 'deb') then execInnerDocker('bash -c "apt install -y mariadb-columnstore-cmapi"', dockerImage)
else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-cmapi"', dockerImage),
local prepareTestStage(dockerImage, pkg_format, result, do_setup) = [
'apk add bash && bash core_dumps/docker-awaiter.sh ' + dockerImage,
if (pkg_format == 'deb')
then execInnerDocker('sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', dockerImage),
execInnerDocker('mkdir core', dockerImage),
execInnerDocker('chmod 777 core', dockerImage),
'docker cp core_dumps/. ' + dockerImage + ':/',
'docker cp setup-repo.sh ' + dockerImage + ':/',
if (do_setup) then execInnerDocker('/setup-repo.sh', dockerImage),
execInnerDocker(installRpmDeb(pkg_format,
"cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which",
"elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget"), dockerImage),
execInnerDocker('sysctl -w kernel.core_pattern="/core/%E_' + result + '_core_dump.%p"', dockerImage),
],
local reportTestStage(dockerImage, result, stage) = [
execInnerDocker('bash -c "/logs.sh ' + stage + '"', dockerImage),
execInnerDocker('bash -c "/core_dump_check.sh core /core/ ' + stage + '"' , dockerImage),
'docker cp ' + dockerImage + ':/core/ /drone/src/' + result + '/',
'docker cp ' + dockerImage + ':/unit_logs/ /drone/src/' + result + '/',
'ls -l /drone/src/' + result,
execInnerDocker('bash -c "/core_dump_drop.sh core"', dockerImage),
'docker stop ' + dockerImage + ' && docker rm ' + dockerImage + ' || echo "cleanup ' + stage + ' failure"',
],
regression(name, depends_on):: { regression(name, depends_on):: {
name: name, name: name,
depends_on: depends_on, depends_on: depends_on,
@ -210,10 +283,12 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
}, },
}, },
commands: [ commands: [
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} mkdir -p reg-logs', execInnerDocker("mkdir -p reg-logs", dockerImage("regression"), "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
"docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' & ", execInnerDocker("bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' & ",
'docker exec --env PRESERVE_LOGS=true -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + ' || ./regression_logs.sh ' + name + '"', dockerImage("regresion")),
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} cat go.log || echo "missing go.log"', execInnerDocker('bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + ' || ./regression_logs.sh ' + name + '"',
dockerImage("regression"),
"--env PRESERVE_LOGS=true --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
], ],
}, },
_volumes:: { _volumes:: {
@ -232,27 +307,21 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
image: 'docker', image: 'docker',
volumes: [pipeline._volumes.docker], volumes: [pipeline._volumes.docker],
commands: [ commands: [
'docker run --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker run --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
'apk add bash && bash core_dumps/docker-awaiter.sh smoke$${DRONE_BUILD_NUMBER}', + prepareTestStage(dockerImage("smoke"), pkg_format, result, true) + [
if (pkg_format == 'rpm') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gdb gawk epel-release which rsyslog hostname procps-ng"' else 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y gdb gawk rsyslog hostname procps wget"', installEngine(dockerImage("smoke"), pkg_format),
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} mkdir core',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} chmod 777 core',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_smoke_core_dump.%p"',
'docker cp core_dumps/. smoke$${DRONE_BUILD_NUMBER}:/',
'docker cp setup-repo.sh smoke$${DRONE_BUILD_NUMBER}:/',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} /setup-repo.sh',
if (pkg_format == 'deb') then 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore"' else 'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine"',
'sleep $${SMOKE_DELAY_SECONDS:-1s}', 'sleep $${SMOKE_DELAY_SECONDS:-1s}',
// start mariadb and mariadb-columnstore services and run simple query // start mariadb and mariadb-columnstore services and run simple query
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb', execInnerDocker('systemctl start mariadb', dockerImage("smoke")),
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl start mariadb-columnstore', execInnerDocker('systemctl start mariadb-columnstore', dockerImage("smoke")),
'docker exec -t smoke$${DRONE_BUILD_NUMBER} mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"', execInnerDocker('mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"',
dockerImage("smoke")),
// restart mariadb and mariadb-columnstore services and run simple query again // restart mariadb and mariadb-columnstore services and run simple query again
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl restart mariadb', execInnerDocker('systemctl restart mariadb', dockerImage("smoke")),
'docker exec -t smoke$${DRONE_BUILD_NUMBER} systemctl restart mariadb-columnstore', execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("smoke")),
'sleep 10', 'sleep 10',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} mariadb -e "insert into test.t1 values (2); select * from test.t1"', execInnerDocker('mariadb -e "insert into test.t1 values (2); select * from test.t1"', dockerImage("smoke")),
], ],
}, },
upgrade(version):: { upgrade(version):: {
@ -266,12 +335,17 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
}, },
}, },
commands: [ commands: [
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name upgrade$${DRONE_BUILD_NUMBER}' + version + ' --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}' + version + ' --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
'apk add bash && bash core_dumps/docker-awaiter.sh upgrade$${DRONE_BUILD_NUMBER}', + prepareTestStage(dockerImage('upgrade') + version, pkg_format, result, false) + [
'docker cp core_dumps/. upgrade$${DRONE_BUILD_NUMBER}' + version + ':/', if (pkg_format == 'deb')
'docker cp setup-repo.sh upgrade$${DRONE_BUILD_NUMBER}' + version + ':/', then execInnerDocker('bash -c "./upgrade_setup_deb.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res +' $${UPGRADE_TOKEN}"',
if (pkg_format == 'deb' && !(result == "debian12") && !(arch == "arm64" && (version == "10.6.4-1" || version == "10.6.5-2" || version == "10.6.7-3" || version == "10.6.8-4")) && !((version == "10.6.4-1" || version == "10.6.8-4") && (result == "debian11" || result == "ubuntu22.04")) && !(result == "ubuntu22.04" && (version == "10.6.5-2" || version == "10.6.7-3"))) then 'docker exec -t upgrade$${DRONE_BUILD_NUMBER}' + version + ' bash -c "./upgrade_setup_deb.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res +' $${UPGRADE_TOKEN}"', dockerImage('upgrade') + version),
if (std.split(platform, ':')[0] == 'rockylinux' && !(result == "rockylinux9" && (version == "10.6.4-1" || version == "10.6.5-2" || version == "10.6.7-3" || version == "10.6.8-4")) && !(result == "rockylinux8" && arch == 'arm64' && (version == "10.6.5-2" || version == "10.6.7-3" || version == "10.6.8-4"))) then 'docker exec -t upgrade$${DRONE_BUILD_NUMBER}' + version + ' bash -c "./upgrade_setup_rpm.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', if (std.split(platform, ':')[0] == 'rockylinux')
then execInnerDocker('bash -c "./upgrade_setup_rpm.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
dockerImage('upgrade') + version),
if (std.split(platform, ':')[0] == 'centos')
then execInnerDocker('bash -c "./upgrade_setup_cent.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
dockerImage('upgrade') + version),
], ],
}, },
upgradelog:: { upgradelog:: {
@ -281,7 +355,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
volumes: [pipeline._volumes.docker], volumes: [pipeline._volumes.docker],
commands: [ commands: [
'echo', 'echo',
] + std.map(function(ver) 'docker stop upgrade$${DRONE_BUILD_NUMBER}' + ver + ' && docker rm upgrade$${DRONE_BUILD_NUMBER}' + ver + ' || echo "cleanup upgrade from version ' + ver + ' failure"', mdb_server_versions), ] + std.flatMap(function(ver) reportTestStage(dockerImage('upgrade') + ver, result, "upgrade_"+ver), mdb_server_versions),
when: { when: {
status: ['success', 'failure'], status: ['success', 'failure'],
}, },
@ -296,40 +370,42 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}', MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}',
}, },
commands: [ commands: [
'docker run --shm-size=500m --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker run --shm-size=500m --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
'apk add bash && bash core_dumps/docker-awaiter.sh mtr$${DRONE_BUILD_NUMBER}', + prepareTestStage('mtr$${DRONE_BUILD_NUMBER}', pkg_format, result, true) + [
installEngine(dockerImage("mtr"), pkg_format),
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget tar lz4 procps-ng"',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y procps wget tar liblz4-tool"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} mkdir core',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} chmod 777 core',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_mtr_core_dump.%p"',
'docker cp core_dumps/. mtr$${DRONE_BUILD_NUMBER}:/',
if (std.split(platform, ':')[0] == 'centos' || std.split(platform, ':')[0] == 'rockylinux') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb epel-release diffutils which rsyslog hostname patch perl cracklib-dicts procps-ng"' else '',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget gawk gdb rsyslog hostname patch"' else '',
'docker cp setup-repo.sh mtr$${DRONE_BUILD_NUMBER}:/',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} /setup-repo.sh',
if (pkg_format == 'deb') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore mariadb-test"' else 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"',
'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/', 'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} chown -R mysql:mysql ' + mtr_path, execInnerDocker('chown -R mysql:mysql ' + mtr_path, dockerImage("mtr")),
// disable systemd 'ProtectSystem' (we need to write to /usr/share/) // disable systemd 'ProtectSystem' (we need to write to /usr/share/)
"docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage('mtr')),
'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl daemon-reload', execInnerDocker('systemctl daemon-reload', dockerImage("mtr")),
'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl start mariadb', execInnerDocker('systemctl start mariadb', dockerImage("mtr")),
'docker exec -t mtr$${DRONE_BUILD_NUMBER} mariadb -e "create database if not exists test;"', execInnerDocker('mariadb -e "create database if not exists test;"', dockerImage("mtr")),
execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("mtr")),
// Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps. // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
//'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 4G"', //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 4G"',
//'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 1G"', //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 1G"',
//'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=mtr$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"', //'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=mtr$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl restart mariadb-columnstore',
// delay mtr for manual debugging on live instance // delay mtr for manual debugging on live instance
'sleep $${MTR_DELAY_SECONDS:-1s}', 'sleep $${MTR_DELAY_SECONDS:-1s}',
'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")', 'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
if (event == 'custom' || event == 'cron') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"', if (event == 'custom' || event == 'cron') then
if (event == 'custom' || event == 'cron') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"', execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"',
if (event == 'cron') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=' + std.join(',', std.map(function(x) 'columnstore/' + x, std.split(mtr_full_set, ','))) + '"' else 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"', dockerImage('mtr')),
if (event == 'custom' || event == 'cron') then
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"',
dockerImage('mtr')),
if (event == 'cron') then
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path +
' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite='
+ std.join(',', std.map(function(x) 'columnstore/' + x, std.split(mtr_full_set, ','))),
dockerImage('mtr')) + '"'
else
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path +
' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"',
dockerImage('mtr')),
], ],
}, },
mtrlog:: { mtrlog:: {
@ -339,21 +415,15 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
volumes: [pipeline._volumes.docker], volumes: [pipeline._volumes.docker],
commands: [ commands: [
'echo "---------- start mariadb service logs ----------"', 'echo "---------- start mariadb service logs ----------"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"', execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage('mtr')),
'echo "---------- end mariadb service logs ----------"', 'echo "---------- end mariadb service logs ----------"',
'echo', 'echo',
'echo "---------- start columnstore debug log ----------"', 'echo "---------- start columnstore debug log ----------"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage('mtr')),
'echo "---------- end columnstore debug log ----------"', 'echo "---------- end columnstore debug log ----------"',
'echo "---------- end columnstore debug log ----------"', 'echo "---------- end columnstore debug log ----------"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/logs.sh mtr"', 'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/' + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"'
'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Mtr"', ] + reportTestStage(dockerImage('mtr'), result, "mtr"),
'docker cp mtr$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/',
'docker cp mtr$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/',
'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/' + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"',
'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"',
'docker stop mtr$${DRONE_BUILD_NUMBER} && docker rm mtr$${DRONE_BUILD_NUMBER} || echo "cleanup mtr failure"',
],
when: { when: {
status: ['success', 'failure'], status: ['success', 'failure'],
}, },
@ -387,40 +457,36 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
'cd mariadb-columnstore-regression-test', 'cd mariadb-columnstore-regression-test',
'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD', 'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD',
'cd ..', 'cd ..',
'docker run --shm-size=500m --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker run --shm-size=500m --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
'apk add bash && bash core_dumps/docker-awaiter.sh regression$${DRONE_BUILD_NUMBER}', + prepareTestStage(dockerImage('regression'), pkg_format, result, true) + [
if (pkg_format == 'rpm') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y wget gawk gdb gcc-c++ epel-release diffutils tar findutils lz4 wget which rsyslog hostname procps-ng elfutils"' else 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y wget tar liblz4-tool procps wget findutils gawk gdb rsyslog hostname g++ elfutils"',
if (platform == 'centos:7') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y sysvinit-tools"', if (platform == 'centos:7') then
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', execInnerDocker('bash -c "yum install -y sysvinit-tools"', dockerImage('regression')),
'docker exec -t regression$${DRONE_BUILD_NUMBER} mkdir core',
'docker exec -t regression$${DRONE_BUILD_NUMBER} chmod 777 core',
'docker exec -t regression$${DRONE_BUILD_NUMBER} sysctl -w kernel.core_pattern="/core/%E_regression_core_dump.%p"',
'docker cp core_dumps/. regression$${DRONE_BUILD_NUMBER}:/',
'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/', 'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/',
// list storage manager binary // list storage manager binary
'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager', 'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager',
'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/', 'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/',
// check storage-manager unit test binary file // check storage-manager unit test binary file
'docker exec -t regression$${DRONE_BUILD_NUMBER} ls -l /storage-manager', execInnerDocker('ls -l /storage-manager',dockerImage('regression')),
// copy test data for regression test suite // copy test data for regression test suite
'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"',dockerImage('regression')),
'docker cp setup-repo.sh regression$${DRONE_BUILD_NUMBER}:/', installEngine(dockerImage('regression'), pkg_format),
'docker exec -t regression$${DRONE_BUILD_NUMBER} /setup-repo.sh',
if (pkg_format == 'deb') then 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore"' else 'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine"',
// set mariadb lower_case_table_names=1 config option // set mariadb lower_case_table_names=1 config option
'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf', execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf', dockerImage('regression')),
// set default client character set to utf-8 // set default client character set to utf-8
'docker exec -t regression$${DRONE_BUILD_NUMBER} sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf', execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf',dockerImage('regression')),
// Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps. // Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps.
//'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 5G"', //'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig HashJoin TotalUmMemory 5G"',
//'docker exec -t regressin$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 2G"', //'docker exec -t regressin$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig DBBC NumBlocksPct 2G"',
//'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=regression$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"', //'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/usr/bin/mcsSetConfig SystemConfig CGroup $(docker ps --filter=name=regression$${DRONE_BUILD_NUMBER} --quiet --no-trunc)"',
// start mariadb and mariadb-columnstore services // start mariadb and mariadb-columnstore services
'docker exec -t regression$${DRONE_BUILD_NUMBER} systemctl start mariadb', execInnerDocker('systemctl start mariadb',dockerImage('regression')),
'docker exec -t regression$${DRONE_BUILD_NUMBER} systemctl restart mariadb-columnstore', execInnerDocker('systemctl restart mariadb-columnstore',dockerImage('regression')),
// delay regression for manual debugging on live instance // delay regression for manual debugging on live instance
'sleep $${REGRESSION_DELAY_SECONDS:-1s}', 'sleep $${REGRESSION_DELAY_SECONDS:-1s}',
'docker exec -t regression$${DRONE_BUILD_NUMBER} /usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester', execInnerDocker('/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester',dockerImage('regression')),
], ],
}, },
smokelog:: { smokelog:: {
@ -430,43 +496,34 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
volumes: [pipeline._volumes.docker], volumes: [pipeline._volumes.docker],
commands: [ commands: [
'echo "---------- start mariadb service logs ----------"', 'echo "---------- start mariadb service logs ----------"',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"', execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage('smoke')),
'echo "---------- end mariadb service logs ----------"', 'echo "---------- end mariadb service logs ----------"',
'echo', 'echo',
'echo "---------- start columnstore debug log ----------"', 'echo "---------- start columnstore debug log ----------"',
'docker exec -t smoke$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage('smoke')),
'echo "---------- end columnstore debug log ----------"', 'echo "---------- end columnstore debug log ----------"'
'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/logs.sh smoke"', ] + reportTestStage(dockerImage('smoke'), result, "smoke"),
'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Smoke"',
'docker cp smoke$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/',
'docker cp smoke$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/',
'ls -l /drone/src/' + result,
'docker exec -t smoke$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"',
'docker stop smoke$${DRONE_BUILD_NUMBER} && docker rm smoke$${DRONE_BUILD_NUMBER} || echo "cleanup smoke failure"',
],
when: { when: {
status: ['success', 'failure'], status: ['success', 'failure'],
}, },
}, },
cmapilog:: { cmapilog:: {
name: 'cmapi log', name: 'cmapilog',
depends_on: ['cmapi test'], depends_on: ['cmapi test'],
image: 'docker', image: 'docker',
volumes: [pipeline._volumes.docker], volumes: [pipeline._volumes.docker],
commands: [ commands: [
'echo "---------- start mariadb service logs ----------"', 'echo "---------- start mariadb service logs ----------"',
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} journalctl -u mariadb --no-pager || echo "mariadb service failure"', execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage('cmapi')),
'echo "---------- end mariadb service logs ----------"', 'echo "---------- end mariadb service logs ----------"',
'echo', 'echo',
'echo "---------- start columnstore debug log ----------"', 'echo "---------- start columnstore debug log ----------"',
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage('cmapi')),
'echo "---------- end columnstore debug log ----------"', 'echo "---------- end columnstore debug log ----------"',
'echo "---------- start cmapi log ----------"', 'echo "---------- start cmapi log ----------"',
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi cmapi_server.log"', execInnerDocker('cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi cmapi_server.log"', dockerImage('cmapi')),
'echo "---------- end cmapi log ----------"', 'echo "---------- end cmapi log ----------"']
'ls -l /drone/src/' + result, + reportTestStage(dockerImage('cmapi'), result, "cmapi"),
'docker stop cmapi$${DRONE_BUILD_NUMBER} && docker rm cmapi$${DRONE_BUILD_NUMBER} || echo "cleanup cmapi failure"',
],
when: { when: {
status: ['success', 'failure'], status: ['success', 'failure'],
}, },
@ -478,23 +535,21 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
volumes: [pipeline._volumes.docker], volumes: [pipeline._volumes.docker],
commands: [ commands: [
'echo "---------- start columnstore regression short report ----------"', 'echo "---------- start columnstore regression short report ----------"',
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} cat go.log || echo "missing go.log"', execInnerDocker('cat go.log || echo "missing go.log"',
dockerImage('regression'),
'--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'),
'echo "---------- end columnstore regression short report ----------"', 'echo "---------- end columnstore regression short report ----------"',
'echo', 'echo',
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/' + result + '/', 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/' + result + '/',
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"', 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"',
'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"', execInnerDocker('bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"',dockerImage('regression')),
'docker exec -t --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest regression$${DRONE_BUILD_NUMBER} bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"', execInnerDocker('bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"',
dockerImage('regression'),
'--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'),
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"', 'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"',
'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/logs.sh regression"', 'docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/' + result + '/'
'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/core_dump_check.sh core /core/ Regression"', ] + reportTestStage(dockerImage('regression'), result, "regression"),
'docker cp regression$${DRONE_BUILD_NUMBER}:/unit_logs/ /drone/src/' + result + '/',
'docker cp regression$${DRONE_BUILD_NUMBER}:/core/ /drone/src/' + result + '/',
'docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/' + result + '/',
'ls -l /drone/src/' + result,
'docker exec -t regression$${DRONE_BUILD_NUMBER} bash -c "/core_dump_drop.sh core"',
'docker stop regression$${DRONE_BUILD_NUMBER} && docker rm regression$${DRONE_BUILD_NUMBER} || echo "cleanup regression failure"',
],
when: { when: {
status: ['success', 'failure'], status: ['success', 'failure'],
}, },
@ -590,24 +645,21 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
PYTHONPATH: '/usr/share/columnstore/cmapi/deps', PYTHONPATH: '/usr/share/columnstore/cmapi/deps',
}, },
commands: [ commands: [
'docker run --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env PYTHONPATH=$${PYTHONPATH} --name cmapi$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker run --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env PYTHONPATH=$${PYTHONPATH} --name cmapi$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target'] +
'apk add bash && bash core_dumps/docker-awaiter.sh cmapi$${DRONE_BUILD_NUMBER}', prepareTestStage(dockerImage('cmapi'), pkg_format, result, true) + [
if (pkg_format == 'rpm') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y iproute sudo epel-release which rsyslog hostname procps-ng"' else 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "apt update --yes && apt install -y iproute2 rsyslog hostname procps sudo"', if (platform == 'rockylinux:9') then execInnerDocker('bash -c "yum install -y libxcrypt-compat"', dockerImage('cmapi')),
if (pkg_format == 'deb') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', installEngine(dockerImage('cmapi'), pkg_format),
if (platform == 'rockylinux:9') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y libxcrypt-compat"', installCmapi(dockerImage('cmapi'), pkg_format),
'docker cp setup-repo.sh cmapi$${DRONE_BUILD_NUMBER}:/',
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} /setup-repo.sh',
if (pkg_format == 'deb') then 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "apt install -y mariadb-plugin-columnstore mariadb-columnstore-cmapi"' else 'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "yum install -y MariaDB-columnstore-engine MariaDB-columnstore-cmapi"',
'cd cmapi', 'cd cmapi',
'for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/$${i}/; done', 'for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/$${i}/; done',
'docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/', 'docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:' + cmapi_path + '/',
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} systemctl start mariadb-columnstore-cmapi', execInnerDocker('systemctl start mariadb-columnstore-cmapi', dockerImage('cmapi')),
// set API key to /etc/columnstore/cmapi_server.conf // set API key to /etc/columnstore/cmapi_server.conf
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "mcs cluster set api-key --key somekey123"', execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', dockerImage('cmapi')),
// copy cmapi conf file for test purposes (there are api key already set inside) // copy cmapi conf file for test purposes (there are api key already set inside)
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], dockerImage('cmapi')),
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} systemctl stop mariadb-columnstore-cmapi', execInnerDocker('systemctl stop mariadb-columnstore-cmapi', dockerImage('cmapi')),
'docker exec -t cmapi$${DRONE_BUILD_NUMBER} bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', dockerImage('cmapi')),
], ],
}, },
multi_node_mtr:: { multi_node_mtr:: {
@ -790,8 +842,10 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
[pipeline.publish('smokelog')] + [pipeline.publish('smokelog')] +
[pipeline.cmapitest] + [pipeline.cmapitest] +
[pipeline.cmapilog] + [pipeline.cmapilog] +
[pipeline.publish('cmapilog')] +
[pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] +
[pipeline.upgradelog] + [pipeline.upgradelog] +
[pipeline.publish('upgradelog')] +
(if (platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish('mtr')] + [pipeline.mtrlog] + [pipeline.publish('mtrlog')]) + (if (platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.publish('mtr')] + [pipeline.mtrlog] + [pipeline.publish('mtrlog')]) +
(if (event == 'cron' && platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.publish('mtr latest', 'latest')] else []) + (if (event == 'cron' && platform == 'rockylinux:8' && arch == 'amd64') then [pipeline.publish('mtr latest', 'latest')] else []) +
[pipeline.prepare_regression] + [pipeline.prepare_regression] +

17
core_dumps/upgrade_setup_cent.sh
View File

@ -1,5 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
source ../build/utils.sh
set -xeuo pipefail set -xeuo pipefail
VERSION="$1" VERSION="$1"
@ -16,6 +18,9 @@ yum -y install MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-
systemctl start mariadb systemctl start mariadb
systemctl start mariadb-columnstore systemctl start mariadb-columnstore
INITIAL_VERSION=$(mariadb -e "select @@version;")
bash -c "./upgrade_data.sh" bash -c "./upgrade_data.sh"
bash -c "./upgrade_verify.sh" bash -c "./upgrade_verify.sh"
@ -33,4 +38,14 @@ EOF
cd / cd /
yum -y update MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo yum -y update MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo
bash -c "./upgrade_verify.sh" UPGRADED_VERSION=$(mariadb -e "select @@version;")
if [[ "$INITIAL_VERSION" == "$UPGRADED_VERSION" ]]; then
error "The upgrade didn't happen!"
exit 1
else
message_splitted "The upgrade from "$INITIAL_VERSION" to "$UPGRADED_VERSION" succeded!"
bash -c "./upgrade_verify.sh"
fi

33
core_dumps/upgrade_setup_deb.sh
View File

@ -1,5 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
source ../build/utils.sh
set -xeuo pipefail set -xeuo pipefail
VERSION="$1" VERSION="$1"
@ -8,6 +10,9 @@ ARCH="$3"
LINK="$4" LINK="$4"
UPGRADE_TOKEN="$5" UPGRADE_TOKEN="$5"
DEBIAN_FRONTEND=noninteractive
UCF_FORCE_CONFNEW=1
apt install --yes rsyslog apt install --yes rsyslog
sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d
bash -c "apt update --yes && apt install -y procps wget curl" bash -c "apt update --yes && apt install -y procps wget curl"
@ -18,6 +23,9 @@ apt update --yes
apt install --yes mariadb-server mariadb-client mariadb-plugin-columnstore apt install --yes mariadb-server mariadb-client mariadb-plugin-columnstore
systemctl start mariadb systemctl start mariadb
systemctl start mariadb-columnstore systemctl start mariadb-columnstore
INITIAL_VERSION=$(mariadb -e "select @@version;")
bash -c "./upgrade_data.sh" bash -c "./upgrade_data.sh"
bash -c "./upgrade_verify.sh" bash -c "./upgrade_verify.sh"
@ -28,5 +36,26 @@ EOF
bash -c "./setup-repo.sh" bash -c "./setup-repo.sh"
apt upgrade --yes
bash -c "./upgrade_verify.sh" # Configuration file '/etc/columnstore/Columnstore.xml'
# ==> Modified (by you or by a script) since installation.
# ==> Package distributor has shipped an updated version.
# What would you like to do about it ? Your options are:
# Y or I : install the package maintainer's version
# N or O : keep your currently-installed version
# D : show the differences between the versions
# Z : start a shell to examine the situation
# The default action is to keep your current version.
# rm to escape halt on interactive question
apt-get --yes --with-new-pkgs -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade
UPGRADED_VERSION=$(mariadb -e "select @@version;")
if [[ "$INITIAL_VERSION" == "$UPGRADED_VERSION" ]]; then
error "The upgrade didn't happen!"
exit 1
else
message_splitted "The upgrade from "$INITIAL_VERSION" to "$UPGRADED_VERSION" succeded!"
bash -c "./upgrade_verify.sh"
fi

18
core_dumps/upgrade_setup_rpm.sh
View File

@ -1,5 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
source ../build/utils.sh
set -xeuo pipefail set -xeuo pipefail
VERSION="$1" VERSION="$1"
@ -9,7 +11,7 @@ LINK="$4"
UPGRADE_TOKEN="$5" UPGRADE_TOKEN="$5"
yum clean all yum clean all
rm -rf /var/cache/dnf/* #rm -rf /var/cache/dnf/*
yum install -y wget which procps-ng diffutils rsyslog yum install -y wget which procps-ng diffutils rsyslog
wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup
chmod +x mariadb_es_repo_setup chmod +x mariadb_es_repo_setup
@ -19,6 +21,9 @@ dnf -y install MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-
systemctl start mariadb systemctl start mariadb
systemctl start mariadb-columnstore systemctl start mariadb-columnstore
INITIAL_VERSION=$(mariadb -e "select @@version;")
bash -c "./upgrade_data.sh" bash -c "./upgrade_data.sh"
bash -c "./upgrade_verify.sh" bash -c "./upgrade_verify.sh"
@ -26,4 +31,13 @@ bash -c "./setup-repo.sh"
dnf repo-pkgs repo list dnf repo-pkgs repo list
dnf -y update MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo dnf -y update MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo
bash -c "./upgrade_verify.sh"
UPGRADED_VERSION=$(mariadb -e "select @@version;")
if [[ "$INITIAL_VERSION" == "$UPGRADED_VERSION" ]]; then
error "The upgrade didn't happen!"
exit 1
else
message_splitted "The upgrade from "$INITIAL_VERSION" to "$UPGRADED_VERSION" succeded!"
bash -c "./upgrade_verify.sh"
fi

2
setup-repo.sh
View File

@ -15,7 +15,7 @@ ubuntu|debian)
cat << EOF > /etc/apt/preferences cat << EOF > /etc/apt/preferences
Package: * Package: *
Pin: origin cspkg.s3.amazonaws.com Pin: origin cspkg.s3.amazonaws.com
Pin-Priority: 700 Pin-Priority: 1700
EOF EOF
apt update -y apt update -y
;; ;;