You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-30 19:23:07 +03:00
MCOL-6038 combine regression and prepare regression
This commit is contained in:
aleksei.bukhalov
committed by
Leonid Fedorov
Leonid Fedorov
parent
c9cf1a5944
commit
0112e7b13c
@ -248,8 +248,8 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
|
||||
if (pkg_format == "deb") then execInnerDocker('bash -c "apt-get clean && apt-get update -y && apt-get install -y mariadb-columnstore-cmapi"', containerName)
|
||||
else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName),
|
||||
|
||||
local prepareTestStage(containerName, result, do_setup) =
|
||||
'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/prepare_test_stage.sh" +
|
||||
local prepareTestContainer(containerName, result, do_setup) =
|
||||
'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/prepare_test_container.sh" +
|
||||
" --container-name " + containerName +
|
||||
" --docker-image " + img +
|
||||
" --result-path " + result +
|
||||
@ -274,7 +274,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
|
||||
image: "docker:28.2.2",
|
||||
volumes: [pipeline._volumes.mdb, pipeline._volumes.docker],
|
||||
commands: [
|
||||
prepareTestStage(getContainerName("smoke"), result, true),
|
||||
prepareTestContainer(getContainerName("smoke"), result, true),
|
||||
"bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_smoke.sh " + getContainerName("smoke"),
|
||||
],
|
||||
},
|
||||
@ -302,7 +302,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
|
||||
},
|
||||
commands: [
|
||||
// why do we mount cgroups here, but miss it on other steps?
|
||||
prepareTestStage(getContainerName("upgrade") + version, result, false),
|
||||
prepareTestContainer(getContainerName("upgrade") + version, result, false),
|
||||
if (pkg_format == "deb")
|
||||
then execInnerDocker('bash -c "./upgrade_setup_deb.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
|
||||
getContainerName("upgrade") + version),
|
||||
@ -341,7 +341,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
|
||||
MTR_FULL_SUITE: "${MTR_FULL_SUITE:-false}",
|
||||
},
|
||||
commands: [
|
||||
prepareTestStage(getContainerName("mtr"), result, true),
|
||||
prepareTestContainer(getContainerName("mtr"), result, true),
|
||||
'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
|
||||
|
||||
'apk add bash && bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/run_mtr.sh' +
|
||||
@ -363,31 +363,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
|
||||
status: ["success", "failure"],
|
||||
},
|
||||
},
|
||||
prepare_regression:: {
|
||||
name: "prepare regression",
|
||||
depends_on: ["publish pkg", "publish cmapi build"],
|
||||
when: {
|
||||
status: ["success", "failure"],
|
||||
},
|
||||
image: "docker:git",
|
||||
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
|
||||
environment: {
|
||||
REGRESSION_BRANCH_REF: "${DRONE_SOURCE_BRANCH}",
|
||||
REGRESSION_REF_AUX: branch_ref,
|
||||
},
|
||||
commands: [
|
||||
// REGRESSION_REF can be empty if there is no appropriate branch in regression repository.
|
||||
// if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR.
|
||||
'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}',
|
||||
"export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}",
|
||||
'echo "$$REGRESSION_REF"',
|
||||
prepareTestStage(getContainerName("regression"), result, true),
|
||||
"apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/prepare_regression.sh " +
|
||||
"--container-name " + getContainerName("regression") +
|
||||
" --regression-branch $$REGRESSION_REF" +
|
||||
" --distro " + platform,
|
||||
],
|
||||
},
|
||||
regression(name, depends_on):: {
|
||||
name: name,
|
||||
depends_on: depends_on,
|
||||
@ -401,11 +376,23 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
|
||||
REGRESSION_TIMEOUT: {
|
||||
from_secret: "regression_timeout",
|
||||
},
|
||||
REGRESSION_BRANCH_REF: "${DRONE_SOURCE_BRANCH}",
|
||||
REGRESSION_REF_AUX: branch_ref,
|
||||
},
|
||||
commands: [
|
||||
"apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_regression.sh " +
|
||||
"--container-name " + getContainerName("regression") +
|
||||
prepareTestContainer(getContainerName("regression"), result, true),
|
||||
|
||||
// REGRESSION_REF can be empty if there is no appropriate branch in regression repository.
|
||||
// if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR.
|
||||
'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}',
|
||||
"export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}",
|
||||
'echo "$$REGRESSION_REF"',
|
||||
|
||||
"apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_regression.sh" +
|
||||
" --container-name " + getContainerName("regression") +
|
||||
" --test-name " + name +
|
||||
" --distro " + platform +
|
||||
" --regression-branch $$REGRESSION_REF" +
|
||||
" --regression-timeout $${REGRESSION_TIMEOUT}",
|
||||
],
|
||||
},
|
||||
@ -482,7 +469,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
|
||||
PYTHONPATH: "/usr/share/columnstore/cmapi/deps",
|
||||
},
|
||||
commands: [
|
||||
prepareTestStage(getContainerName("cmapi"), result, true),
|
||||
prepareTestContainer(getContainerName("cmapi"), result, true),
|
||||
installCmapi(getContainerName("cmapi"), pkg_format),
|
||||
"cd cmapi",
|
||||
"for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/$${i}/; done",
|
||||
@ -695,8 +682,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
|
||||
[pipeline.cmapilog] +
|
||||
[pipeline.publish("cmapilog")] +
|
||||
(if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) +
|
||||
[pipeline.prepare_regression] +
|
||||
[pipeline.regression(regression_tests[i], [if (i == 0) then "prepare regression" else regression_tests[i - 1]]) for i in indexes(regression_tests)] +
|
||||
[pipeline.regression(regression_tests[i], if (i == 0) then ["mtr", "publish pkg", "publish cmapi build"] else [regression_tests[i - 1]]) for i in indexes(regression_tests)] +
|
||||
[pipeline.regressionlog] +
|
||||
[pipeline.publish("regressionlog")] +
|
||||
// [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] +
|
||||
|
||||
@ -1,64 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
SCRIPT_LOCATION=$(dirname "$0")
|
||||
source "$SCRIPT_LOCATION"/utils.sh
|
||||
|
||||
optparse.define short=c long=container-name desc="Name of the Docker container where regression tests will run" variable=CONTAINER_NAME
|
||||
optparse.define short=b long=regression-branch desc="Branch from regression tests repo" variable=REGRESSION_BRANCH
|
||||
optparse.define short=d long=distro desc="Linux distro for which regression is executed" variable=DISTRO
|
||||
source $(optparse.build)
|
||||
|
||||
for var in CONTAINER_NAME REGRESSION_BRANCH DISTRO; do
|
||||
if [[ -z "${!var}" ]]; then
|
||||
error "Missing required flag: -${var:0:1} / --${var,,}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
|
||||
error "Container '${CONTAINER_NAME}' is not running."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$DISTRO" == *rocky* ]]; then
|
||||
CONFIG_PATH_PREFIX="/etc/my.cnf.d/"
|
||||
else
|
||||
CONFIG_PATH_PREFIX="/etc/mysql/mariadb.conf.d/50-"
|
||||
fi
|
||||
|
||||
BUILD_DIR="verylongdirnameforverystrangecpackbehavior"
|
||||
|
||||
git clone --recurse-submodules --branch "${REGRESSION_BRANCH}" --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test
|
||||
|
||||
cd mariadb-columnstore-regression-test
|
||||
git rev-parse --abbrev-ref HEAD && git rev-parse HEAD
|
||||
cd ..
|
||||
|
||||
docker cp mariadb-columnstore-regression-test "${CONTAINER_NAME}:/"
|
||||
|
||||
echo "list storage manager binary:"
|
||||
ls -la "/mdb/${BUILD_DIR}/storage/columnstore/columnstore/storage-manager"
|
||||
|
||||
docker cp "/mdb/${BUILD_DIR}/storage/columnstore/columnstore/storage-manager" "${CONTAINER_NAME}:/"
|
||||
|
||||
#copy test data for regression test suite
|
||||
execInnerDocker "$CONTAINER_NAME" 'bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"'
|
||||
|
||||
# set mariadb lower_case_table_names=1 config option
|
||||
execInnerDocker "$CONTAINER_NAME" "sed -i '/^\[mariadb\]/a lower_case_table_names=1' ${CONFIG_PATH_PREFIX}server.cnf"
|
||||
|
||||
# set default client character set to utf-8
|
||||
execInnerDocker "$CONTAINER_NAME" "sed -i '/^\[client\]/a default-character-set=utf8' ${CONFIG_PATH_PREFIX}client.cnf"
|
||||
|
||||
|
||||
execInnerDocker "$CONTAINER_NAME" "systemctl start mariadb"
|
||||
execInnerDocker "$CONTAINER_NAME" "systemctl restart mariadb-columnstore"
|
||||
|
||||
# Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
|
||||
execInnerDocker "$CONTAINER_NAME" '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local'
|
||||
|
||||
# Compile queryTester inside container
|
||||
execInnerDocker "$CONTAINER_NAME" "g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester"
|
||||
|
||||
echo "Prepare_regression stage has completed"
|
||||
@ -75,8 +75,7 @@ start_container() {
|
||||
fi
|
||||
}
|
||||
|
||||
start_container
|
||||
|
||||
prepare_container() {
|
||||
if [[ "$RESULT" != *rocky* ]]; then
|
||||
execInnerDocker "$CONTAINER_NAME" 'sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d'
|
||||
fi
|
||||
@ -123,3 +122,11 @@ fi
|
||||
|
||||
sleep 5
|
||||
echo "PrepareTestStage completed in $CONTAINER_NAME"
|
||||
}
|
||||
|
||||
|
||||
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
|
||||
start_container
|
||||
prepare_container
|
||||
else warn "Container ${CONTAINER_NAME} is already running!"
|
||||
fi
|
||||
@ -1,37 +1,91 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eo pipefail
|
||||
set -o pipefail
|
||||
|
||||
SCRIPT_LOCATION=$(dirname "$0")
|
||||
source "$SCRIPT_LOCATION"/utils.sh
|
||||
|
||||
optparse.define short=c long=container-name desc="Name of the Docker container where regression tests will be executed" variable=CONTAINER_NAME
|
||||
optparse.define short=t long=regression-timeout desc="Name of the Docker container where regression tests will be executed" variable=REGRESSION_TIMEOUT
|
||||
optparse.define short=n long=test-name desc="Test name" variable=TEST_NAME
|
||||
source $(optparse.build)
|
||||
optparse.define short=c long=container-name desc="Name of the Docker container where regression tests will run" variable=CONTAINER_NAME
|
||||
optparse.define short=b long=regression-branch desc="Branch from regression tests repo" variable=REGRESSION_BRANCH
|
||||
optparse.define short=d long=distro desc="Linux distro for which regression is executed" variable=DISTRO
|
||||
optparse.define short=t long=regression-timeout desc="Timeout for the regression test run" variable=REGRESSION_TIMEOUT
|
||||
optparse.define short=n long=test-name desc="Name of regression test to execute" variable=TEST_NAME
|
||||
source "$(optparse.build)"
|
||||
|
||||
echo "Arguments received: $@"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Please run script as root"
|
||||
for flag in CONTAINER_NAME REGRESSION_BRANCH DISTRO; do
|
||||
if [[ -z "${!flag}" ]]; then
|
||||
error "Missing required flag: -${flag:0:1} / --${flag,,}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${CONTAINER_NAME}" ]]; then
|
||||
echo "Please provide mtr container name as a parameter, e.g. ./run_mtr.sh -c mtr183"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
|
||||
error "Container '${CONTAINER_NAME}' is not running."
|
||||
exit 1
|
||||
error "Container '${CONTAINER_NAME}' is not running."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
execInnerDocker "$CONTAINER_NAME" "cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && mkdir -p reg-logs"
|
||||
BUILD_DIR="verylongdirnameforverystrangecpackbehavior"
|
||||
|
||||
execInnerDocker "$CONTAINER_NAME" "bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' &"
|
||||
prepare_regression() {
|
||||
if execInnerDocker "${CONTAINER_NAME}" "test -f /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
message "Running one-time preparation for regression tests"
|
||||
|
||||
# Set config path prefix based on distro
|
||||
if [[ "$DISTRO" == *rocky* ]]; then
|
||||
CONFIG_PATH_PREFIX="/etc/my.cnf.d/"
|
||||
else
|
||||
CONFIG_PATH_PREFIX="/etc/mysql/mariadb.conf.d/50-"
|
||||
fi
|
||||
|
||||
git clone --recurse-submodules --branch "${REGRESSION_BRANCH}" --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test
|
||||
cd mariadb-columnstore-regression-test
|
||||
git rev-parse --abbrev-ref HEAD && git rev-parse HEAD
|
||||
cd ..
|
||||
|
||||
docker cp mariadb-columnstore-regression-test "${CONTAINER_NAME}:/"
|
||||
docker cp "/mdb/${BUILD_DIR}/storage/columnstore/columnstore/storage-manager" "${CONTAINER_NAME}:/"
|
||||
|
||||
#copy test data for regression test suite
|
||||
execInnerDocker "${CONTAINER_NAME}" 'bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"'
|
||||
|
||||
# set mariadb lower_case_table_names=1 config option
|
||||
execInnerDocker "${CONTAINER_NAME}" "sed -i '/^\[mariadb\]/a lower_case_table_names=1' ${CONFIG_PATH_PREFIX}server.cnf"
|
||||
|
||||
# set default client character set to utf-8
|
||||
execInnerDocker "${CONTAINER_NAME}" "sed -i '/^\[client\]/a default-character-set=utf8' ${CONFIG_PATH_PREFIX}client.cnf"
|
||||
|
||||
# Start services and build queryTester
|
||||
execInnerDocker "${CONTAINER_NAME}" "systemctl start mariadb"
|
||||
execInnerDocker "${CONTAINER_NAME}" "systemctl restart mariadb-columnstore"
|
||||
execInnerDocker "${CONTAINER_NAME}" "g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester"
|
||||
|
||||
message "Regression preparation complete"
|
||||
}
|
||||
|
||||
run_test() {
|
||||
message "Running test: ${TEST_NAME:-<none>}"
|
||||
|
||||
execInnerDocker "${CONTAINER_NAME}" "bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' &"
|
||||
|
||||
execInnerDockerNoTTY "${CONTAINER_NAME}" \
|
||||
"export PRESERVE_LOGS=true && cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && \
|
||||
timeout -k 1m -s SIGKILL --preserve-status ${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=${TEST_NAME} \
|
||||
|| ./regression_logs.sh ${TEST_NAME}"
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
exit_code=$?
|
||||
if [[ $exit_code -eq 0 ]]; then
|
||||
message "Regression finished successfully"
|
||||
else
|
||||
message "Some of regression tests has failed"
|
||||
fi
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
prepare_regression
|
||||
run_test
|
||||
|
||||
execInnerDockerNoTTY "$CONTAINER_NAME" \
|
||||
"export PRESERVE_LOGS=true && cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && \
|
||||
bash -c \"timeout -k 1m -s SIGKILL --preserve-status ${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=${TEST_NAME} \
|
||||
|| ./regression_logs.sh ${TEST_NAME}\""
|
||||
|
||||
@ -531,7 +531,7 @@ function execInnerDocker() {
|
||||
|
||||
if [[ $dockerCommandExitCode -ne 0 ]]; then
|
||||
error "Command \"$@\" failed in container \"$container_name\""
|
||||
exit $dockerCommandExitCode
|
||||
return $dockerCommandExitCode
|
||||
fi
|
||||
}
|
||||
|
||||
@ -544,7 +544,7 @@ function execInnerDockerNoTTY() {
|
||||
|
||||
if [[ $dockerCommandExitCode -ne 0 ]]; then
|
||||
error "Command \"$@\" failed in container \"$container_name\""
|
||||
exit $dockerCommandExitCode
|
||||
return $dockerCommandExitCode
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user