database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-6038 regression to bash

Browse Source
This commit is contained in:
aleksei.bukhalov
2025-06-19 18:11:41 +02:00
committed by Leonid Fedorov
parent 5d0b3a88f3
commit c9cf1a5944
4 changed files with 126 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
.drone.jsonnet
View File

@ -344,7 +344,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
prepareTestStage(getContainerName("mtr"), result, true), prepareTestStage(getContainerName("mtr"), result, true),
'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")', 'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
'bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/run_mtr.sh' + 'apk add bash && bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/run_mtr.sh' +
' --container-name ' + getContainerName("mtr") + ' --container-name ' + getContainerName("mtr") +
' --distro ' + platform + ' --distro ' + platform +
' --suite-list $${MTR_SUITE_LIST}' + ' --suite-list $${MTR_SUITE_LIST}' +
@ -365,7 +365,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
}, },
prepare_regression:: { prepare_regression:: {
name: "prepare regression", name: "prepare regression",
depends_on: ["mtr", "publish pkg", "publish cmapi build"], depends_on: ["publish pkg", "publish cmapi build"],
when: { when: {
status: ["success", "failure"], status: ["success", "failure"],
}, },
@ -376,53 +376,23 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
REGRESSION_REF_AUX: branch_ref, REGRESSION_REF_AUX: branch_ref,
}, },
commands: [ commands: [
// compute branch. // REGRESSION_REF can be empty if there is no appropriate branch in regression repository.
'echo "$$REGRESSION_REF"',
'echo "$$REGRESSION_BRANCH_REF"',
// if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR. // if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR.
'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}', 'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}',
'echo "$$REGRESSION_REF"',
// REGRESSION_REF can be empty if there is no appropriate branch in regression repository.
// assign what is appropriate by default.
"export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}", "export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}",
'echo "$$REGRESSION_REF"', 'echo "$$REGRESSION_REF"',
// clone regression test repo
"git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test",
// where are we now?
"cd mariadb-columnstore-regression-test",
"git rev-parse --abbrev-ref HEAD && git rev-parse HEAD",
"cd ..",
prepareTestStage(getContainerName("regression"), result, true), prepareTestStage(getContainerName("regression"), result, true),
"apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/prepare_regression.sh " +
"docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/", "--container-name " + getContainerName("regression") +
// list storage manager binary " --regression-branch $$REGRESSION_REF" +
"ls -la /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager", " --distro " + platform,
"docker cp /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/",
// check storage-manager unit test binary file
execInnerDocker("ls -l /storage-manager", getContainerName("regression")),
// copy test data for regression test suite
execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', getContainerName("regression")),
// set mariadb lower_case_table_names=1 config option
execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + "server.cnf", getContainerName("regression")),
// set default client character set to utf-8
execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + "client.cnf", getContainerName("regression")),
// Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps.
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", getContainerName("regression")),
execInnerDocker("systemctl start mariadb", getContainerName("regression")),
execInnerDocker("systemctl restart mariadb-columnstore", getContainerName("regression")),
// delay regression for manual debugging on live instance
"sleep $${REGRESSION_DELAY_SECONDS:-1s}",
execInnerDocker("/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester", getContainerName("regression")),
], ],
}, },
regression(name, depends_on):: { regression(name, depends_on):: {
name: name, name: name,
depends_on: depends_on, depends_on: depends_on,
image: "docker:git", image: "docker:git",
volumes: [pipeline._volumes.docker], volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
when: { when: {
status: ["success", "failure"], status: ["success", "failure"],
}, },
@ -433,12 +403,10 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
}, },
}, },
commands: [ commands: [
execInnerDocker("mkdir -p reg-logs", getContainerName("regression"), "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"), "apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_regression.sh " +
execInnerDocker("bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' & ", "--container-name " + getContainerName("regression") +
getContainerName("regresion")), " --test-name " + name +
execInnerDockerNoTTY('bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + " || ./regression_logs.sh " + name + '"', " --regression-timeout $${REGRESSION_TIMEOUT}",
getContainerName("regression"),
"--env PRESERVE_LOGS=true --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
], ],
}, },
regressionlog:: { regressionlog:: {

64
build/prepare_regression.sh Executable file
View File

@ -0,0 +1,64 @@
#!/bin/bash
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
source "$SCRIPT_LOCATION"/utils.sh
optparse.define short=c long=container-name desc="Name of the Docker container where regression tests will run" variable=CONTAINER_NAME
optparse.define short=b long=regression-branch desc="Branch from regression tests repo" variable=REGRESSION_BRANCH
optparse.define short=d long=distro desc="Linux distro for which regression is executed" variable=DISTRO
source $(optparse.build)
for var in CONTAINER_NAME REGRESSION_BRANCH DISTRO; do
if [[ -z "${!var}" ]]; then
error "Missing required flag: -${var:0:1} / --${var,,}"
exit 1
fi
done
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
error "Container '${CONTAINER_NAME}' is not running."
exit 1
fi
if [[ "$DISTRO" == *rocky* ]]; then
CONFIG_PATH_PREFIX="/etc/my.cnf.d/"
else
CONFIG_PATH_PREFIX="/etc/mysql/mariadb.conf.d/50-"
fi
BUILD_DIR="verylongdirnameforverystrangecpackbehavior"
git clone --recurse-submodules --branch "${REGRESSION_BRANCH}" --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test
cd mariadb-columnstore-regression-test
git rev-parse --abbrev-ref HEAD && git rev-parse HEAD
cd ..
docker cp mariadb-columnstore-regression-test "${CONTAINER_NAME}:/"
echo "list storage manager binary:"
ls -la "/mdb/${BUILD_DIR}/storage/columnstore/columnstore/storage-manager"
docker cp "/mdb/${BUILD_DIR}/storage/columnstore/columnstore/storage-manager" "${CONTAINER_NAME}:/"
#copy test data for regression test suite
execInnerDocker "$CONTAINER_NAME" 'bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"'
# set mariadb lower_case_table_names=1 config option
execInnerDocker "$CONTAINER_NAME" "sed -i '/^\[mariadb\]/a lower_case_table_names=1' ${CONFIG_PATH_PREFIX}server.cnf"
# set default client character set to utf-8
execInnerDocker "$CONTAINER_NAME" "sed -i '/^\[client\]/a default-character-set=utf8' ${CONFIG_PATH_PREFIX}client.cnf"
execInnerDocker "$CONTAINER_NAME" "systemctl start mariadb"
execInnerDocker "$CONTAINER_NAME" "systemctl restart mariadb-columnstore"
# Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
execInnerDocker "$CONTAINER_NAME" '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local'
# Compile queryTester inside container
execInnerDocker "$CONTAINER_NAME" "g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester"
echo "Prepare_regression stage has completed"

37
build/run_regression.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
source "$SCRIPT_LOCATION"/utils.sh
optparse.define short=c long=container-name desc="Name of the Docker container where regression tests will be executed" variable=CONTAINER_NAME
optparse.define short=t long=regression-timeout desc="Name of the Docker container where regression tests will be executed" variable=REGRESSION_TIMEOUT
optparse.define short=n long=test-name desc="Test name" variable=TEST_NAME
source $(optparse.build)
echo "Arguments received: $@"
if [[ "$EUID" -ne 0 ]]; then
error "Please run script as root"
exit 1
fi
if [[ -z "${CONTAINER_NAME}" ]]; then
echo "Please provide mtr container name as a parameter, e.g. ./run_mtr.sh -c mtr183"
exit 1
fi
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
error "Container '${CONTAINER_NAME}' is not running."
exit 1
fi
execInnerDocker "$CONTAINER_NAME" "cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && mkdir -p reg-logs"
execInnerDocker "$CONTAINER_NAME" "bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' &"
execInnerDockerNoTTY "$CONTAINER_NAME" \
"export PRESERVE_LOGS=true && cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && \
bash -c \"timeout -k 1m -s SIGKILL --preserve-status ${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=${TEST_NAME} \
|| ./regression_logs.sh ${TEST_NAME}\""

13
build/utils.sh
View File

@ -535,6 +535,19 @@ function execInnerDocker() {
fi fi
} }
function execInnerDockerNoTTY() {
local container_name=$1
shift 1
docker exec "$container_name" bash -c "$@"
local dockerCommandExitCode=$?
if [[ $dockerCommandExitCode -ne 0 ]]; then
error "Command \"$@\" failed in container \"$container_name\""
exit $dockerCommandExitCode
fi
}
function change_ubuntu_mirror() { function change_ubuntu_mirror() {
local region="$1" local region="$1"
message "Changing Ubuntu mirror to $region" message "Changing Ubuntu mirror to $region"