diff --git a/.drone.jsonnet b/.drone.jsonnet index 03a31d08a..1d77c227f 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -248,8 +248,8 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", if (pkg_format == "deb") then execInnerDocker('bash -c "apt-get clean && apt-get update -y && apt-get install -y mariadb-columnstore-cmapi"', containerName) else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName), - local prepareTestStage(containerName, result, do_setup) = - 'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/prepare_test_stage.sh" + + local prepareTestContainer(containerName, result, do_setup) = + 'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/prepare_test_container.sh" + " --container-name " + containerName + " --docker-image " + img + " --result-path " + result + @@ -274,7 +274,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", image: "docker:28.2.2", volumes: [pipeline._volumes.mdb, pipeline._volumes.docker], commands: [ - prepareTestStage(getContainerName("smoke"), result, true), + prepareTestContainer(getContainerName("smoke"), result, true), "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_smoke.sh " + getContainerName("smoke"), ], }, @@ -302,7 +302,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, commands: [ // why do we mount cgroups here, but miss it on other steps? - prepareTestStage(getContainerName("upgrade") + version, result, false), + prepareTestContainer(getContainerName("upgrade") + version, result, false), if (pkg_format == "deb") then execInnerDocker('bash -c "./upgrade_setup_deb.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"', getContainerName("upgrade") + version), @@ -341,7 +341,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", MTR_FULL_SUITE: "${MTR_FULL_SUITE:-false}", }, commands: [ - prepareTestStage(getContainerName("mtr"), result, true), + prepareTestContainer(getContainerName("mtr"), result, true), 'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")', 'apk add bash && bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/run_mtr.sh' + @@ -363,31 +363,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", status: ["success", "failure"], }, }, - prepare_regression:: { - name: "prepare regression", - depends_on: ["publish pkg", "publish cmapi build"], - when: { - status: ["success", "failure"], - }, - image: "docker:git", - volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], - environment: { - REGRESSION_BRANCH_REF: "${DRONE_SOURCE_BRANCH}", - REGRESSION_REF_AUX: branch_ref, - }, - commands: [ - // REGRESSION_REF can be empty if there is no appropriate branch in regression repository. - // if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR. - 'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}', - "export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}", - 'echo "$$REGRESSION_REF"', - prepareTestStage(getContainerName("regression"), result, true), - "apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/prepare_regression.sh " + - "--container-name " + getContainerName("regression") + - " --regression-branch $$REGRESSION_REF" + - " --distro " + platform, - ], - }, regression(name, depends_on):: { name: name, depends_on: depends_on, @@ -401,11 +376,23 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", REGRESSION_TIMEOUT: { from_secret: "regression_timeout", }, + REGRESSION_BRANCH_REF: "${DRONE_SOURCE_BRANCH}", + REGRESSION_REF_AUX: branch_ref, }, commands: [ - "apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_regression.sh " + - "--container-name " + getContainerName("regression") + + prepareTestContainer(getContainerName("regression"), result, true), + + // REGRESSION_REF can be empty if there is no appropriate branch in regression repository. + // if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR. + 'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}', + "export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}", + 'echo "$$REGRESSION_REF"', + + "apk add bash && bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_regression.sh" + + " --container-name " + getContainerName("regression") + " --test-name " + name + + " --distro " + platform + + " --regression-branch $$REGRESSION_REF" + " --regression-timeout $${REGRESSION_TIMEOUT}", ], }, @@ -482,7 +469,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", PYTHONPATH: "/usr/share/columnstore/cmapi/deps", }, commands: [ - prepareTestStage(getContainerName("cmapi"), result, true), + prepareTestContainer(getContainerName("cmapi"), result, true), installCmapi(getContainerName("cmapi"), pkg_format), "cd cmapi", "for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/$${i}/; done", @@ -695,8 +682,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", [pipeline.cmapilog] + [pipeline.publish("cmapilog")] + (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) + - [pipeline.prepare_regression] + - [pipeline.regression(regression_tests[i], [if (i == 0) then "prepare regression" else regression_tests[i - 1]]) for i in indexes(regression_tests)] + + [pipeline.regression(regression_tests[i], if (i == 0) then ["mtr", "publish pkg", "publish cmapi build"] else [regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + [pipeline.publish("regressionlog")] + // [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + diff --git a/build/prepare_regression.sh b/build/prepare_regression.sh deleted file mode 100755 index 1e08c0c5d..000000000 --- a/build/prepare_regression.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -set -eo pipefail - -SCRIPT_LOCATION=$(dirname "$0") -source "$SCRIPT_LOCATION"/utils.sh - -optparse.define short=c long=container-name desc="Name of the Docker container where regression tests will run" variable=CONTAINER_NAME -optparse.define short=b long=regression-branch desc="Branch from regression tests repo" variable=REGRESSION_BRANCH -optparse.define short=d long=distro desc="Linux distro for which regression is executed" variable=DISTRO -source $(optparse.build) - -for var in CONTAINER_NAME REGRESSION_BRANCH DISTRO; do - if [[ -z "${!var}" ]]; then - error "Missing required flag: -${var:0:1} / --${var,,}" - exit 1 - fi -done - -if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then - error "Container '${CONTAINER_NAME}' is not running." - exit 1 -fi - -if [[ "$DISTRO" == *rocky* ]]; then - CONFIG_PATH_PREFIX="/etc/my.cnf.d/" -else - CONFIG_PATH_PREFIX="/etc/mysql/mariadb.conf.d/50-" -fi - -BUILD_DIR="verylongdirnameforverystrangecpackbehavior" - -git clone --recurse-submodules --branch "${REGRESSION_BRANCH}" --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test - -cd mariadb-columnstore-regression-test -git rev-parse --abbrev-ref HEAD && git rev-parse HEAD -cd .. - -docker cp mariadb-columnstore-regression-test "${CONTAINER_NAME}:/" - -echo "list storage manager binary:" -ls -la "/mdb/${BUILD_DIR}/storage/columnstore/columnstore/storage-manager" - -docker cp "/mdb/${BUILD_DIR}/storage/columnstore/columnstore/storage-manager" "${CONTAINER_NAME}:/" - -#copy test data for regression test suite -execInnerDocker "$CONTAINER_NAME" 'bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"' - -# set mariadb lower_case_table_names=1 config option -execInnerDocker "$CONTAINER_NAME" "sed -i '/^\[mariadb\]/a lower_case_table_names=1' ${CONFIG_PATH_PREFIX}server.cnf" - -# set default client character set to utf-8 -execInnerDocker "$CONTAINER_NAME" "sed -i '/^\[client\]/a default-character-set=utf8' ${CONFIG_PATH_PREFIX}client.cnf" - - -execInnerDocker "$CONTAINER_NAME" "systemctl start mariadb" -execInnerDocker "$CONTAINER_NAME" "systemctl restart mariadb-columnstore" - -# Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps. -execInnerDocker "$CONTAINER_NAME" '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local' - -# Compile queryTester inside container -execInnerDocker "$CONTAINER_NAME" "g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester" - -echo "Prepare_regression stage has completed" diff --git a/build/prepare_test_stage.sh b/build/prepare_test_container.sh similarity index 96% rename from build/prepare_test_stage.sh rename to build/prepare_test_container.sh index 11a6eeeec..924382bcf 100755 --- a/build/prepare_test_stage.sh +++ b/build/prepare_test_container.sh @@ -75,8 +75,7 @@ start_container() { fi } -start_container - +prepare_container() { if [[ "$RESULT" != *rocky* ]]; then execInnerDocker "$CONTAINER_NAME" 'sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d' fi @@ -123,3 +122,11 @@ fi sleep 5 echo "PrepareTestStage completed in $CONTAINER_NAME" +} + + +if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then + start_container + prepare_container +else warn "Container ${CONTAINER_NAME} is already running!" +fi diff --git a/build/run_regression.sh b/build/run_regression.sh index 73c2e240c..83b41fb02 100755 --- a/build/run_regression.sh +++ b/build/run_regression.sh @@ -1,37 +1,91 @@ #!/bin/bash -set -eo pipefail +set -o pipefail SCRIPT_LOCATION=$(dirname "$0") source "$SCRIPT_LOCATION"/utils.sh -optparse.define short=c long=container-name desc="Name of the Docker container where regression tests will be executed" variable=CONTAINER_NAME -optparse.define short=t long=regression-timeout desc="Name of the Docker container where regression tests will be executed" variable=REGRESSION_TIMEOUT -optparse.define short=n long=test-name desc="Test name" variable=TEST_NAME -source $(optparse.build) +optparse.define short=c long=container-name desc="Name of the Docker container where regression tests will run" variable=CONTAINER_NAME +optparse.define short=b long=regression-branch desc="Branch from regression tests repo" variable=REGRESSION_BRANCH +optparse.define short=d long=distro desc="Linux distro for which regression is executed" variable=DISTRO +optparse.define short=t long=regression-timeout desc="Timeout for the regression test run" variable=REGRESSION_TIMEOUT +optparse.define short=n long=test-name desc="Name of regression test to execute" variable=TEST_NAME +source "$(optparse.build)" -echo "Arguments received: $@" - -if [[ "$EUID" -ne 0 ]]; then - error "Please run script as root" +for flag in CONTAINER_NAME REGRESSION_BRANCH DISTRO; do + if [[ -z "${!flag}" ]]; then + error "Missing required flag: -${flag:0:1} / --${flag,,}" exit 1 -fi - -if [[ -z "${CONTAINER_NAME}" ]]; then - echo "Please provide mtr container name as a parameter, e.g. ./run_mtr.sh -c mtr183" - exit 1 -fi + fi +done if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then - error "Container '${CONTAINER_NAME}' is not running." - exit 1 + error "Container '${CONTAINER_NAME}' is not running." + exit 1 fi -execInnerDocker "$CONTAINER_NAME" "cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && mkdir -p reg-logs" +BUILD_DIR="verylongdirnameforverystrangecpackbehavior" -execInnerDocker "$CONTAINER_NAME" "bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' &" +prepare_regression() { + if execInnerDocker "${CONTAINER_NAME}" "test -f /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp"; then + return 0 + fi + + message "Running one-time preparation for regression tests" + + # Set config path prefix based on distro + if [[ "$DISTRO" == *rocky* ]]; then + CONFIG_PATH_PREFIX="/etc/my.cnf.d/" + else + CONFIG_PATH_PREFIX="/etc/mysql/mariadb.conf.d/50-" + fi + + git clone --recurse-submodules --branch "${REGRESSION_BRANCH}" --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test + cd mariadb-columnstore-regression-test + git rev-parse --abbrev-ref HEAD && git rev-parse HEAD + cd .. + + docker cp mariadb-columnstore-regression-test "${CONTAINER_NAME}:/" + docker cp "/mdb/${BUILD_DIR}/storage/columnstore/columnstore/storage-manager" "${CONTAINER_NAME}:/" + + #copy test data for regression test suite + execInnerDocker "${CONTAINER_NAME}" 'bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"' + + # set mariadb lower_case_table_names=1 config option + execInnerDocker "${CONTAINER_NAME}" "sed -i '/^\[mariadb\]/a lower_case_table_names=1' ${CONFIG_PATH_PREFIX}server.cnf" + + # set default client character set to utf-8 + execInnerDocker "${CONTAINER_NAME}" "sed -i '/^\[client\]/a default-character-set=utf8' ${CONFIG_PATH_PREFIX}client.cnf" + + # Start services and build queryTester + execInnerDocker "${CONTAINER_NAME}" "systemctl start mariadb" + execInnerDocker "${CONTAINER_NAME}" "systemctl restart mariadb-columnstore" + execInnerDocker "${CONTAINER_NAME}" "g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester" + + message "Regression preparation complete" +} + +run_test() { + message "Running test: ${TEST_NAME:-}" + + execInnerDocker "${CONTAINER_NAME}" "bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' &" + + execInnerDockerNoTTY "${CONTAINER_NAME}" \ + "export PRESERVE_LOGS=true && cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && \ + timeout -k 1m -s SIGKILL --preserve-status ${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=${TEST_NAME} \ + || ./regression_logs.sh ${TEST_NAME}" +} + +on_exit() { + exit_code=$? + if [[ $exit_code -eq 0 ]]; then + message "Regression finished successfully" + else + message "Some of regression tests has failed" + fi +} +trap on_exit EXIT + +prepare_regression +run_test -execInnerDockerNoTTY "$CONTAINER_NAME" \ - "export PRESERVE_LOGS=true && cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest && \ - bash -c \"timeout -k 1m -s SIGKILL --preserve-status ${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=${TEST_NAME} \ - || ./regression_logs.sh ${TEST_NAME}\"" diff --git a/build/utils.sh b/build/utils.sh index 771b575d2..d2551655a 100644 --- a/build/utils.sh +++ b/build/utils.sh @@ -531,7 +531,7 @@ function execInnerDocker() { if [[ $dockerCommandExitCode -ne 0 ]]; then error "Command \"$@\" failed in container \"$container_name\"" - exit $dockerCommandExitCode + return $dockerCommandExitCode fi } @@ -544,7 +544,7 @@ function execInnerDockerNoTTY() { if [[ $dockerCommandExitCode -ne 0 ]]; then error "Command \"$@\" failed in container \"$container_name\"" - exit $dockerCommandExitCode + return $dockerCommandExitCode fi }