diff --git a/.drone.jsonnet b/.drone.jsonnet index 64cfe43e0..6cd235be8 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -15,10 +15,16 @@ local platforms_arm = { "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"], }; +local rewrite_ubuntu_mirror = @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list || true; " + + @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list.d/ubuntu.sources || true; " + + "cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list | grep archive || true; "; + local customEnvCommandsMap = { // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'], "clang-20": [ - "apt install -y wget curl lsb-release software-properties-common gnupg", + rewrite_ubuntu_mirror, + "apt-get clean && apt-get update", + "apt-get install -y wget curl lsb-release software-properties-common gnupg", "wget https://apt.llvm.org/llvm.sh", "bash llvm.sh 20", "export CC=/usr/bin/clang", @@ -114,7 +120,7 @@ local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " + local rockylinux_common_deps = " && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel"; -local deb_deps = "apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev"; +local deb_deps = rewrite_ubuntu_mirror + "apt-get clean && apt-get update && apt-get install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev"; local testPreparation(platform) = local platform_map = { @@ -158,10 +164,12 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", local server_remote = if (std.endsWith(server, "enterprise")) then "https://github.com/mariadb-corporation/MariaDBEnterprise" else "https://github.com/MariaDB/server", local sccache_arch = if (arch == "amd64") then "x86_64" else "aarch64", - local get_sccache = "echo getting sccache... && (apt update -y && apt install -y curl || yum install -y curl || true) " + - "&& curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz " + - "&& tar xzf sccache.tar.gz " + - "&& install sccache*/sccache /usr/local/bin/ && echo sccache installed", + local get_sccache = ["echo getting sccache...", + rewrite_ubuntu_mirror, + "(apt-get clean && apt-get update -y && apt-get install -y curl || yum install -y curl || true)", + "curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz &&", + "tar xzf sccache.tar.gz", + "install sccache*/sccache /usr/local/bin/ && echo sccache installed"], local pipeline = self, @@ -235,7 +243,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", local getContainerName(stepname) = stepname + "$${DRONE_BUILD_NUMBER}", local installCmapi(containerName, pkg_format) = - if (pkg_format == "deb") then execInnerDocker('bash -c "apt update -y && apt install -y mariadb-columnstore-cmapi"', containerName) + if (pkg_format == "deb") then execInnerDocker('bash -c "apt-get clean && apt-get update -y && apt-get install -y mariadb-columnstore-cmapi"', containerName) else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName), local prepareTestStage(containerName, result, do_setup) = @@ -646,8 +654,8 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", }, commands: [ "mkdir /mdb/" + builddir + "/" + result, - get_sccache, ] + + get_sccache + customEnvCommands(customBuildEnvCommandsMapKey, builddir) + [ 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 0a1b823e5..5b022be8d 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -67,6 +67,21 @@ source $(optparse.build) message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal" +if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then + getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS + MCS_BUILD_TYPE=$selectedChoice +fi + +if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then + echo "OS is empty, trying to detect..." + detect_distro +fi + +pkg_format="deb" +if [[ "$OS" == *"rocky"* ]]; then + pkg_format="rpm" +fi + install_deps() { if [[ $INSTALL_DEPS = false ]]; then return @@ -95,7 +110,6 @@ install_deps() { dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}" elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then - prereq="apt-get clean && rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin" command="${DEB_BUILD_DEPS}" else echo "Unsupported OS: $OS" @@ -103,8 +117,12 @@ install_deps() { fi if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then + if [ -f /.dockerenv ]; then + change_ubuntu_mirror us + fi command="${command} lto-disabled-list" fi + eval "$prereq" message "Installing dependencies for $OS" retry_eval 5 "$command" @@ -125,21 +143,6 @@ if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}" fi -if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then - getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS - MCS_BUILD_TYPE=$selectedChoice -fi - -if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then - echo "OS is empty, trying to detect..." - detect_distro -fi - -pkg_format="deb" -if [[ "$OS" == *"rocky"* ]]; then - pkg_format="rpm" -fi - disable_git_restore_frozen_revision() { cd $MDB_SOURCE_PATH git config submodule.storage/columnstore/columnstore.update none diff --git a/build/prepare_test_stage.sh b/build/prepare_test_stage.sh index 6bea2edd9..ea0a638e0 100755 --- a/build/prepare_test_stage.sh +++ b/build/prepare_test_stage.sh @@ -67,7 +67,6 @@ start_container() { docker run "${docker_run_args[@]}" sleep 5 - apk add bash bash "$COLUMNSTORE_SOURCE_PATH"/core_dumps/docker-awaiter.sh "$CONTAINER_NAME" if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then @@ -79,7 +78,7 @@ start_container() { start_container if [[ "$RESULT" != *rocky* ]]; then - execInnerDocker 'sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d' "$CONTAINER_NAME" + execInnerDocker "$CONTAINER_NAME" 'sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d' fi #list_cgroups @@ -88,39 +87,39 @@ ls -al /sys/fs/cgroup/cgroup.controllers || true ls -al /sys/fs/cgroup/ || true ls -al /sys/fs/cgroup/memory || true -execInnerDocker 'echo Inner Docker CGroups opts here' "$CONTAINER_NAME" -execInnerDocker 'ls -al /sys/fs/cgroup/cgroup.controllers || true' "$CONTAINER_NAME" -execInnerDocker 'ls -al /sys/fs/cgroup/ || true' "$CONTAINER_NAME" -execInnerDocker 'ls -al /sys/fs/cgroup/memory || true' "$CONTAINER_NAME" +execInnerDocker "$CONTAINER_NAME" 'echo Inner Docker CGroups opts here' +execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/cgroup.controllers || true' +execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/ || true' +execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/memory || true' # Prepare core dump directory inside container -execInnerDocker 'mkdir -p core && chmod 777 core' "$CONTAINER_NAME" +execInnerDocker "$CONTAINER_NAME" 'mkdir -p core && chmod 777 core' docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/. "$CONTAINER_NAME":/ docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh "$CONTAINER_NAME":/ docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh "$CONTAINER_NAME":/ if [[ "$DO_SETUP" == "true" ]]; then - execInnerDocker '/setup-repo.sh' "$CONTAINER_NAME" + execInnerDocker "$CONTAINER_NAME" '/setup-repo.sh' fi # install deps if [[ "$RESULT" == *rocky* ]]; then # sudo bypass - execInnerDocker "printf '%s\n' '#!/bin/sh' 'exec \"\$@\"' > /usr/bin/sudo && chmod +x /usr/bin/sudo" "$CONTAINER_NAME" - execInnerDockerWithRetry 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' "$CONTAINER_NAME" + execInnerDockerWithRetry "$CONTAINER_NAME" 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' else - execInnerDockerWithRetry 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME" + change_ubuntu_mirror_in_docker "$CONTAINER_NAME" "us" + execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' fi # Configure core dump naming pattern -execInnerDocker 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"' "$CONTAINER_NAME" +execInnerDocker "$CONTAINER_NAME" 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"' #Install columnstore in container echo "Installing columnstore..." if [[ "$RESULT" == *rocky* ]]; then - execInnerDockerWithRetry 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME" + execInnerDockerWithRetry "$CONTAINER_NAME" 'yum install -y MariaDB-columnstore-engine MariaDB-test' else - execInnerDockerWithRetry 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME" + execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' fi sleep 5 diff --git a/build/report_test_stage.sh b/build/report_test_stage.sh index 611582438..23546a382 100755 --- a/build/report_test_stage.sh +++ b/build/report_test_stage.sh @@ -34,11 +34,11 @@ echo "Reporting test stage: ${STAGE} executed in ${CONTAINER_NAME} container" if [[ "${CONTAINER_NAME}" == *smoke* ]] || [[ "${CONTAINER_NAME}" == *mtr* ]] || [[ "${CONTAINER_NAME}" == *cmapi* ]]; then # common logs for smoke, mtr, cmapi echo "---------- start mariadb service logs ----------" - execInnerDocker 'journalctl -u mariadb --no-pager || echo "mariadb service failure"' "$CONTAINER_NAME" + execInnerDocker "$CONTAINER_NAME" 'journalctl -u mariadb --no-pager || echo "mariadb service failure"' echo "---------- end mariadb service logs ----------" echo echo "---------- start columnstore debug log ----------" - execInnerDocker 'cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"' "$CONTAINER_NAME" + execInnerDocker "$CONTAINER_NAME" 'cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"' echo "---------- end columnstore debug log ----------" if [[ "${CONTAINER_NAME}" == *mtr* ]]; then @@ -49,7 +49,7 @@ if [[ "${CONTAINER_NAME}" == *smoke* ]] || [[ "${CONTAINER_NAME}" == *mtr* ]] || if [[ "${CONTAINER_NAME}" == *cmapi* ]]; then echo echo "---------- start cmapi log ----------" - execInnerDocker 'cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi_server.log"' "$CONTAINER_NAME" + execInnerDocker "$CONTAINER_NAME" 'cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi_server.log"' echo "---------- end cmapi log ----------" fi @@ -59,16 +59,14 @@ elif [[ "${CONTAINER_NAME}" == *upgrade* ]]; then elif [[ "${CONTAINER_NAME}" == *regression* ]]; then echo "---------- start columnstore regression short report ----------" - execInnerDocker 'cat go.log || echo "missing go.log"' "$CONTAINER_NAME" '--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest' + execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; cat go.log || echo "missing go.log"' echo "---------- end columnstore regression short report ----------" echo docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/" "/drone/src/${RESULT}/" || echo "missing regression logs" docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs.tgz" - execInnerDocker 'tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/' "$CONTAINER_NAME" - execInnerDocker 'tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore || echo "failed to grab regression results"' \ - "$CONTAINER_NAME" \ - '--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest' + execInnerDocker "$CONTAINER_NAME" 'tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/' + execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore || echo "failed to grab regression results"' docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs2.tgz" docker cp "${CONTAINER_NAME}:regressionQueries.tgz" "/drone/src/${RESULT}/" || echo "missing regressionQueries.tgz" @@ -77,13 +75,13 @@ else exit 1 fi -execInnerDocker "/logs.sh ${STAGE}" "$CONTAINER_NAME" -execInnerDocker "/core_dump_check.sh core /core/ ${STAGE}" "$CONTAINER_NAME" +execInnerDocker "$CONTAINER_NAME" "/logs.sh ${STAGE}" +execInnerDocker "$CONTAINER_NAME" "/core_dump_check.sh core /core/ ${STAGE}" -docker cp "${CONTAINER_NAME}:/core/" "/drone/src/${RESULT}/" +docker cp "${CONTAINER_NAME}:/core/" "/drone/src/${RESULT}/" docker cp "${CONTAINER_NAME}:/unit_logs/" "/drone/src/${RESULT}/" -execInnerDocker "/core_dump_drop.sh core" "$CONTAINER_NAME" +execInnerDocker "$CONTAINER_NAME" "/core_dump_drop.sh core" echo "Saved artifacts:" ls -R "/drone/src/${RESULT}/" echo "Done reporting ${STAGE}" diff --git a/build/run_smoke.sh b/build/run_smoke.sh index 88214e201..c94164662 100755 --- a/build/run_smoke.sh +++ b/build/run_smoke.sh @@ -10,8 +10,8 @@ CONTAINER_NAME="$1" echo "Arguments received: $@" if [[ "$EUID" -ne 0 ]]; then - error "Please run script as root" - exit 1 + error "Please run script as root" + exit 1 fi if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then @@ -22,19 +22,19 @@ fi message "Running smoke checks..." # start mariadb and mariadb-columnstore services and run simple query -execInnerDocker 'systemctl start mariadb' "$CONTAINER_NAME" -execInnerDocker '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local' "$CONTAINER_NAME" -execInnerDocker 'systemctl restart mariadb-columnstore' "$CONTAINER_NAME" +execInnerDocker "$CONTAINER_NAME" 'systemctl start mariadb' +execInnerDocker "$CONTAINER_NAME" '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local' +execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb-columnstore' -execInnerDocker 'mariadb -e "CREATE DATABASE IF NOT EXISTS test; \ +execInnerDocker "$CONTAINER_NAME" 'mariadb -e "CREATE DATABASE IF NOT EXISTS test; \ CREATE TABLE test.t1 (a INT) ENGINE=Columnstore; \ INSERT INTO test.t1 VALUES (1); \ -SELECT * FROM test.t1;"' "$CONTAINER_NAME" +SELECT * FROM test.t1;"' # restart both services, wait a bit, then insert and select again -execInnerDocker 'systemctl restart mariadb' "$CONTAINER_NAME" -execInnerDocker 'systemctl restart mariadb-columnstore' "$CONTAINER_NAME" +execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb' +execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb-columnstore' sleep 10 -execInnerDocker 'mariadb -e "INSERT INTO test.t1 VALUES (2); SELECT * FROM test.t1;"' "$CONTAINER_NAME" +execInnerDocker "$CONTAINER_NAME" 'mariadb -e "INSERT INTO test.t1 VALUES (2); SELECT * FROM test.t1;"' diff --git a/build/utils.sh b/build/utils.sh index caec203a2..123171be0 100644 --- a/build/utils.sh +++ b/build/utils.sh @@ -490,57 +490,89 @@ EOF } function retry_eval() { + if [ "$#" -lt 2 ]; then + error "Usage: retry_eval " + return 1 + fi + local max_retries=$1 - local command=$2 + shift # Remove max_retries from arguments local attempt=1 local initial_delay=1 while [ "$attempt" -le "$max_retries" ]; do message_split - message "Attempt $attempt of $max_retries: $command" - if eval "$command"; then - message "Command '$command' done" + message "Attempt $attempt of $max_retries: $*" + if eval "$@"; then + message "Command '$@' done" message_split return 0 fi if [ "$attempt" -lt "$max_retries" ]; then delay=$((initial_delay * 2 ** (attempt - 1))) - message "Retrying command '$command' in $delay seconds..." + warn "Retrying command "$@" in $delay seconds..." message_split sleep "$delay" fi ((attempt++)) done - error "Max retries reached for command: $command" + error "Max retries reached for command: $*" message_split exit 13 } function execInnerDocker() { - local cmd_str="$1" - local img="$2" - local flags="${3:-}" + local container_name=$1 + shift 1 # Remove first arg (container_name) - docker exec $flags -t "$img" bash -c "$cmd_str" + docker exec -t "$container_name" bash -c "$@" local dockerCommandExitCode=$? if [[ $dockerCommandExitCode -ne 0 ]]; then - error "Command \"$cmd_str\" failed in container \"$img\"" + error "Command \"${cmd[@]}\" failed in container \"$container_name\"" exit $dockerCommandExitCode fi } +function change_ubuntu_mirror() { + local region="$1" + message "Changing Ubuntu mirror to $region" + sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list 2>/dev/null || true + sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || true + cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null | grep archive || true + message_split +} + function execInnerDockerWithRetry() { - local cmd_str="$1" - local img="$2" - local flags="${3:-}" + local container_name=$1 + shift 1 # Remove first three args (container_name, max_retries, retry_delay) - docker exec $flags -t "$img" bash -c " $(declare -f retry_eval); retry_eval 5 '$cmd_str'" - local dockerCommandExitCode=$? + local cmd=("$@") + local attempt=1 + local dockerCommandExitCode=0 + + local docker_funcs=$(declare -f retry_eval color_normal color_cyan color_yellow color_red error warn message message_split) + + # Build the full command to execute in docker + local full_command="$docker_funcs; retry_eval 5 \"${cmd[*]}\"" + + # Execute the command in docker + docker exec -t "$container_name" bash -c "$full_command" + dockerCommandExitCode=$? if [[ $dockerCommandExitCode -ne 0 ]]; then - error "Command \"$cmd_str\" failed in container \"$img\"" - exit $dockerCommandExitCode + error "Command \"${cmd[*]}\" failed in container \"$container_name\" after $max_retries attempts" + return $dockerCommandExitCode fi + + return 0 +} + +change_ubuntu_mirror_in_docker() { + local container_name=$1 + local region=$2 + local docker_funcs=$(declare -f color_normal color_cyan color_yellow color_red error warn message message_split change_ubuntu_mirror) + + execInnerDocker "$container_name" "$docker_funcs; change_ubuntu_mirror ${region}" }