From aa3db28074c813ec9e85568f8ae49cf499d74b00 Mon Sep 17 00:00:00 2001 From: Aleksei Bukhalov Date: Wed, 9 Jul 2025 12:20:21 +0200 Subject: [PATCH] MCOL-6071 drone refactor (#3641) * chore(ci): MCOL-6071 less sccache granularity * chore(ci): MCOL-6071 clean old versions of .deb/.rpm files in s3 when publishing * chore(ci): MCOL-6071 fix unittests execution * install sccache after deps, to have curl * chore(ci) MCOL-6071 review fixes --------- Co-authored-by: Leonid Fedorov --- .drone.jsonnet | 39 +++++++++++++++++++++++--------------- build/bootstrap_mcs.sh | 35 +++++++++++++--------------------- build/install_clang_deb.sh | 28 +++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 37 deletions(-) create mode 100755 build/install_clang_deb.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index ac352a2aa..6e687b951 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -4,27 +4,21 @@ local events = ["pull_request", "cron"]; local current_branch = "stable-23.10"; local servers = { - "stable-23.10": ["10.6-enterprise"], + [current_branch]: ["10.6-enterprise"], }; local platforms = { - "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], + [current_branch]: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], }; local platforms_arm = { - "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], + [current_branch]: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"], }; +local builddir = "verylongdirnameforverystrangecpackbehavior"; local customEnvCommandsMap = { - // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'], - "clang-20": [ - "apt-get clean && apt-get update", - "apt-get install -y wget curl lsb-release software-properties-common gnupg", - "wget https://apt.llvm.org/llvm.sh", - "bash llvm.sh 20", - "export CC=/usr/bin/clang", - "export CXX=/usr/bin/clang++", - ], + // 'clang-18': ["bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/install_clang_deb.sh 18"], + "clang-20": ["bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/install_clang_deb.sh 20"], }; local customEnvCommands(envkey, builddir) = @@ -52,7 +46,6 @@ local customBootstrapParamsForAdditionalPipelinesMap = { local any_branch = "**"; -local builddir = "verylongdirnameforverystrangecpackbehavior"; local mtr_suite_list = "basic,bugfixes"; local mtr_full_set = "basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly"; @@ -115,6 +108,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", name: "publish " + step_prefix, depends_on: [std.strReplace(step_prefix, " latest", ""), "createrepo"], image: "amazon/aws-cli:2.22.30", + volumes: [pipeline._volumes.mdb], when: { status: ["success", "failure"], }, @@ -132,7 +126,21 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", "sleep 10", "ls -lR " + result, - "aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors --debug", + //clean old versions of .deb/.rpm files + "source /mdb/" + builddir + "/storage/columnstore/columnstore/VERSION && " + + "CURRENT_VERSION=${COLUMNSTORE_VERSION_MAJOR}.${COLUMNSTORE_VERSION_MINOR}.${COLUMNSTORE_VERSION_PATCH} && " + + "aws s3 rm s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + "/ " + + "--recursive " + + "--exclude \"*\" " + + // include only debs/rpms with columnstore in names + "--include \"*columnstore*.deb\" " + + "--include \"*columnstore*.rpm\" " + + // but do not delete the ones matching CURRENT_VERSION + "--exclude \"*${CURRENT_VERSION}*.deb\" " + + "--exclude \"*${CURRENT_VERSION}*.rpm\" " + + "--only-show-errors", + + "aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors", 'echo "Data uploaded to: ' + publish_pkg_url + '"', make_clickable_link(publish_pkg_url), ], @@ -504,7 +512,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", SCCACHE_BUCKET: "cs-sccache", SCCACHE_REGION: "us-east-1", SCCACHE_S3_USE_SSL: "true", - SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + "${DRONE_PULL_REQUEST}", + SCCACHE_S3_KEY_PREFIX: result + branch + server + arch, }, commands: [ "mkdir /mdb/" + builddir + "/" + result, @@ -514,6 +522,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + "--build-type RelWithDebInfo " + "--distro " + platform + " " + + "--build-path " + "/mdb/" + builddir + "/builddir " + "--build-packages --install-deps --sccache" + " " + customBootstrapParams + " " + customBootstrapParamsForExisitingPipelines(platform) + " | " + diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index eed9873c8..250cb245f 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -79,41 +79,30 @@ fi select_pkg_format ${OS} if [[ "$PKG_FORMAT" == "rpm" ]]; then - CTEST_BIN_NAME=:"ctest3" + CTEST_BIN_NAME="ctest3" else CTEST_BIN_NAME="ctest" fi install_sccache() { if [[ "$SCCACHE" == false ]]; then - return + return fi if [[ "$(arch)" == "x86_64" ]]; then - sccache_arch="x86_64" + sccache_arch="x86_64" else - sccache_arch="aarch64" + sccache_arch="aarch64" fi message "getting sccache..." - - if command -v apt-get &>/dev/null; then - apt-get clean - apt-get update -y - apt-get install -y curl - elif command -v yum &>/dev/null; then - yum install -y curl - fi || true - curl -L -o sccache.tar.gz \ - "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz" + "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz" tar xzf sccache.tar.gz install sccache*/sccache /usr/local/bin/ && message "sccache installed" } -install_sccache - install_deps() { if [[ $INSTALL_DEPS = false ]]; then return @@ -161,6 +150,7 @@ install_deps() { } install_deps +install_sccache cd $COLUMSNTORE_SOURCE_PATH COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD) @@ -776,15 +766,16 @@ construct_cmake_flags init_submodules if [[ $BUILD_PACKAGES = true ]]; then - modify_packaging - build_package - message_splitted "PACKAGES BUILD FINISHED" - run_unit_tests + + ( build_package && run_unit_tests ) + exit_code=$? + if [[ $SCCACHE = true ]]; then - sccache --show-stats + sccache --show-stats fi - exit 0 + + exit $exit_code fi stop_service diff --git a/build/install_clang_deb.sh b/build/install_clang_deb.sh new file mode 100755 index 000000000..b1c9fa9b3 --- /dev/null +++ b/build/install_clang_deb.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +source "$SCRIPT_LOCATION"/utils.sh + +CLANG_VERSION="$1" + +if [[ $# -ne 1 ]]; then + echo "Please pass clang-version as a first parameter" + exit 1 +fi + +change_ubuntu_mirror us + +message "Installing clang-${CLANG_VERSION}" + +retry_eval 5 apt-get clean && apt-get update && apt-get install -y wget curl lsb-release software-properties-common gnupg +wget https://apt.llvm.org/llvm.sh +bash llvm.sh $CLANG_VERSION +export CC=/usr/bin/clang +export CXX=/usr/bin/clang++ + + + + +