MCOL-6071 drone refactor (#3641)

* chore(ci): MCOL-6071 less sccache granularity

* chore(ci): MCOL-6071 clean old versions of .deb/.rpm files in s3 when publishing

* chore(ci): MCOL-6071 fix unittests execution

* install sccache after deps, to have curl

* chore(ci) MCOL-6071 review fixes

---------

Co-authored-by: Leonid Fedorov <leonid.fedorov@mariadb.com>

.drone.jsonnet

@@ -4,27 +4,21 @@ local events = ["pull_request", "cron"];
 local current_branch = "stable-23.10";
 
 local servers = {
-  "stable-23.10": ["10.6-enterprise"],
+  [current_branch]: ["10.6-enterprise"],
 };
 
 local platforms = {
-  "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
+  [current_branch]: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
 };
 
 local platforms_arm = {
-  "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
+  [current_branch]: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
 };
 
+local builddir = "verylongdirnameforverystrangecpackbehavior";
 local customEnvCommandsMap = {
-  // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'],
-  "clang-20": [
-    "apt-get clean && apt-get update",
-    "apt-get install -y wget curl lsb-release software-properties-common gnupg",
-    "wget https://apt.llvm.org/llvm.sh",
-    "bash llvm.sh 20",
-    "export CC=/usr/bin/clang",
-    "export CXX=/usr/bin/clang++",
-  ],
+  // 'clang-18': ["bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/install_clang_deb.sh 18"],
+  "clang-20": ["bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/install_clang_deb.sh 20"],
 };
 
 local customEnvCommands(envkey, builddir) =
@@ -52,7 +46,6 @@ local customBootstrapParamsForAdditionalPipelinesMap = {
 
 local any_branch = "**";
 
-local builddir = "verylongdirnameforverystrangecpackbehavior";
 
 local mtr_suite_list = "basic,bugfixes";
 local mtr_full_set = "basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly";
@@ -115,6 +108,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
     name: "publish " + step_prefix,
     depends_on: [std.strReplace(step_prefix, " latest", ""), "createrepo"],
     image: "amazon/aws-cli:2.22.30",
+    volumes: [pipeline._volumes.mdb],
     when: {
       status: ["success", "failure"],
     },
@@ -132,7 +126,21 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
       "sleep 10",
       "ls -lR " + result,
 
-      "aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors --debug",
+     //clean old versions of .deb/.rpm files
+     "source /mdb/" + builddir + "/storage/columnstore/columnstore/VERSION && " +
+     "CURRENT_VERSION=${COLUMNSTORE_VERSION_MAJOR}.${COLUMNSTORE_VERSION_MINOR}.${COLUMNSTORE_VERSION_PATCH} && " +
+     "aws s3 rm s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + "/ " +
+     "--recursive " +
+     "--exclude \"*\" " +
+     // include only debs/rpms with columnstore in names
+     "--include \"*columnstore*.deb\" " +
+     "--include \"*columnstore*.rpm\" " +
+     // but do not delete the ones matching CURRENT_VERSION
+     "--exclude \"*${CURRENT_VERSION}*.deb\" " +
+     "--exclude \"*${CURRENT_VERSION}*.rpm\" " +
+     "--only-show-errors",
+
+      "aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors",
       'echo "Data uploaded to: ' + publish_pkg_url + '"',
       make_clickable_link(publish_pkg_url),
     ],
@@ -504,7 +512,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
                SCCACHE_BUCKET: "cs-sccache",
                SCCACHE_REGION: "us-east-1",
                SCCACHE_S3_USE_SSL: "true",
-               SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + "${DRONE_PULL_REQUEST}",
+               SCCACHE_S3_KEY_PREFIX: result + branch + server + arch,
              },
              commands: [
                          "mkdir /mdb/" + builddir + "/" + result,
@@ -514,6 +522,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
                         'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
                          "--build-type RelWithDebInfo " +
                          "--distro " + platform + " " +
+                         "--build-path " + "/mdb/" + builddir + "/builddir " +
                          "--build-packages --install-deps --sccache" +
                          " " + customBootstrapParams +
                          " " + customBootstrapParamsForExisitingPipelines(platform) + " | " +

build/bootstrap_mcs.sh

@@ -79,41 +79,30 @@ fi
 select_pkg_format ${OS}
 
 if [[ "$PKG_FORMAT" == "rpm" ]]; then
-    CTEST_BIN_NAME=:"ctest3"
+    CTEST_BIN_NAME="ctest3"
 else
     CTEST_BIN_NAME="ctest"
 fi
 
 install_sccache() {
     if [[ "$SCCACHE" == false ]]; then
-      return
+        return
     fi
 
     if [[ "$(arch)" == "x86_64" ]]; then
-      sccache_arch="x86_64"
+        sccache_arch="x86_64"
     else
-      sccache_arch="aarch64"
+        sccache_arch="aarch64"
     fi
 
     message "getting sccache..."
-
-    if command -v apt-get &>/dev/null; then
-      apt-get clean
-      apt-get update -y
-      apt-get install -y curl
-    elif command -v yum &>/dev/null; then
-      yum install -y curl
-    fi || true
-
     curl -L -o sccache.tar.gz \
-      "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz"
+        "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz"
 
     tar xzf sccache.tar.gz
     install sccache*/sccache /usr/local/bin/ && message "sccache installed"
 }
 
-install_sccache
-
 install_deps() {
     if [[ $INSTALL_DEPS = false ]]; then
         return
@@ -161,6 +150,7 @@ install_deps() {
 }
 
 install_deps
+install_sccache
 
 cd $COLUMSNTORE_SOURCE_PATH
 COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
@@ -776,15 +766,16 @@ construct_cmake_flags
 init_submodules
 
 if [[ $BUILD_PACKAGES = true ]]; then
-
     modify_packaging
-    build_package
-    message_splitted "PACKAGES BUILD FINISHED"
-    run_unit_tests
+
+    ( build_package && run_unit_tests )
+    exit_code=$?
+
     if [[ $SCCACHE = true ]]; then
-      sccache --show-stats
+        sccache --show-stats
     fi
-    exit 0
+
+    exit $exit_code
 fi
 
 stop_service

build/install_clang_deb.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -eo pipefail
+
+SCRIPT_LOCATION=$(dirname "$0")
+source "$SCRIPT_LOCATION"/utils.sh
+
+CLANG_VERSION="$1"
+
+if [[ $# -ne 1 ]]; then
+  echo "Please pass clang-version as a first parameter"
+  exit 1
+fi
+
+change_ubuntu_mirror us
+
+message "Installing clang-${CLANG_VERSION}"
+
+retry_eval 5 apt-get clean && apt-get update && apt-get install -y wget curl lsb-release software-properties-common gnupg
+wget https://apt.llvm.org/llvm.sh
+bash llvm.sh $CLANG_VERSION
+export CC=/usr/bin/clang
+export CXX=/usr/bin/clang++
+
+
+
+
+