1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00

MCOL-6071 drone refactor (#3641)

* chore(ci): MCOL-6071 less sccache granularity

* chore(ci): MCOL-6071 clean old versions of .deb/.rpm files in s3 when publishing

* chore(ci): MCOL-6071 fix unittests execution

* install sccache after deps, to have curl

* chore(ci) MCOL-6071 review fixes

---------

Co-authored-by: Leonid Fedorov <leonid.fedorov@mariadb.com>
This commit is contained in:
Aleksei Bukhalov
2025-07-09 12:20:21 +02:00
committed by GitHub
parent b6a5c1d71f
commit aa3db28074
3 changed files with 65 additions and 37 deletions

View File

@ -4,27 +4,21 @@ local events = ["pull_request", "cron"];
local current_branch = "stable-23.10";
local servers = {
"stable-23.10": ["10.6-enterprise"],
[current_branch]: ["10.6-enterprise"],
};
local platforms = {
"stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
[current_branch]: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
};
local platforms_arm = {
"stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
[current_branch]: ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:22.04", "ubuntu:24.04"],
};
local builddir = "verylongdirnameforverystrangecpackbehavior";
local customEnvCommandsMap = {
// 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'],
"clang-20": [
"apt-get clean && apt-get update",
"apt-get install -y wget curl lsb-release software-properties-common gnupg",
"wget https://apt.llvm.org/llvm.sh",
"bash llvm.sh 20",
"export CC=/usr/bin/clang",
"export CXX=/usr/bin/clang++",
],
// 'clang-18': ["bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/install_clang_deb.sh 18"],
"clang-20": ["bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/install_clang_deb.sh 20"],
};
local customEnvCommands(envkey, builddir) =
@ -52,7 +46,6 @@ local customBootstrapParamsForAdditionalPipelinesMap = {
local any_branch = "**";
local builddir = "verylongdirnameforverystrangecpackbehavior";
local mtr_suite_list = "basic,bugfixes";
local mtr_full_set = "basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly";
@ -115,6 +108,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
name: "publish " + step_prefix,
depends_on: [std.strReplace(step_prefix, " latest", ""), "createrepo"],
image: "amazon/aws-cli:2.22.30",
volumes: [pipeline._volumes.mdb],
when: {
status: ["success", "failure"],
},
@ -132,7 +126,21 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
"sleep 10",
"ls -lR " + result,
"aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors --debug",
//clean old versions of .deb/.rpm files
"source /mdb/" + builddir + "/storage/columnstore/columnstore/VERSION && " +
"CURRENT_VERSION=${COLUMNSTORE_VERSION_MAJOR}.${COLUMNSTORE_VERSION_MINOR}.${COLUMNSTORE_VERSION_PATCH} && " +
"aws s3 rm s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + "/ " +
"--recursive " +
"--exclude \"*\" " +
// include only debs/rpms with columnstore in names
"--include \"*columnstore*.deb\" " +
"--include \"*columnstore*.rpm\" " +
// but do not delete the ones matching CURRENT_VERSION
"--exclude \"*${CURRENT_VERSION}*.deb\" " +
"--exclude \"*${CURRENT_VERSION}*.rpm\" " +
"--only-show-errors",
"aws s3 sync " + result + "/" + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors",
'echo "Data uploaded to: ' + publish_pkg_url + '"',
make_clickable_link(publish_pkg_url),
],
@ -504,7 +512,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
SCCACHE_BUCKET: "cs-sccache",
SCCACHE_REGION: "us-east-1",
SCCACHE_S3_USE_SSL: "true",
SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + "${DRONE_PULL_REQUEST}",
SCCACHE_S3_KEY_PREFIX: result + branch + server + arch,
},
commands: [
"mkdir /mdb/" + builddir + "/" + result,
@ -514,6 +522,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
"--build-type RelWithDebInfo " +
"--distro " + platform + " " +
"--build-path " + "/mdb/" + builddir + "/builddir " +
"--build-packages --install-deps --sccache" +
" " + customBootstrapParams +
" " + customBootstrapParamsForExisitingPipelines(platform) + " | " +

View File

@ -79,7 +79,7 @@ fi
select_pkg_format ${OS}
if [[ "$PKG_FORMAT" == "rpm" ]]; then
CTEST_BIN_NAME=:"ctest3"
CTEST_BIN_NAME="ctest3"
else
CTEST_BIN_NAME="ctest"
fi
@ -96,15 +96,6 @@ install_sccache() {
fi
message "getting sccache..."
if command -v apt-get &>/dev/null; then
apt-get clean
apt-get update -y
apt-get install -y curl
elif command -v yum &>/dev/null; then
yum install -y curl
fi || true
curl -L -o sccache.tar.gz \
"https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${sccache_arch}-unknown-linux-musl.tar.gz"
@ -112,8 +103,6 @@ install_sccache() {
install sccache*/sccache /usr/local/bin/ && message "sccache installed"
}
install_sccache
install_deps() {
if [[ $INSTALL_DEPS = false ]]; then
return
@ -161,6 +150,7 @@ install_deps() {
}
install_deps
install_sccache
cd $COLUMSNTORE_SOURCE_PATH
COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
@ -776,15 +766,16 @@ construct_cmake_flags
init_submodules
if [[ $BUILD_PACKAGES = true ]]; then
modify_packaging
build_package
message_splitted "PACKAGES BUILD FINISHED"
run_unit_tests
( build_package && run_unit_tests )
exit_code=$?
if [[ $SCCACHE = true ]]; then
sccache --show-stats
fi
exit 0
exit $exit_code
fi
stop_service

28
build/install_clang_deb.sh Executable file
View File

@ -0,0 +1,28 @@
#!/bin/bash
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
source "$SCRIPT_LOCATION"/utils.sh
CLANG_VERSION="$1"
if [[ $# -ne 1 ]]; then
echo "Please pass clang-version as a first parameter"
exit 1
fi
change_ubuntu_mirror us
message "Installing clang-${CLANG_VERSION}"
retry_eval 5 apt-get clean && apt-get update && apt-get install -y wget curl lsb-release software-properties-common gnupg
wget https://apt.llvm.org/llvm.sh
bash llvm.sh $CLANG_VERSION
export CC=/usr/bin/clang
export CXX=/usr/bin/clang++