diff --git a/.drone.jsonnet b/.drone.jsonnet index 6d28db57d..732ae9cae 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -36,8 +36,9 @@ local customEnvCommands(envkey, builddir) = local customBootstrapParamsForExisitingPipelines(envkey) = + # errorprone if we pass --custom-cmake-flags twice, the last one will win local customBootstrapMap = { -// "ubuntu:24.04": "--custom-cmake-flags '-DCOLUMNSTORE_ASAN_FOR_UNITTESTS=YES'", + "ubuntu:24.04": "--custom-cmake-flags '-DCOLUMNSTORE_ASAN_FOR_UNITTESTS=YES'", }; (if (std.objectHas(customBootstrapMap, envkey)) then customBootstrapMap[envkey] else ""); @@ -48,6 +49,7 @@ local customBootstrapParamsForAdditionalPipelinesMap = { UBSan: "--ubsan", MSan: "--msan", "libcpp": "--libcpp", + "gcc-toolset": "--gcc-toolset-for-rocky-8" }; local customBuildFlags(buildKey) = @@ -204,9 +206,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", " --docker-image " + img + " --result-path " + result + " --packages-url " + packages_url + - " --do-setup " + std.toString(do_setup) + - if result=="ubuntu24.04_clang-20_libcpp" then "" else " --install-libcpp " + //FIX THIS HACK - '"', + " --do-setup " + std.toString(do_setup) + '"', local reportTestStage(containerName, result, stage) = 'sh -c "apk add bash && ' + get_build_command("report_test_stage.sh") + @@ -530,6 +530,8 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", SCCACHE_S3_USE_SSL: "true", SCCACHE_S3_KEY_PREFIX: result + branch + server + arch, }, + + # errorprone if we pass --custom-cmake-flags twice, the last one will win commands: [ "mkdir /mdb/" + builddir + "/" + result, ] @@ -609,7 +611,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", [pipeline.cmapitest] + [pipeline.cmapilog] + [pipeline.publish("cmapilog")] + - (if (platform == "rockylinux:8" && arch == "amd64") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) + + (if (platform == "rockylinux:8" && arch == "amd64" && customBootstrapParamsKey == "gcc-toolset") then [pipeline.dockerfile] + [pipeline.dockerhub] + [pipeline.multi_node_mtr] else [pipeline.mtr] + [pipeline.mtrlog] + [pipeline.publish("mtrlog")]) + [pipeline.regression(regression_tests[i], if (i == 0) then ["mtr", "publish pkg", "publish cmapi build"] else [regression_tests[i - 1]]) for i in indexes(regression_tests)] + [pipeline.regressionlog] + // [pipeline.upgrade(mdb_server_versions[i]) for i in indexes(mdb_server_versions)] + @@ -691,13 +693,11 @@ local FinalPipeline(branch, event) = { ] + [ - Pipeline(b, platform, triggeringEvent, a, server, flag, envcommand) + Pipeline(b, platform, triggeringEvent, a, server, flag, "") for a in ["amd64"] for b in std.objectFields(platforms) - for platform in ["ubuntu:24.04"] - for flag in ["libcpp"] - for envcommand in ["clang-20"] -// for flag in std.objectFields(customBootstrapParamsForAdditionalPipelinesMap) + for platform in ["rockylinux:8"] + for flag in ["gcc-toolset"] for triggeringEvent in events for server in servers[current_branch] ] diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 250cb245f..a7846c933 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -52,6 +52,7 @@ optparse.define short=O long=static desc="Build all with static libraries" varia optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false +optparse.define short=R long=gcc-toolset-for-rocky-8 variable=GCC_TOOLSET default=false value=true optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true @@ -109,12 +110,12 @@ install_deps() { fi message_split prereq="" - RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \ + RPM_BUILD_DEPS="lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \ snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \ rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \ cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel" - DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \ + DEB_BUILD_DEPS="build-essential automake libboost-all-dev \ bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \ libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \ @@ -122,16 +123,20 @@ install_deps() { if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \ - dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \ - ${RPM_BUILD_DEPS}" + dnf install -y libarchive cmake ${RPM_BUILD_DEPS}" + if [[ $GCC_TOOLSET = false ]]; then + command="$command && dnf group install -y \"Development Tools\"" + else + command="$command && dnf install -y gcc-toolset-${GCC_VERSION} && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable" + fi elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]] then command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \ - dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}" + dnf install -y pcre2-devel gcc gcc-c++ curl-minimal ${RPM_BUILD_DEPS}" elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then - command="${DEB_BUILD_DEPS}" + command="apt-get -y update && apt-get -y install ${DEB_BUILD_DEPS}" else echo "Unsupported OS: $OS" exit 17 @@ -768,7 +773,7 @@ init_submodules if [[ $BUILD_PACKAGES = true ]]; then modify_packaging - ( build_package && run_unit_tests ) + (build_package && run_unit_tests) exit_code=$? if [[ $SCCACHE = true ]]; then diff --git a/build/install_libc++.sh b/build/install_libc++.sh new file mode 100755 index 000000000..c2c5a32ac --- /dev/null +++ b/build/install_libc++.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPT_LOCATION=$(dirname "$0") +source "$SCRIPT_LOCATION"/utils.sh + +VERSION="$1" + +if [[ $# -ne 1 ]]; then + echo "Please pass clang-version as a first parameter" + exit 1 +fi + +change_ubuntu_mirror us + +message "Installing libc++-${VERSION}" + +retry_eval 5 apt-get clean && apt-get update && apt-get install -y libc++-${VERSION}-dev libc++abi-${VERSION}-dev