diff --git a/.drone.jsonnet b/.drone.jsonnet index f0dd7ad73..94ccceb79 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -23,42 +23,6 @@ local platforms_mtr = platforms.develop; local builddir = 'verylongdirnameforverystrangecpackbehavior'; -local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CONFIG=mysql_release ' + - '-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ' + - '-DPLUGIN_COLUMNSTORE=YES -DWITH_UNITTESTS=YES ' + - '-DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO ' + - '-DPLUGIN_CONNECT=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' + - '-DPLUGIN_GSSAPI=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' + - '-DWITH_EMBEDDED_SERVER=NO -DWITH_WSREP=NO -DWITH_COREDUMPS=ON'; - -local clang_version = '16'; -local gcc_version = '11'; - -local clang_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-' + clang_version + ' 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-' + clang_version + ' && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 '; - - -local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison ' + - 'snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool ' + - 'policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel ' + - 'expect createrepo python3 '; - -local rockylinux8_build_deps = "dnf install -y 'dnf-command(config-manager)' " + - '&& dnf config-manager --set-enabled powertools ' + - '&& dnf install -y gcc-toolset-' + gcc_version + ' libarchive cmake lz4-devel ' + - '&& . /opt/rh/gcc-toolset-' + gcc_version + '/enable '; - - -local rockylinux9_build_deps = "dnf install -y 'dnf-command(config-manager)' " + - '&& dnf config-manager --set-enabled crb ' + - '&& dnf install -y pcre2-devel lz4-devel gcc gcc-c++'; - -local debian11_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' && ' + clang_update_alternatives; -local ubuntu20_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' && ' + clang_update_alternatives; - -local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata libssl-dev && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i '; -local turnon_clang = 'export CC=/usr/bin/clang; export CXX=/usr/bin/clang++ '; -local bootstrap_deps = 'apt-get -y update && apt-get -y install build-essential automake libboost-all-dev bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl '; - local mtr_suite_list = 'basic,bugfixes'; local mtr_full_set = 'basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly'; @@ -94,20 +58,6 @@ local upgrade_test_lists = { }, }; -local platformMap(platform, arch) = - local platform_map = { - 'rockylinux:8': rockylinux8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package', - 'rockylinux:9': rockylinux9_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux9 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package', - 'debian:11': bootstrap_deps + ' && ' + deb_build_deps + ' && ' + debian11_deps + ' && ' + turnon_clang + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=bullseye' debian/autobake-deb.sh", - 'debian:12': bootstrap_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=bookworm' debian/autobake-deb.sh", - 'ubuntu:20.04': bootstrap_deps + ' && ' + deb_build_deps + ' && ' + ubuntu20_04_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh", - 'ubuntu:22.04': bootstrap_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=jammy' debian/autobake-deb.sh", - 'ubuntu:24.04': bootstrap_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=jammy' debian/autobake-deb.sh", - }; - local result = std.strReplace(std.strReplace(platform, ':', ''), '/', '-'); - 'export CLICOLOR_FORCE=1; ' + platform_map[platform] + ' | storage/columnstore/columnstore/build/ansi2txt.sh ' + result + '/build.log'; - - local testRun(platform) = local platform_map = { 'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure', @@ -121,16 +71,30 @@ local testRun(platform) = }; platform_map[platform]; +local gcc_version = '11'; + +local rockylinux8_deps = "dnf install -y 'dnf-command(config-manager)' " + + '&& dnf config-manager --set-enabled powertools ' + + '&& dnf install -y gcc-toolset-' + gcc_version + ' libarchive cmake ' + + '&& . /opt/rh/gcc-toolset-' + gcc_version + '/enable '; + +local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " + + '&& dnf config-manager --set-enabled crb ' + + '&& dnf install -y gcc gcc-c++'; + +local rockylinux_common_deps = ' && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel'; + +local deb_deps = 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev'; local testPreparation(platform) = local platform_map = { - 'rockylinux:8': rockylinux8_build_deps + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel', - 'rockylinux:9': rockylinux9_build_deps + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel', - 'debian:11': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev', - 'debian:12': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev', - 'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev', - 'ubuntu:22.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev', - 'ubuntu:24.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev', + 'rockylinux:8': rockylinux8_deps + rockylinux_common_deps, + 'rockylinux:9': rockylinux9_deps + rockylinux_common_deps, + 'debian:11': deb_deps, + 'debian:12': deb_deps, + 'ubuntu:20.04': deb_deps, + 'ubuntu:22.04': deb_deps, + 'ubuntu:24.04': deb_deps, }; platform_map[platform]; @@ -161,9 +125,10 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') local server_remote = if (std.endsWith(server, 'enterprise')) then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server', local sccache_arch = if (arch == 'amd64') then 'x86_64' else 'aarch64', - local get_sccache = 'curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.0/sccache-v0.3.0-' + sccache_arch + '-unknown-linux-musl.tar.gz ' + + local get_sccache = 'echo getting sccache... && (apt update -y && apt install -y curl || yum install -y curl || true) ' + + '&& curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-' + sccache_arch + '-unknown-linux-musl.tar.gz ' + '&& tar xzf sccache.tar.gz ' + - '&& install sccache*/sccache /usr/local/bin/', + '&& install sccache*/sccache /usr/local/bin/ && echo sccache installed', local pipeline = self, @@ -238,11 +203,11 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') local dockerImage(stepname) = stepname + "$${DRONE_BUILD_NUMBER}", local installEngine(dockerImage, pkg_format) = - if (pkg_format == 'deb') then execInnerDocker('bash -c "apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage) + if (pkg_format == 'deb') then execInnerDocker('bash -c "apt update && apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage) else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"', dockerImage), local installCmapi(dockerImage, pkg_format) = - if (pkg_format == 'deb') then execInnerDocker('bash -c "apt install -y mariadb-columnstore-cmapi"', dockerImage) + if (pkg_format == 'deb') then execInnerDocker('bash -c "apt update && apt install -y mariadb-columnstore-cmapi"', dockerImage) else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-cmapi"', dockerImage), local prepareTestStage(dockerImage, pkg_format, result, do_setup) = [ @@ -770,41 +735,32 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') from_secret: 'aws_secret_access_key', }, SCCACHE_BUCKET: 'cs-sccache', + SCCACHE_REGION: 'us-east-1', SCCACHE_S3_USE_SSL: 'true', SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + '${DRONE_PULL_REQUEST}', //SCCACHE_ERROR_LOG: '/tmp/sccache_log.txt', //SCCACHE_LOG: 'debug', }, commands: [ - 'cd /mdb/' + builddir, - 'ls -la ../', - 'mkdir ' + result, - "sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh", - if (std.startsWith(server, '10.6')) then "sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control", - // Remove Debian build flags that could prevent ColumnStore from building - "sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules", - // Disable dh_missing strict check for missing files - 'sed -i s/--fail-missing/--list-missing/ debian/rules', - // Tweak debian packaging stuff - 'for i in mariadb-plugin libmariadbd; do sed -i "/Package: $i.*/,/^$/d" debian/control; done', - "sed -i 's/Depends: galera.*/Depends:/' debian/control", - 'for i in galera wsrep ha_sphinx embedded; do sed -i /$i/d debian/*.install; done', - // Install build dependencies for deb - if (pkg_format == 'deb') then "apt-cache madison liburing-dev | grep liburing-dev || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules && apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules && sed '/libfmt-dev/d' -i debian/control", - // Change plugin_maturity level - // "sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt", - if (pkg_format == 'deb') then 'apt update -y && apt install -y curl' else if (platform == 'rockylinux:9') then 'yum install -y curl-minimal' else 'yum install -y curl', - get_sccache, - testPreparation(platform), - // disable LTO for 22.04 for now - if (platform == 'ubuntu:22.04' || platform == 'ubuntu:24.04') then 'apt install -y lto-disabled-list && for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do echo "$i any" >> /usr/share/lto-disabled-list/lto-disabled-list; done && grep mariadb /usr/share/lto-disabled-list/lto-disabled-list', - platformMap(platform, arch), - 'sccache --show-stats', - // move engine and cmapi packages to one dir to make a repo - 'mv -v -t ./%s/ %s/*.%s /drone/src/cmapi/%s/*.%s ' % [result, if (pkg_format == 'rpm') then '.' else '..', pkg_format, result, pkg_format], - if (pkg_format == 'rpm') then 'createrepo ./' + result else 'dpkg-scanpackages %s | gzip > ./%s/Packages.gz' % [result, result], - // list storage manager binary - 'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager', + 'export CLICOLOR_FORCE=1', + get_sccache, + 'mkdir /mdb/' + builddir + '/' + result, + + 'bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/bootstrap_mcs.sh ' + + '--build-type RelWithDebInfo ' + + '--distro ' + platform + ' ' + + '--build-packages --sccache ' + + '--server-version ' + server + '|' + + '/mdb/' + builddir + '/storage/columnstore/columnstore/build/ansi2txt.sh ' + + '/mdb/' + builddir + '/' + result + '/build.log', + 'sccache --show-stats', + + // move engine and cmapi packages to one dir and make a repo + if (pkg_format == 'rpm') then "mv -v -t ./" + result + "/ /mdb/" + builddir + "/*.rpm /drone/src/cmapi/" + result + "/*.rpm && createrepo ./" + result + else "mv -v -t ./" + result + "/ /mdb/*.deb /drone/src/cmapi/" + result + "/*.deb && dpkg-scanpackages " + result + " | gzip > ./" + result + "/Packages.gz", + + // list storage manager binary + 'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager', ], }, { diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 3821eef46..cd6774d24 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -5,26 +5,37 @@ # - the script is to be run under root. set -o pipefail + +INSTALL_PREFIX="/usr/" +DATA_DIR="/var/lib/mysql/data" +CMAKE_BIN_NAME=cmake +CTEST_BIN_NAME=ctest +RPM_CONFIG_DIR="/etc/my.cnf.d" +DEB_CONFIG_DIR="/etc/mysql/mariadb.conf.d" +CONFIG_DIR=$RPM_CONFIG_DIR + SCRIPT_LOCATION=$(dirname "$0") MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../..) +BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") +DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9") + +GCC_VERSION="11" +BUILD_DELAY_SECONDS="${BUILD_DELAY_SECONDS:-1s}" +BUILD_DIR="verylongdirnameforverystrangecpackbehavior" +MDB_CMAKE_FLAGS="" + source $SCRIPT_LOCATION/utils.sh + if [ "$EUID" -ne 0 ] then error "Please run script as root to install MariaDb to system paths" exit 1 fi +echo "Arguments received: $@" message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" -BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") -DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky") - -cd $SCRIPT_LOCATION -CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) -cd - > /dev/null - optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=d long=distro desc="Choose your OS: ${DISTRO_OPTIONS[*]}" variable=OS @@ -32,7 +43,7 @@ optparse.define short=D long=install-deps desc="Install dependences" variable=IN optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true -optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true +optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true @@ -51,7 +62,9 @@ optparse.define short=f long=do-not-freeze-revision desc="Disable revision freez optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$MDB_SOURCE_PATH/../MariaDBBuild optparse.define short=o long=recompile-only variable=RECOMPILE_ONLY default=false value=true optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false - +optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE default=false value=true +optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true +optparse.define short=R long=server-version desc="MariaDB server version" variable=MARIADB_SERVER_VERSION source $( optparse.build ) @@ -60,24 +73,19 @@ if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then MCS_BUILD_TYPE=$selectedChoice fi -if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "CentOS" ]]; then +if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then + echo "OS is empty, trying to detect..." detect_distro fi -INSTALL_PREFIX="/usr/" -DATA_DIR="/var/lib/mysql/data" -CMAKE_BIN_NAME=cmake -CTEST_BIN_NAME=ctest -RPM_CONFIG_DIR="/etc/my.cnf.d" -DEB_CONFIG_DIR="/etc/mysql/mariadb.conf.d" -CONFIG_DIR=$RPM_CONFIG_DIR - -if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then - CONFIG_DIR=$DEB_CONFIG_DIR +pkg_format="deb" +if [[ "$OS" == *"rocky"* ]]; then + pkg_format="rpm" fi -export CLICOLOR_FORCE=1 - +if [[ $pkg_format = "deb" ]]; then + CONFIG_DIR=$DEB_CONFIG_DIR +fi disable_git_restore_frozen_revision() { @@ -90,6 +98,7 @@ select_branch() { cd $SCRIPT_LOCATION CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then if [[ $BRANCH = "" ]]; then @@ -111,43 +120,37 @@ select_branch() install_deps() { message_split - message "Installing deps" - if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then - apt-get -y update - apt-get -y install build-essential automake libboost-all-dev bison cmake \ - libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ - libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ - libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev \ - liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev graphviz - elif [[ $OS = 'CentOS' || $OS = 'Rocky' || $OS = 'Fedora' ]]; then - if [[ "$OS_VERSION" == "7" ]]; then - yum -y install cmake3 epel-release centos-release-scl - CMAKE_BIN_NAME=cmake3 - CTEST_BIN_NAME=ctest3 - else - yum -y install cmake - fi - if [ $OS = 'Rocky' ]; then - if [[ "$OS_VERSION" == "9" ]]; then - dnf install -y 'dnf-command(config-manager)' - dnf config-manager --set-enabled crb - fi - yum -y groupinstall "Development Tools" && yum config-manager --set-enabled powertools - yum install -y checkpolicy - fi - if [[ $OS != 'Fedora' ]]; then - yum -y install epel-release - fi + RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \ + snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \ + rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \ + cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel" + + DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \ + bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ + libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \ + libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \ + graphviz devscripts ccache equivs eatmydata curl" + + if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then + command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \ + dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \ + ${RPM_BUILD_DEPS}" + + elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]; then + command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \ + dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}" + + elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then + command="${DEB_BUILD_DEPS}" - yum install -y bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ - vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts \ - jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel lzo-devel xz-devel lz4-devel bzip2-devel \ - pcre2-devel flex graphviz libaio-devel openssl-devel flex else - error "Unsupported OS $OS" - exit 17 + echo "Unsupported OS: $OS" + exit 17 fi + + message "Installing dependencies for $OS" + eval "$command" } stop_service() @@ -217,7 +220,213 @@ clean_old_installation() rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf } -build() +modify_packaging() { + echo "Modifying_packaging..." + cd $MDB_SOURCE_PATH + + if [[ $pkg_format == "deb" ]]; then + sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh + fi + + if [[ "$MARIADB_SERVER_VERSION" == 10.6* ]]; then + sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control + fi + + #disable LTO for 22.04 for now + if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then + apt install -y lto-disabled-list && + for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do + echo "$i any" >> /usr/share/lto-disabled-list/lto-disabled-list + done && + grep mariadb /usr/share/lto-disabled-list/lto-disabled-list + fi + + if [[ $pkg_format == "deb" ]]; then + apt-cache madison liburing-dev | grep liburing-dev || { + sed 's/liburing-dev/libaio-dev/g' -i debian/control && + sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && + sed '/-DWITH_URING=yes/d' -i debian/rules + } + apt-cache madison libpmem-dev | grep 'libpmem-dev' || { + sed '/libpmem-dev/d' -i debian/control && + sed '/-DWITH_PMEM/d' -i debian/rules + } + sed '/libfmt-dev/d' -i debian/control + + # Remove Debian build flags that could prevent ColumnStore from building + sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules + + # Disable dh_missing strict check for missing files + sed -i 's/--fail-missing/--list-missing/' debian/rules + + # Tweak debian packaging stuff + for i in mariadb-plugin libmariadbd; + do + sed -i "/Package: $i.*/,/^$/d" debian/control + done + + sed -i 's/Depends: galera.*/Depends:/' debian/control + + for i in galera wsrep ha_sphinx embedded; + do + sed -i "/$i/d" debian/*.install + done + fi +} + +construct_cmake_flags(){ + + MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes \ + -DPLUGIN_COLUMNSTORE=YES \ + -DPLUGIN_MROONGA=NO \ + -DPLUGIN_ROCKSDB=NO \ + -DPLUGIN_TOKUDB=NO \ + -DPLUGIN_CONNECT=NO \ + -DPLUGIN_SPIDER=NO \ + -DPLUGIN_OQGRAPH=NO \ + -DPLUGIN_SPHINX=NO \ + -DWITH_EMBEDDED_SERVER=NO \ + -DBUILD_CONFIG=mysql_release \ + -DWITH_WSREP=NO \ + -DWITH_SSL=system \ + -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=1 \ + -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE \ + -DPLUGIN_GSSAPI=NO" + + if [[ $SKIP_UNIT_TESTS = true ]] ; then + warn "Unittests are not build" + + else + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UNITTESTS=YES" + message "Buiding with unittests" + fi + + if [[ $DRAW_DEPS = true ]] ; then + warn "Generating dependendies graph to mariadb.dot" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} --graphviz=mariadb.dot" + fi + + if [[ $USE_NINJA = true ]] ; then + warn "Using Ninja instead of Makefiles" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja" + fi + + if [[ $ASAN = true ]] ; then + warn "Building with Address Sanitizer " + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + fi + + if [[ $TSAN = true ]] ; then + warn "Building with Thread Sanitizer" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + fi + + if [[ $UBSAN = true ]] ; then + warn "Building with UB Sanitizer" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + fi + + if [[ $WITHOUT_COREDUMPS = true ]] ; then + warn "Cores are not dumped" + else + warn "Building with CoreDumps" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON" + + if [ -f /.dockerenv ] ; then + warn "Build is executed in Docker, core dumps saving path /proc/sys/kernel/core_pattern will not be configured!" + else + warn "/proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p" + echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern + fi + fi + + if [[ $MAKEFILE_VERBOSE = true ]] ; then + warn "Verbosing Makefile Commands" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" + fi + + if [[ $SCCACHE = true ]] ; then + warn "Use sccache" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" + fi + + if [[ $RUN_BENCHMARKS = true ]] ; then + if [[ $MCS_BUILD_TYPE = 'Debug' ]] ; then + error "Benchmarks will not be build in run in Debug build Mode" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" + $RUN_BENCHMARKS = false + elif [[ $OS != *"ubuntu"* && $OS != *"debian"* ]] ; then + error "Benchmarks are now avaliable only at Ubuntu or Debian" + MAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" + $RUN_BENCHMARKS = false + else + message "Compile with microbenchmarks" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES" + fi + else + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" + message "Buiding without microbenchmarks" + fi + + + if [[ "$OS" == *"rocky"* ]]; then + OS_VERSION=${OS//[^0-9]/} + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=rockylinux${OS_VERSION}" + elif [[ "$OS" == "debian:11" ]]; then + CODENAME="bullseye" + elif [[ "$OS" == "debian:12" ]]; then + CODENAME="bookworm" + elif [[ "$OS" == "ubuntu:20.04" ]]; then + CODENAME="focal" + elif [[ "$OS" == "ubuntu:22.04" ]]; then + CODENAME="jammy" + elif [[ "$OS" == "ubuntu:24.04" ]]; then + CODENAME="noble" + else + echo "Unsupported OS: $OS" + exit 17 + fi + + if [[ -n "$CODENAME" ]]; then + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=${CODENAME}" + fi + + if [[ $PRINT_CMAKE_FLAGS = true ]] ; then + message "Building with flags" + newline_array ${MDB_CMAKE_FLAGS[@]} + fi +} + +init_submodules(){ + if [[ $SKIP_SUBMODULES = true ]] ; then + warn "Skipping initialization of columnstore submodules" + else + message "Initialization of columnstore submodules" + cd $MDB_SOURCE_PATH + git submodule update --init --recursive + cd - > /dev/null + fi +} + +build_package() { + modify_packaging + RESULT_DIR=$(echo "$OS" | sed 's/://g' | sed 's/\//-/g') + mkdir $RESULT_DIR + + if [[ $pkg_format == "rpm" ]]; then + command="cmake ${MDB_CMAKE_FLAGS} && sleep ${BUILD_DELAY_SECONDS} && make -j\$(nproc) package" + else + command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \ + sleep ${BUILD_DELAY_SECONDS} && CMAKEFLAGS='${MDB_CMAKE_FLAGS}' debian/autobake-deb.sh" + fi + + echo "Building a package for $OS" + echo "Build command: $command" + eval "$command" + } + +build_binary() { MARIA_BUILD_PATH=$(realpath $MARIA_BUILD_PATH) message_split @@ -225,122 +434,16 @@ build() message "Compiled artifacts will be written to $color_yellow$MARIA_BUILD_PATH$color_cyan" mkdir -p $MARIA_BUILD_PATH - local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes - -DPLUGIN_COLUMNSTORE=YES - -DPLUGIN_MROONGA=NO - -DPLUGIN_ROCKSDB=NO - -DPLUGIN_TOKUDB=NO - -DPLUGIN_CONNECT=NO - -DPLUGIN_SPIDER=NO - -DPLUGIN_OQGRAPH=NO - -DPLUGIN_SPHINX=NO - -DWITH_EMBEDDED_SERVER=OFF - -DBUILD_CONFIG=mysql_release - -DWITH_WSREP=OFF - -DWITH_SSL=system - -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX - -DCMAKE_EXPORT_COMPILE_COMMANDS=1 - " - - - if [[ $SKIP_UNIT_TESTS = true ]] ; then - warn "Unittests are not build" - - else - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UNITTESTS=YES" - message "Buiding with unittests" - fi - - if [[ $DRAW_DEPS = true ]] ; then - warn "Generating dependendies graph to mariadb.dot" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} --graphviz=mariadb.dot" - fi - - if [[ $USE_NINJA = true ]] ; then - warn "Using Ninja instead of Makefiles" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja" - fi - - if [[ $ASAN = true ]] ; then - warn "Building with Address Sanitizer " - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" - fi - - if [[ $TSAN = true ]] ; then - warn "Building with Thread Sanitizer" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" - fi - - if [[ $UBSAN = true ]] ; then - warn "Building with UB Sanitizer" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" - fi - - if [[ $WITHOUT_COREDUMPS = true ]] ; then - warn "Cores are not dumped" - else - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON" - warn Building with CoreDumps: /proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p - echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern - fi - - if [[ $MAKEFILE_VERBOSE = true ]] ; then - warn "Verbosing Makefile Commands" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" - fi - - if [[ $RUN_BENCHMARKS = true ]] ; then - if [[ $MCS_BUILD_TYPE = 'Debug' ]] ; then - error "Benchmarks will not be build in run in Debug build Mode" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" - $RUN_BENCHMARKS = false - elif [[ $OS != 'Ubuntu' && $OS != 'Debian' ]] ; then - error "Benchmarks are now avaliable only at Ubuntu or Debian" - MAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" - $RUN_BENCHMARKS = false - else - message "Compile with microbenchmarks" - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES" - fi - else - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" - message "Buiding without microbenchmarks" - fi - cd $MDB_SOURCE_PATH - if [[ $SKIP_SUBMODULES = true ]] ; then - warn "Skipping initialization of columnstore submodules" - else - message "Initialization of columnstore submodules" - cd storage/columnstore/columnstore - git submodule update --init - cd - > /dev/null - fi - if [[ $FORCE_CMAKE_CONFIG = true ]] ; then warn "Erasing cmake cache" rm -f "$MDB_SOURCE_PATH/CMakeCache.txt" rm -rf "$MDB_SOURCE_PATH/CMakeFiles" fi - if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic" - elif [ $OS = 'CentOS' ]; then - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" - elif [ $OS = 'Rocky' ]; then - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7" - elif [ $OS = 'openSUSE' ]; then - MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15" - fi - - if [[ $PRINT_CMAKE_FLAGS = true ]] ; then - message "Building with flags" - newline_array ${MDB_CMAKE_FLAGS[@]} - fi - message "Configuring cmake silently" - ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS -S$MDB_SOURCE_PATH -B$MARIA_BUILD_PATH | spinner + ${CMAKE_BIN_NAME} $MDB_CMAKE_FLAGS -S$MDB_SOURCE_PATH -B$MARIA_BUILD_PATH | spinner message_split ${CMAKE_BIN_NAME} --build $MARIA_BUILD_PATH -j $CPUS | onelinearizator && \ @@ -500,7 +603,7 @@ install() cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/ cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then + if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then make_dir /usr/share/mysql make_dir /etc/mysql/ cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh @@ -574,32 +677,35 @@ generate_svgs() fi } +if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]] ; then + install_deps +fi + if [[ $DO_NOT_FREEZE_REVISION = false ]] ; then disable_git_restore_frozen_revision fi -select_branch +construct_cmake_flags +init_submodules -if [[ $INSTALL_DEPS = true ]] ; then - install_deps -fi - -if [[ $RESTART_SERVICES = true ]] ; then +if [[ $BUILD_PACKAGES = false ]] ; then + select_branch stop_service -fi -if [[ $NO_CLEAN = false ]] ; then - clean_old_installation -fi - -build -run_unit_tests -run_microbenchmarks_tests -install + if [[ $NO_CLEAN = false ]] ; then + clean_old_installation + fi + build_binary + run_unit_tests + run_microbenchmarks_tests + install if [[ $RESTART_SERVICES = true ]] ; then start_service smoke generate_svgs fi +else + build_package +fi message_splitted "FINISHED" diff --git a/build/utils.sh b/build/utils.sh index d367347d7..184431b9a 100644 --- a/build/utils.sh +++ b/build/utils.sh @@ -1,53 +1,82 @@ -color_normal=$(tput sgr0) -color_bold=$(tput bold) -color_red="$color_bold$(tput setaf 1)" -color_green="$color_bold$(tput setaf 2)" -color_fawn=$(tput setaf 3); color_beige="$color_fawn" -color_yellow="$color_bold$color_fawn" -color_darkblue=$(tput setaf 4) -color_blue="$color_bold$color_darkblue" -color_purple=$(tput setaf 5); color_magenta="$color_purple" -color_pink="$color_bold$color_purple" -color_darkcyan=$(tput setaf 6) -color_cyan="$color_bold$color_darkcyan" -color_gray=$(tput setaf 7) -color_darkgray="$color_bold"$(tput setaf 0) -color_white="$color_bold$color_gray" +if [[ -n "$TERM" && "$TERM" != "dumb" && $(command -v tput) ]]; then + TPUT_AVAILABLE=true +else + TPUT_AVAILABLE=false +fi + +if [[ $TPUT_AVAILABLE == true ]]; then + color_normal=$(tput sgr0) + color_bold=$(tput bold) + color_red="$color_bold$(tput setaf 1)" + color_green="$color_bold$(tput setaf 2)" + color_fawn=$(tput setaf 3); color_beige="$color_fawn" + color_yellow="$color_bold$color_fawn" + color_darkblue=$(tput setaf 4) + + color_blue="$color_bold$color_darkblue" + color_purple=$(tput setaf 5); color_magenta="$color_purple" + color_pink="$color_bold$color_purple" + color_darkcyan=$(tput setaf 6) + color_cyan="$color_bold$color_darkcyan" + color_gray=$(tput setaf 7) + color_darkgray="$color_bold"$(tput setaf 0) + color_white="$color_bold$color_gray" -if [[ $(tput colors) == '256' ]]; then - color_red=$(tput setaf 196) - color_yellow=$(tput setaf 228) - color_cyan=$(tput setaf 87) - color_green=$(tput setaf 156) - color_darkgray=$(tput setaf 59) + if [[ $(tput colors) == '256' ]]; then + color_red=$(tput setaf 196) + color_yellow=$(tput setaf 228) + color_cyan=$(tput setaf 87) + color_green=$(tput setaf 156) + color_darkgray=$(tput setaf 59) + fi fi message() { - echo $color_cyan ・ $@$color_normal + if [[ $TPUT_AVAILABLE == true ]]; then + echo $color_cyan ・ $@$color_normal + else + echo "$@" + fi } warn() { - echo $color_yellow ・ $@$color_normal + if [[ $TPUT_AVAILABLE == true ]]; then + echo $color_yellow ・ $@$color_normal + else + echo "$@" + fi } error() { - echo $color_red ・ $@$color_normal + if [[ $TPUT_AVAILABLE == true ]]; then + echo $color_red ・ $@$color_normal + else + echo "$@" + fi } message_split() { - echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal + if [[ $TPUT_AVAILABLE == true ]]; then + echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal + else + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + fi } message_splitted() { message_split - echo $color_green ・ $@$color_normal + if [[ $TPUT_AVAILABLE == true ]]; then + echo $color_green ・ $@$color_normal + else + echo "$@" + fi message_split } @@ -57,11 +86,15 @@ colorify_array() PROMT="" for a in "$@" do - i=$((((i+1) % (123-106)) + 106)) - if [[ $(tput colors) == '256' ]]; then + if [[ $TPUT_AVAILABLE == true ]]; then + i=$((((i+1) % (123-106)) + 106)) + if [[ $(tput colors) == '256' ]]; then PROMT="$PROMT $(tput setaf $i)$a$color_normal" - else + else PROMT="$PROMT $a" + fi + else + PROMT="$PROMT $a" fi done echo $PROMT @@ -91,7 +124,11 @@ function spinner line_num=$((line_num+1)) if [[ $((line_num % freq)) = 0 ]]; then point_num=$(((point_num + 1) % len )) - echo -ne "\r${colored_points[point_num]}" + if [[ $TPUT_AVAILABLE == true ]]; then + echo -ne "\r${colored_points[point_num]}" + else + echo -ne "\r${points[point_num]}" + fi fi done; echo @@ -129,7 +166,7 @@ detect_distro() OS=$(uname -s) OS_VERSION=$(uname -r) fi - OS=$(echo $OS | cut -f 1 -d " ") + OS=$(echo $OS | cut -f 1 -d " " | tr '[:upper:]' '[:lower:]')":"$OS_VERSION message "Detected $color_yellow$OS $OS_VERSION$color_normal" }