1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00

MCOL-5911 migrate build packages step from drone to bootstrap_mcs.sh

This commit is contained in:
aleksei.bukhalov
2025-03-25 17:02:10 +01:00
committed by Aleksei Bukhalov
parent 667d5a05a0
commit ea6d2d873e
3 changed files with 400 additions and 301 deletions

View File

@ -23,42 +23,6 @@ local platforms_mtr = platforms.develop;
local builddir = 'verylongdirnameforverystrangecpackbehavior';
local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CONFIG=mysql_release ' +
'-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ' +
'-DPLUGIN_COLUMNSTORE=YES -DWITH_UNITTESTS=YES ' +
'-DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO ' +
'-DPLUGIN_CONNECT=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' +
'-DPLUGIN_GSSAPI=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' +
'-DWITH_EMBEDDED_SERVER=NO -DWITH_WSREP=NO -DWITH_COREDUMPS=ON';
local clang_version = '16';
local gcc_version = '11';
local clang_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-' + clang_version + ' 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-' + clang_version + ' && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 ';
local rpm_build_deps = 'install -y lz4 systemd-devel git make libaio-devel openssl-devel boost-devel bison ' +
'snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool ' +
'policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel ' +
'expect createrepo python3 ';
local rockylinux8_build_deps = "dnf install -y 'dnf-command(config-manager)' " +
'&& dnf config-manager --set-enabled powertools ' +
'&& dnf install -y gcc-toolset-' + gcc_version + ' libarchive cmake lz4-devel ' +
'&& . /opt/rh/gcc-toolset-' + gcc_version + '/enable ';
local rockylinux9_build_deps = "dnf install -y 'dnf-command(config-manager)' " +
'&& dnf config-manager --set-enabled crb ' +
'&& dnf install -y pcre2-devel lz4-devel gcc gcc-c++';
local debian11_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' && ' + clang_update_alternatives;
local ubuntu20_04_deps = 'apt update && apt install -y gnupg wget && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-' + clang_version + ' main" >> /etc/apt/sources.list && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt install -y clang-' + clang_version + ' && ' + clang_update_alternatives;
local deb_build_deps = 'apt update --yes && apt install --yes --no-install-recommends build-essential devscripts git ccache equivs eatmydata libssl-dev && mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i ';
local turnon_clang = 'export CC=/usr/bin/clang; export CXX=/usr/bin/clang++ ';
local bootstrap_deps = 'apt-get -y update && apt-get -y install build-essential automake libboost-all-dev bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl ';
local mtr_suite_list = 'basic,bugfixes';
local mtr_full_set = 'basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly';
@ -94,20 +58,6 @@ local upgrade_test_lists = {
},
};
local platformMap(platform, arch) =
local platform_map = {
'rockylinux:8': rockylinux8_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux8 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package',
'rockylinux:9': rockylinux9_build_deps + ' && dnf ' + rpm_build_deps + ' && cmake ' + cmakeflags + ' -DRPM=rockylinux9 && sleep $${BUILD_DELAY_SECONDS:-1s} && make -j$(nproc) package',
'debian:11': bootstrap_deps + ' && ' + deb_build_deps + ' && ' + debian11_deps + ' && ' + turnon_clang + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=bullseye' debian/autobake-deb.sh",
'debian:12': bootstrap_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=bookworm' debian/autobake-deb.sh",
'ubuntu:20.04': bootstrap_deps + ' && ' + deb_build_deps + ' && ' + ubuntu20_04_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=focal' debian/autobake-deb.sh",
'ubuntu:22.04': bootstrap_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=jammy' debian/autobake-deb.sh",
'ubuntu:24.04': bootstrap_deps + ' && ' + deb_build_deps + " && sleep $${BUILD_DELAY_SECONDS:-1s} && CMAKEFLAGS='" + cmakeflags + " -DDEB=jammy' debian/autobake-deb.sh",
};
local result = std.strReplace(std.strReplace(platform, ':', ''), '/', '-');
'export CLICOLOR_FORCE=1; ' + platform_map[platform] + ' | storage/columnstore/columnstore/build/ansi2txt.sh ' + result + '/build.log';
local testRun(platform) =
local platform_map = {
'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
@ -121,16 +71,30 @@ local testRun(platform) =
};
platform_map[platform];
local gcc_version = '11';
local rockylinux8_deps = "dnf install -y 'dnf-command(config-manager)' " +
'&& dnf config-manager --set-enabled powertools ' +
'&& dnf install -y gcc-toolset-' + gcc_version + ' libarchive cmake ' +
'&& . /opt/rh/gcc-toolset-' + gcc_version + '/enable ';
local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " +
'&& dnf config-manager --set-enabled crb ' +
'&& dnf install -y gcc gcc-c++';
local rockylinux_common_deps = ' && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel';
local deb_deps = 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev';
local testPreparation(platform) =
local platform_map = {
'rockylinux:8': rockylinux8_build_deps + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel',
'rockylinux:9': rockylinux9_build_deps + ' && dnf install -y git lz4 cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel',
'debian:11': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev',
'debian:12': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev',
'ubuntu:20.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev',
'ubuntu:22.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev',
'ubuntu:24.04': 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev',
'rockylinux:8': rockylinux8_deps + rockylinux_common_deps,
'rockylinux:9': rockylinux9_deps + rockylinux_common_deps,
'debian:11': deb_deps,
'debian:12': deb_deps,
'ubuntu:20.04': deb_deps,
'ubuntu:22.04': deb_deps,
'ubuntu:24.04': deb_deps,
};
platform_map[platform];
@ -161,9 +125,10 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
local server_remote = if (std.endsWith(server, 'enterprise')) then 'https://github.com/mariadb-corporation/MariaDBEnterprise' else 'https://github.com/MariaDB/server',
local sccache_arch = if (arch == 'amd64') then 'x86_64' else 'aarch64',
local get_sccache = 'curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.0/sccache-v0.3.0-' + sccache_arch + '-unknown-linux-musl.tar.gz ' +
local get_sccache = 'echo getting sccache... && (apt update -y && apt install -y curl || yum install -y curl || true) ' +
'&& curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-' + sccache_arch + '-unknown-linux-musl.tar.gz ' +
'&& tar xzf sccache.tar.gz ' +
'&& install sccache*/sccache /usr/local/bin/',
'&& install sccache*/sccache /usr/local/bin/ && echo sccache installed',
local pipeline = self,
@ -238,11 +203,11 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
local dockerImage(stepname) = stepname + "$${DRONE_BUILD_NUMBER}",
local installEngine(dockerImage, pkg_format) =
if (pkg_format == 'deb') then execInnerDocker('bash -c "apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage)
if (pkg_format == 'deb') then execInnerDocker('bash -c "apt update && apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage)
else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"', dockerImage),
local installCmapi(dockerImage, pkg_format) =
if (pkg_format == 'deb') then execInnerDocker('bash -c "apt install -y mariadb-columnstore-cmapi"', dockerImage)
if (pkg_format == 'deb') then execInnerDocker('bash -c "apt update && apt install -y mariadb-columnstore-cmapi"', dockerImage)
else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-cmapi"', dockerImage),
local prepareTestStage(dockerImage, pkg_format, result, do_setup) = [
@ -770,39 +735,30 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
from_secret: 'aws_secret_access_key',
},
SCCACHE_BUCKET: 'cs-sccache',
SCCACHE_REGION: 'us-east-1',
SCCACHE_S3_USE_SSL: 'true',
SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + '${DRONE_PULL_REQUEST}',
//SCCACHE_ERROR_LOG: '/tmp/sccache_log.txt',
//SCCACHE_LOG: 'debug',
},
commands: [
'cd /mdb/' + builddir,
'ls -la ../',
'mkdir ' + result,
"sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh",
if (std.startsWith(server, '10.6')) then "sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control",
// Remove Debian build flags that could prevent ColumnStore from building
"sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules",
// Disable dh_missing strict check for missing files
'sed -i s/--fail-missing/--list-missing/ debian/rules',
// Tweak debian packaging stuff
'for i in mariadb-plugin libmariadbd; do sed -i "/Package: $i.*/,/^$/d" debian/control; done',
"sed -i 's/Depends: galera.*/Depends:/' debian/control",
'for i in galera wsrep ha_sphinx embedded; do sed -i /$i/d debian/*.install; done',
// Install build dependencies for deb
if (pkg_format == 'deb') then "apt-cache madison liburing-dev | grep liburing-dev || sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DWITH_URING=yes/d' -i debian/rules && apt-cache madison libpmem-dev | grep 'libpmem-dev' || sed '/libpmem-dev/d' -i debian/control && sed '/-DWITH_PMEM/d' -i debian/rules && sed '/libfmt-dev/d' -i debian/control",
// Change plugin_maturity level
// "sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt",
if (pkg_format == 'deb') then 'apt update -y && apt install -y curl' else if (platform == 'rockylinux:9') then 'yum install -y curl-minimal' else 'yum install -y curl',
'export CLICOLOR_FORCE=1',
get_sccache,
testPreparation(platform),
// disable LTO for 22.04 for now
if (platform == 'ubuntu:22.04' || platform == 'ubuntu:24.04') then 'apt install -y lto-disabled-list && for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do echo "$i any" >> /usr/share/lto-disabled-list/lto-disabled-list; done && grep mariadb /usr/share/lto-disabled-list/lto-disabled-list',
platformMap(platform, arch),
'mkdir /mdb/' + builddir + '/' + result,
'bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/bootstrap_mcs.sh ' +
'--build-type RelWithDebInfo ' +
'--distro ' + platform + ' ' +
'--build-packages --sccache ' +
'--server-version ' + server + '|' +
'/mdb/' + builddir + '/storage/columnstore/columnstore/build/ansi2txt.sh ' +
'/mdb/' + builddir + '/' + result + '/build.log',
'sccache --show-stats',
// move engine and cmapi packages to one dir to make a repo
'mv -v -t ./%s/ %s/*.%s /drone/src/cmapi/%s/*.%s ' % [result, if (pkg_format == 'rpm') then '.' else '..', pkg_format, result, pkg_format],
if (pkg_format == 'rpm') then 'createrepo ./' + result else 'dpkg-scanpackages %s | gzip > ./%s/Packages.gz' % [result, result],
// move engine and cmapi packages to one dir and make a repo
if (pkg_format == 'rpm') then "mv -v -t ./" + result + "/ /mdb/" + builddir + "/*.rpm /drone/src/cmapi/" + result + "/*.rpm && createrepo ./" + result
else "mv -v -t ./" + result + "/ /mdb/*.deb /drone/src/cmapi/" + result + "/*.deb && dpkg-scanpackages " + result + " | gzip > ./" + result + "/Packages.gz",
// list storage manager binary
'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager',
],

View File

@ -5,26 +5,37 @@
# - the script is to be run under root.
set -o pipefail
INSTALL_PREFIX="/usr/"
DATA_DIR="/var/lib/mysql/data"
CMAKE_BIN_NAME=cmake
CTEST_BIN_NAME=ctest
RPM_CONFIG_DIR="/etc/my.cnf.d"
DEB_CONFIG_DIR="/etc/mysql/mariadb.conf.d"
CONFIG_DIR=$RPM_CONFIG_DIR
SCRIPT_LOCATION=$(dirname "$0")
MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../..)
BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9")
GCC_VERSION="11"
BUILD_DELAY_SECONDS="${BUILD_DELAY_SECONDS:-1s}"
BUILD_DIR="verylongdirnameforverystrangecpackbehavior"
MDB_CMAKE_FLAGS=""
source $SCRIPT_LOCATION/utils.sh
if [ "$EUID" -ne 0 ]
then error "Please run script as root to install MariaDb to system paths"
exit 1
fi
echo "Arguments received: $@"
message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky")
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo))
cd - > /dev/null
optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
optparse.define short=d long=distro desc="Choose your OS: ${DISTRO_OPTIONS[*]}" variable=OS
@ -32,7 +43,7 @@ optparse.define short=D long=install-deps desc="Install dependences" variable=IN
optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true
optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH
optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
@ -51,7 +62,9 @@ optparse.define short=f long=do-not-freeze-revision desc="Disable revision freez
optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$MDB_SOURCE_PATH/../MariaDBBuild
optparse.define short=o long=recompile-only variable=RECOMPILE_ONLY default=false value=true
optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false
optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE default=false value=true
optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true
optparse.define short=R long=server-version desc="MariaDB server version" variable=MARIADB_SERVER_VERSION
source $( optparse.build )
@ -60,24 +73,19 @@ if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
MCS_BUILD_TYPE=$selectedChoice
fi
if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "CentOS" ]]; then
if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then
echo "OS is empty, trying to detect..."
detect_distro
fi
INSTALL_PREFIX="/usr/"
DATA_DIR="/var/lib/mysql/data"
CMAKE_BIN_NAME=cmake
CTEST_BIN_NAME=ctest
RPM_CONFIG_DIR="/etc/my.cnf.d"
DEB_CONFIG_DIR="/etc/mysql/mariadb.conf.d"
CONFIG_DIR=$RPM_CONFIG_DIR
if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
CONFIG_DIR=$DEB_CONFIG_DIR
pkg_format="deb"
if [[ "$OS" == *"rocky"* ]]; then
pkg_format="rpm"
fi
export CLICOLOR_FORCE=1
if [[ $pkg_format = "deb" ]]; then
CONFIG_DIR=$DEB_CONFIG_DIR
fi
disable_git_restore_frozen_revision()
{
@ -90,6 +98,7 @@ select_branch()
{
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo))
if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then
if [[ $BRANCH = "" ]]; then
@ -111,43 +120,37 @@ select_branch()
install_deps()
{
message_split
message "Installing deps"
if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
apt-get -y update
apt-get -y install build-essential automake libboost-all-dev bison cmake \
libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \
libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \
libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev \
liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev graphviz
elif [[ $OS = 'CentOS' || $OS = 'Rocky' || $OS = 'Fedora' ]]; then
if [[ "$OS_VERSION" == "7" ]]; then
yum -y install cmake3 epel-release centos-release-scl
CMAKE_BIN_NAME=cmake3
CTEST_BIN_NAME=ctest3
else
yum -y install cmake
fi
if [ $OS = 'Rocky' ]; then
if [[ "$OS_VERSION" == "9" ]]; then
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --set-enabled crb
fi
yum -y groupinstall "Development Tools" && yum config-manager --set-enabled powertools
yum install -y checkpolicy
fi
if [[ $OS != 'Fedora' ]]; then
yum -y install epel-release
fi
RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \
snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \
rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \
cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
graphviz devscripts ccache equivs eatmydata curl"
if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \
${RPM_BUILD_DEPS}"
elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]; then
command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \
dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}"
elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then
command="${DEB_BUILD_DEPS}"
yum install -y bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \
vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts \
jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel lzo-devel xz-devel lz4-devel bzip2-devel \
pcre2-devel flex graphviz libaio-devel openssl-devel flex
else
error "Unsupported OS $OS"
echo "Unsupported OS: $OS"
exit 17
fi
message "Installing dependencies for $OS"
eval "$command"
}
stop_service()
@ -217,31 +220,79 @@ clean_old_installation()
rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf
}
build()
{
MARIA_BUILD_PATH=$(realpath $MARIA_BUILD_PATH)
message_split
message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_cyan mode"
message "Compiled artifacts will be written to $color_yellow$MARIA_BUILD_PATH$color_cyan"
mkdir -p $MARIA_BUILD_PATH
modify_packaging() {
echo "Modifying_packaging..."
cd $MDB_SOURCE_PATH
local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes
-DPLUGIN_COLUMNSTORE=YES
-DPLUGIN_MROONGA=NO
-DPLUGIN_ROCKSDB=NO
-DPLUGIN_TOKUDB=NO
-DPLUGIN_CONNECT=NO
-DPLUGIN_SPIDER=NO
-DPLUGIN_OQGRAPH=NO
-DPLUGIN_SPHINX=NO
-DWITH_EMBEDDED_SERVER=OFF
-DBUILD_CONFIG=mysql_release
-DWITH_WSREP=OFF
-DWITH_SSL=system
-DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX
-DCMAKE_EXPORT_COMPILE_COMMANDS=1
"
if [[ $pkg_format == "deb" ]]; then
sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh
fi
if [[ "$MARIADB_SERVER_VERSION" == 10.6* ]]; then
sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control
fi
#disable LTO for 22.04 for now
if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
apt install -y lto-disabled-list &&
for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do
echo "$i any" >> /usr/share/lto-disabled-list/lto-disabled-list
done &&
grep mariadb /usr/share/lto-disabled-list/lto-disabled-list
fi
if [[ $pkg_format == "deb" ]]; then
apt-cache madison liburing-dev | grep liburing-dev || {
sed 's/liburing-dev/libaio-dev/g' -i debian/control &&
sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules &&
sed '/-DWITH_URING=yes/d' -i debian/rules
}
apt-cache madison libpmem-dev | grep 'libpmem-dev' || {
sed '/libpmem-dev/d' -i debian/control &&
sed '/-DWITH_PMEM/d' -i debian/rules
}
sed '/libfmt-dev/d' -i debian/control
# Remove Debian build flags that could prevent ColumnStore from building
sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules
# Disable dh_missing strict check for missing files
sed -i 's/--fail-missing/--list-missing/' debian/rules
# Tweak debian packaging stuff
for i in mariadb-plugin libmariadbd;
do
sed -i "/Package: $i.*/,/^$/d" debian/control
done
sed -i 's/Depends: galera.*/Depends:/' debian/control
for i in galera wsrep ha_sphinx embedded;
do
sed -i "/$i/d" debian/*.install
done
fi
}
construct_cmake_flags(){
MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes \
-DPLUGIN_COLUMNSTORE=YES \
-DPLUGIN_MROONGA=NO \
-DPLUGIN_ROCKSDB=NO \
-DPLUGIN_TOKUDB=NO \
-DPLUGIN_CONNECT=NO \
-DPLUGIN_SPIDER=NO \
-DPLUGIN_OQGRAPH=NO \
-DPLUGIN_SPHINX=NO \
-DWITH_EMBEDDED_SERVER=NO \
-DBUILD_CONFIG=mysql_release \
-DWITH_WSREP=NO \
-DWITH_SSL=system \
-DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX \
-DCMAKE_EXPORT_COMPILE_COMMANDS=1 \
-DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE \
-DPLUGIN_GSSAPI=NO"
if [[ $SKIP_UNIT_TESTS = true ]] ; then
warn "Unittests are not build"
@ -279,22 +330,33 @@ build()
if [[ $WITHOUT_COREDUMPS = true ]] ; then
warn "Cores are not dumped"
else
warn "Building with CoreDumps"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON"
warn Building with CoreDumps: /proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p
if [ -f /.dockerenv ] ; then
warn "Build is executed in Docker, core dumps saving path /proc/sys/kernel/core_pattern will not be configured!"
else
warn "/proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p"
echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern
fi
fi
if [[ $MAKEFILE_VERBOSE = true ]] ; then
warn "Verbosing Makefile Commands"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
fi
if [[ $SCCACHE = true ]] ; then
warn "Use sccache"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
fi
if [[ $RUN_BENCHMARKS = true ]] ; then
if [[ $MCS_BUILD_TYPE = 'Debug' ]] ; then
error "Benchmarks will not be build in run in Debug build Mode"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO"
$RUN_BENCHMARKS = false
elif [[ $OS != 'Ubuntu' && $OS != 'Debian' ]] ; then
elif [[ $OS != *"ubuntu"* && $OS != *"debian"* ]] ; then
error "Benchmarks are now avaliable only at Ubuntu or Debian"
MAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO"
$RUN_BENCHMARKS = false
@ -307,16 +369,72 @@ build()
message "Buiding without microbenchmarks"
fi
cd $MDB_SOURCE_PATH
if [[ "$OS" == *"rocky"* ]]; then
OS_VERSION=${OS//[^0-9]/}
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=rockylinux${OS_VERSION}"
elif [[ "$OS" == "debian:11" ]]; then
CODENAME="bullseye"
elif [[ "$OS" == "debian:12" ]]; then
CODENAME="bookworm"
elif [[ "$OS" == "ubuntu:20.04" ]]; then
CODENAME="focal"
elif [[ "$OS" == "ubuntu:22.04" ]]; then
CODENAME="jammy"
elif [[ "$OS" == "ubuntu:24.04" ]]; then
CODENAME="noble"
else
echo "Unsupported OS: $OS"
exit 17
fi
if [[ -n "$CODENAME" ]]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=${CODENAME}"
fi
if [[ $PRINT_CMAKE_FLAGS = true ]] ; then
message "Building with flags"
newline_array ${MDB_CMAKE_FLAGS[@]}
fi
}
init_submodules(){
if [[ $SKIP_SUBMODULES = true ]] ; then
warn "Skipping initialization of columnstore submodules"
else
message "Initialization of columnstore submodules"
cd storage/columnstore/columnstore
git submodule update --init
cd $MDB_SOURCE_PATH
git submodule update --init --recursive
cd - > /dev/null
fi
}
build_package() {
modify_packaging
RESULT_DIR=$(echo "$OS" | sed 's/://g' | sed 's/\//-/g')
mkdir $RESULT_DIR
if [[ $pkg_format == "rpm" ]]; then
command="cmake ${MDB_CMAKE_FLAGS} && sleep ${BUILD_DELAY_SECONDS} && make -j\$(nproc) package"
else
command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \
sleep ${BUILD_DELAY_SECONDS} && CMAKEFLAGS='${MDB_CMAKE_FLAGS}' debian/autobake-deb.sh"
fi
echo "Building a package for $OS"
echo "Build command: $command"
eval "$command"
}
build_binary()
{
MARIA_BUILD_PATH=$(realpath $MARIA_BUILD_PATH)
message_split
message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_cyan mode"
message "Compiled artifacts will be written to $color_yellow$MARIA_BUILD_PATH$color_cyan"
mkdir -p $MARIA_BUILD_PATH
cd $MDB_SOURCE_PATH
if [[ $FORCE_CMAKE_CONFIG = true ]] ; then
warn "Erasing cmake cache"
@ -324,23 +442,8 @@ build()
rm -rf "$MDB_SOURCE_PATH/CMakeFiles"
fi
if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic"
elif [ $OS = 'CentOS' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7"
elif [ $OS = 'Rocky' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7"
elif [ $OS = 'openSUSE' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15"
fi
if [[ $PRINT_CMAKE_FLAGS = true ]] ; then
message "Building with flags"
newline_array ${MDB_CMAKE_FLAGS[@]}
fi
message "Configuring cmake silently"
${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS -S$MDB_SOURCE_PATH -B$MARIA_BUILD_PATH | spinner
${CMAKE_BIN_NAME} $MDB_CMAKE_FLAGS -S$MDB_SOURCE_PATH -B$MARIA_BUILD_PATH | spinner
message_split
${CMAKE_BIN_NAME} --build $MARIA_BUILD_PATH -j $CPUS | onelinearizator && \
@ -500,7 +603,7 @@ install()
cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/
cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then
if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then
make_dir /usr/share/mysql
make_dir /etc/mysql/
cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
@ -574,25 +677,25 @@ generate_svgs()
fi
}
if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]] ; then
install_deps
fi
if [[ $DO_NOT_FREEZE_REVISION = false ]] ; then
disable_git_restore_frozen_revision
fi
construct_cmake_flags
init_submodules
if [[ $BUILD_PACKAGES = false ]] ; then
select_branch
if [[ $INSTALL_DEPS = true ]] ; then
install_deps
fi
if [[ $RESTART_SERVICES = true ]] ; then
stop_service
fi
if [[ $NO_CLEAN = false ]] ; then
clean_old_installation
fi
build
build_binary
run_unit_tests
run_microbenchmarks_tests
install
@ -601,5 +704,8 @@ if [[ $RESTART_SERVICES = true ]] ; then
smoke
generate_svgs
fi
else
build_package
fi
message_splitted "FINISHED"

View File

@ -1,3 +1,11 @@
if [[ -n "$TERM" && "$TERM" != "dumb" && $(command -v tput) ]]; then
TPUT_AVAILABLE=true
else
TPUT_AVAILABLE=false
fi
if [[ $TPUT_AVAILABLE == true ]]; then
color_normal=$(tput sgr0)
color_bold=$(tput bold)
color_red="$color_bold$(tput setaf 1)"
@ -23,31 +31,52 @@ if [[ $(tput colors) == '256' ]]; then
color_green=$(tput setaf 156)
color_darkgray=$(tput setaf 59)
fi
fi
message()
{
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_cyan$@$color_normal
else
echo "$@"
fi
}
warn()
{
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_yellow$@$color_normal
else
echo "$@"
fi
}
error()
{
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_red$@$color_normal
else
echo "$@"
fi
}
message_split()
{
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal
else
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
fi
}
message_splitted()
{
message_split
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_green$@$color_normal
else
echo "$@"
fi
message_split
}
@ -57,12 +86,16 @@ colorify_array()
PROMT=""
for a in "$@"
do
if [[ $TPUT_AVAILABLE == true ]]; then
i=$((((i+1) % (123-106)) + 106))
if [[ $(tput colors) == '256' ]]; then
PROMT="$PROMT $(tput setaf $i)$a$color_normal"
else
PROMT="$PROMT $a"
fi
else
PROMT="$PROMT $a"
fi
done
echo $PROMT
}
@ -91,7 +124,11 @@ function spinner
line_num=$((line_num+1))
if [[ $((line_num % freq)) = 0 ]]; then
point_num=$(((point_num + 1) % len ))
if [[ $TPUT_AVAILABLE == true ]]; then
echo -ne "\r${colored_points[point_num]}"
else
echo -ne "\r${points[point_num]}"
fi
fi
done;
echo
@ -129,7 +166,7 @@ detect_distro()
OS=$(uname -s)
OS_VERSION=$(uname -r)
fi
OS=$(echo $OS | cut -f 1 -d " ")
OS=$(echo $OS | cut -f 1 -d " " | tr '[:upper:]' '[:lower:]')":"$OS_VERSION
message "Detected $color_yellow$OS $OS_VERSION$color_normal"
}