1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00

errorcode check + shell formatting applied

This commit is contained in:
Leonid Fedorov
2025-04-14 21:21:30 +00:00
committed by Leonid Fedorov
parent 7c27d803ba
commit 4b17d1aa02

View File

@ -21,22 +21,19 @@ BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9") DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9")
GCC_VERSION="11" GCC_VERSION="11"
BUILD_DELAY_SECONDS="${BUILD_DELAY_SECONDS:-1s}"
BUILD_DIR="verylongdirnameforverystrangecpackbehavior" BUILD_DIR="verylongdirnameforverystrangecpackbehavior"
MDB_CMAKE_FLAGS="" MDB_CMAKE_FLAGS=""
source $SCRIPT_LOCATION/utils.sh source $SCRIPT_LOCATION/utils.sh
if [ "$EUID" -ne 0 ]; then
if [ "$EUID" -ne 0 ] error "Please run script as root to install MariaDb to system paths"
then error "Please run script as root to install MariaDb to system paths"
exit 1 exit 1
fi fi
echo "Arguments received: $@" echo "Arguments received: $@"
message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
optparse.define short=d long=distro desc="Choose your OS: ${DISTRO_OPTIONS[*]}" variable=OS optparse.define short=d long=distro desc="Choose your OS: ${DISTRO_OPTIONS[*]}" variable=OS
optparse.define short=D long=install-deps desc="Install dependences" variable=INSTALL_DEPS default=false value=true optparse.define short=D long=install-deps desc="Install dependences" variable=INSTALL_DEPS default=false value=true
@ -66,7 +63,7 @@ optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE
optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true
optparse.define short=R long=server-version desc="MariaDB server version" variable=MARIADB_SERVER_VERSION optparse.define short=R long=server-version desc="MariaDB server version" variable=MARIADB_SERVER_VERSION
source $( optparse.build ) source $(optparse.build)
if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS
@ -80,25 +77,23 @@ fi
pkg_format="deb" pkg_format="deb"
if [[ "$OS" == *"rocky"* ]]; then if [[ "$OS" == *"rocky"* ]]; then
pkg_format="rpm" pkg_format="rpm"
fi fi
if [[ $pkg_format = "deb" ]]; then if [[ $pkg_format = "deb" ]]; then
CONFIG_DIR=$DEB_CONFIG_DIR CONFIG_DIR=$DEB_CONFIG_DIR
fi fi
disable_git_restore_frozen_revision() disable_git_restore_frozen_revision() {
{
cd $MDB_SOURCE_PATH cd $MDB_SOURCE_PATH
git config submodule.storage/columnstore/columnstore.update none git config submodule.storage/columnstore/columnstore.update none
cd - > /dev/null cd - >/dev/null
} }
select_branch() select_branch() {
{
cd $SCRIPT_LOCATION cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) BRANCHES=($(git branch --list --no-color | grep "[^* ]+" -Eo))
if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then
if [[ $BRANCH = "" ]]; then if [[ $BRANCH = "" ]]; then
@ -113,48 +108,46 @@ select_branch()
message "Turning off Columnstore submodule auto update via gitconfig" message "Turning off Columnstore submodule auto update via gitconfig"
fi fi
cd - > /dev/null cd - >/dev/null
message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch" message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch"
} }
install_deps() install_deps() {
{
message_split message_split
RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \ RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \
snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \ snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \
rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \ rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \
cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel" cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \ DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \ libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \ libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
graphviz devscripts ccache equivs eatmydata curl" graphviz devscripts ccache equivs eatmydata curl"
if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \ command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \ dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \
${RPM_BUILD_DEPS}" ${RPM_BUILD_DEPS}"
elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]; then elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]; then
command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \ command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \
dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}" dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}"
elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then
command="${DEB_BUILD_DEPS}" command="${DEB_BUILD_DEPS}"
else else
echo "Unsupported OS: $OS" echo "Unsupported OS: $OS"
exit 17 exit 17
fi fi
message "Installing dependencies for $OS" message "Installing dependencies for $OS"
eval "$command" eval "$command"
} }
stop_service() stop_service() {
{
message_split message_split
message "Stopping MariaDB services" message "Stopping MariaDB services"
systemctl stop mariadb systemctl stop mariadb
@ -162,8 +155,7 @@ stop_service()
systemctl stop mcs-storagemanager systemctl stop mcs-storagemanager
} }
check_service() check_service() {
{
if systemctl is-active --quiet $1; then if systemctl is-active --quiet $1; then
message "$1 $color_normal[$color_green OK $color_normal]" message "$1 $color_normal[$color_green OK $color_normal]"
else else
@ -172,8 +164,7 @@ check_service()
fi fi
} }
start_service() start_service() {
{
message_split message_split
message "Starting MariaDB services" message "Starting MariaDB services"
systemctl start mariadb-columnstore systemctl start mariadb-columnstore
@ -189,19 +180,17 @@ start_service()
check_service mcs-writeengineserver check_service mcs-writeengineserver
} }
start_storage_manager_if_needed() start_storage_manager_if_needed() {
{ if [[ $CLOUD_STORAGE_ENABLED = true ]]; then
if [[ $CLOUD_STORAGE_ENABLED = true ]]; then export MCS_USE_S3_STORAGE=1
export MCS_USE_S3_STORAGE=1; message_split
message_split message "Starting Storage Manager service"
message "Starting Storage Manager service" systemctl start mcs-storagemanager
systemctl start mcs-storagemanager check_service mcs-storagemanager
check_service mcs-storagemanager fi
fi
} }
clean_old_installation() clean_old_installation() {
{
message_split message_split
message "Cleaning old installation" message "Cleaning old installation"
rm -rf /var/lib/columnstore/data1/* rm -rf /var/lib/columnstore/data1/*
@ -221,62 +210,60 @@ clean_old_installation()
} }
modify_packaging() { modify_packaging() {
echo "Modifying_packaging..." echo "Modifying_packaging..."
cd $MDB_SOURCE_PATH cd $MDB_SOURCE_PATH
if [[ $pkg_format == "deb" ]]; then if [[ $pkg_format == "deb" ]]; then
sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh
fi fi
if [[ "$MARIADB_SERVER_VERSION" == 10.6* ]]; then if [[ "$MARIADB_SERVER_VERSION" == 10.6* ]]; then
sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control sed -i 's/mariadb-server/mariadb-server-10.6/' storage/columnstore/columnstore/debian/control
fi fi
#disable LTO for 22.04 for now #disable LTO for 22.04 for now
if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
apt install -y lto-disabled-list && apt install -y lto-disabled-list &&
for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do
echo "$i any" >> /usr/share/lto-disabled-list/lto-disabled-list echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list
done && done &&
grep mariadb /usr/share/lto-disabled-list/lto-disabled-list grep mariadb /usr/share/lto-disabled-list/lto-disabled-list
fi fi
if [[ $pkg_format == "deb" ]]; then if [[ $pkg_format == "deb" ]]; then
apt-cache madison liburing-dev | grep liburing-dev || { apt-cache madison liburing-dev | grep liburing-dev || {
sed 's/liburing-dev/libaio-dev/g' -i debian/control && sed 's/liburing-dev/libaio-dev/g' -i debian/control &&
sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules && sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules &&
sed '/-DWITH_URING=yes/d' -i debian/rules sed '/-DWITH_URING=yes/d' -i debian/rules
} }
apt-cache madison libpmem-dev | grep 'libpmem-dev' || { apt-cache madison libpmem-dev | grep 'libpmem-dev' || {
sed '/libpmem-dev/d' -i debian/control && sed '/libpmem-dev/d' -i debian/control &&
sed '/-DWITH_PMEM/d' -i debian/rules sed '/-DWITH_PMEM/d' -i debian/rules
} }
sed '/libfmt-dev/d' -i debian/control sed '/libfmt-dev/d' -i debian/control
# Remove Debian build flags that could prevent ColumnStore from building # Remove Debian build flags that could prevent ColumnStore from building
sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules
# Disable dh_missing strict check for missing files # Disable dh_missing strict check for missing files
sed -i 's/--fail-missing/--list-missing/' debian/rules sed -i 's/--fail-missing/--list-missing/' debian/rules
# Tweak debian packaging stuff # Tweak debian packaging stuff
for i in mariadb-plugin libmariadbd; for i in mariadb-plugin libmariadbd; do
do sed -i "/Package: $i.*/,/^$/d" debian/control
sed -i "/Package: $i.*/,/^$/d" debian/control done
done
sed -i 's/Depends: galera.*/Depends:/' debian/control sed -i 's/Depends: galera.*/Depends:/' debian/control
for i in galera wsrep ha_sphinx embedded; for i in galera wsrep ha_sphinx embedded; do
do sed -i "/$i/d" debian/*.install
sed -i "/$i/d" debian/*.install done
done fi
fi
} }
construct_cmake_flags(){ construct_cmake_flags() {
MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes \ MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes \
-DPLUGIN_COLUMNSTORE=YES \ -DPLUGIN_COLUMNSTORE=YES \
-DPLUGIN_MROONGA=NO \ -DPLUGIN_MROONGA=NO \
-DPLUGIN_ROCKSDB=NO \ -DPLUGIN_ROCKSDB=NO \
@ -294,119 +281,128 @@ construct_cmake_flags(){
-DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE \ -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE \
-DPLUGIN_GSSAPI=NO" -DPLUGIN_GSSAPI=NO"
if [[ $SKIP_UNIT_TESTS = true ]] ; then if [[ $SKIP_UNIT_TESTS = true ]]; then
warn "Unittests are not build" warn "Unittests are not build"
else else
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UNITTESTS=YES" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UNITTESTS=YES"
message "Buiding with unittests" message "Buiding with unittests"
fi fi
if [[ $DRAW_DEPS = true ]] ; then if [[ $DRAW_DEPS = true ]]; then
warn "Generating dependendies graph to mariadb.dot" warn "Generating dependendies graph to mariadb.dot"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} --graphviz=mariadb.dot" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} --graphviz=mariadb.dot"
fi fi
if [[ $USE_NINJA = true ]] ; then if [[ $USE_NINJA = true ]]; then
warn "Using Ninja instead of Makefiles" warn "Using Ninja instead of Makefiles"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja"
fi fi
if [[ $ASAN = true ]] ; then if [[ $ASAN = true ]]; then
warn "Building with Address Sanitizer " warn "Building with Address Sanitizer "
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
fi fi
if [[ $TSAN = true ]] ; then if [[ $TSAN = true ]]; then
warn "Building with Thread Sanitizer" warn "Building with Thread Sanitizer"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
fi fi
if [[ $UBSAN = true ]] ; then if [[ $UBSAN = true ]]; then
warn "Building with UB Sanitizer" warn "Building with UB Sanitizer"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
fi fi
if [[ $WITHOUT_COREDUMPS = true ]] ; then if [[ $WITHOUT_COREDUMPS = true ]]; then
warn "Cores are not dumped" warn "Cores are not dumped"
else else
warn "Building with CoreDumps" warn "Building with CoreDumps"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON"
if [ -f /.dockerenv ] ; then if [ -f /.dockerenv ]; then
warn "Build is executed in Docker, core dumps saving path /proc/sys/kernel/core_pattern will not be configured!" warn "Build is executed in Docker, core dumps saving path /proc/sys/kernel/core_pattern will not be configured!"
else else
warn "/proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p" warn "/proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p"
echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern echo "${REPORT_PATH}/core_%e.%p" >/proc/sys/kernel/core_pattern
fi fi
fi fi
if [[ $MAKEFILE_VERBOSE = true ]] ; then if [[ $MAKEFILE_VERBOSE = true ]]; then
warn "Verbosing Makefile Commands" warn "Verbosing Makefile Commands"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
fi fi
if [[ $SCCACHE = true ]] ; then if [[ $SCCACHE = true ]]; then
warn "Use sccache" warn "Use sccache"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
fi fi
if [[ $RUN_BENCHMARKS = true ]] ; then if [[ $RUN_BENCHMARKS = true ]]; then
if [[ $MCS_BUILD_TYPE = 'Debug' ]] ; then if [[ $MCS_BUILD_TYPE = 'Debug' ]]; then
error "Benchmarks will not be build in run in Debug build Mode" error "Benchmarks will not be build in run in Debug build Mode"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO"
$RUN_BENCHMARKS = false $RUN_BENCHMARKS = false
elif [[ $OS != *"ubuntu"* && $OS != *"debian"* ]] ; then elif [[ $OS != *"ubuntu"* && $OS != *"debian"* ]]; then
error "Benchmarks are now avaliable only at Ubuntu or Debian" error "Benchmarks are now avaliable only at Ubuntu or Debian"
MAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" MAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO"
$RUN_BENCHMARKS = false $RUN_BENCHMARKS = false
else else
message "Compile with microbenchmarks" message "Compile with microbenchmarks"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES"
fi fi
else else
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO"
message "Buiding without microbenchmarks" message "Buiding without microbenchmarks"
fi fi
if [[ "$OS" == *"rocky"* ]]; then
if [[ "$OS" == *"rocky"* ]]; then
OS_VERSION=${OS//[^0-9]/} OS_VERSION=${OS//[^0-9]/}
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=rockylinux${OS_VERSION}" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=rockylinux${OS_VERSION}"
elif [[ "$OS" == "debian:11" ]]; then elif [[ "$OS" == "debian:11" ]]; then
CODENAME="bullseye" CODENAME="bullseye"
elif [[ "$OS" == "debian:12" ]]; then elif [[ "$OS" == "debian:12" ]]; then
CODENAME="bookworm" CODENAME="bookworm"
elif [[ "$OS" == "ubuntu:20.04" ]]; then elif [[ "$OS" == "ubuntu:20.04" ]]; then
CODENAME="focal" CODENAME="focal"
elif [[ "$OS" == "ubuntu:22.04" ]]; then elif [[ "$OS" == "ubuntu:22.04" ]]; then
CODENAME="jammy" CODENAME="jammy"
elif [[ "$OS" == "ubuntu:24.04" ]]; then elif [[ "$OS" == "ubuntu:24.04" ]]; then
CODENAME="noble" CODENAME="noble"
else else
echo "Unsupported OS: $OS" echo "Unsupported OS: $OS"
exit 17 exit 17
fi fi
if [[ -n "$CODENAME" ]]; then if [[ -n "$CODENAME" ]]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=${CODENAME}" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=${CODENAME}"
fi fi
if [[ $PRINT_CMAKE_FLAGS = true ]] ; then if [[ $PRINT_CMAKE_FLAGS = true ]]; then
message "Building with flags" message "Building with flags"
newline_array ${MDB_CMAKE_FLAGS[@]} newline_array ${MDB_CMAKE_FLAGS[@]}
fi fi
} }
init_submodules(){ init_submodules() {
if [[ $SKIP_SUBMODULES = true ]] ; then if [[ $SKIP_SUBMODULES = true ]]; then
warn "Skipping initialization of columnstore submodules" warn "Skipping initialization of columnstore submodules"
else else
message "Initialization of columnstore submodules" message "Initialization of columnstore submodules"
cd $MDB_SOURCE_PATH cd $MDB_SOURCE_PATH
git submodule update --init --recursive git submodule update --init --recursive
cd - > /dev/null cd - >/dev/null
fi fi
}
check_errorcode() {
if [ $? -ne 0 ]; then
message_split
error "!!!! BUILD FAILED !!!!"
message_split
exit 1
fi
cd - >/dev/null
} }
build_package() { build_package() {
@ -415,19 +411,21 @@ build_package() {
mkdir $RESULT_DIR mkdir $RESULT_DIR
if [[ $pkg_format == "rpm" ]]; then if [[ $pkg_format == "rpm" ]]; then
command="cmake ${MDB_CMAKE_FLAGS} && sleep ${BUILD_DELAY_SECONDS} && make -j\$(nproc) package" command="cmake ${MDB_CMAKE_FLAGS} && make -j\$(nproc) package"
else else
command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \ command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \
sleep ${BUILD_DELAY_SECONDS} && CMAKEFLAGS='${MDB_CMAKE_FLAGS}' debian/autobake-deb.sh" CMAKEFLAGS='${MDB_CMAKE_FLAGS}' debian/autobake-deb.sh"
fi fi
echo "Building a package for $OS" echo "Building a package for $OS"
echo "Build command: $command" echo "Build command: $command"
eval "$command" eval "$command"
}
build_binary() check_errorcode
{
}
build_binary() {
MARIA_BUILD_PATH=$(realpath $MARIA_BUILD_PATH) MARIA_BUILD_PATH=$(realpath $MARIA_BUILD_PATH)
message_split message_split
message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_cyan mode" message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_cyan mode"
@ -436,7 +434,7 @@ build_binary()
cd $MDB_SOURCE_PATH cd $MDB_SOURCE_PATH
if [[ $FORCE_CMAKE_CONFIG = true ]] ; then if [[ $FORCE_CMAKE_CONFIG = true ]]; then
warn "Erasing cmake cache" warn "Erasing cmake cache"
rm -f "$MDB_SOURCE_PATH/CMakeCache.txt" rm -f "$MDB_SOURCE_PATH/CMakeCache.txt"
rm -rf "$MDB_SOURCE_PATH/CMakeFiles" rm -rf "$MDB_SOURCE_PATH/CMakeFiles"
@ -446,21 +444,14 @@ build_binary()
${CMAKE_BIN_NAME} $MDB_CMAKE_FLAGS -S$MDB_SOURCE_PATH -B$MARIA_BUILD_PATH | spinner ${CMAKE_BIN_NAME} $MDB_CMAKE_FLAGS -S$MDB_SOURCE_PATH -B$MARIA_BUILD_PATH | spinner
message_split message_split
${CMAKE_BIN_NAME} --build $MARIA_BUILD_PATH -j $CPUS | onelinearizator && \ ${CMAKE_BIN_NAME} --build $MARIA_BUILD_PATH -j $CPUS | onelinearizator &&
message "Installing silently" && message "Installing silently" &&
${CMAKE_BIN_NAME} --install $MARIA_BUILD_PATH | spinner 30 ${CMAKE_BIN_NAME} --install $MARIA_BUILD_PATH | spinner 30
if [ $? -ne 0 ]; then check_errorcode
message_split
error "!!!! BUILD FAILED !!!!"
message_split
exit 1
fi
cd - > /dev/null
} }
check_user_and_group() check_user_and_group() {
{
user=$1 user=$1
if [ -z "$(grep $user /etc/passwd)" ]; then if [ -z "$(grep $user /etc/passwd)" ]; then
message "Adding user $user into /etc/passwd" message "Adding user $user into /etc/passwd"
@ -468,52 +459,47 @@ check_user_and_group()
fi fi
if [ -z "$(grep $user /etc/group)" ]; then if [ -z "$(grep $user /etc/group)" ]; then
GroupID = `awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group` GroupID = $(awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group)
message "Adding group $user with id $GroupID" message "Adding group $user with id $GroupID"
groupadd -g $GroupID $user groupadd -g $GroupID $user
fi fi
} }
run_unit_tests() run_unit_tests() {
{
message_split message_split
if [[ $SKIP_UNIT_TESTS = true ]] ; then if [[ $SKIP_UNIT_TESTS = true ]]; then
warn "Skipping unittests" warn "Skipping unittests"
else else
message "Running unittests" message "Running unittests"
cd $MARIA_BUILD_PATH cd $MARIA_BUILD_PATH
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure
cd - > /dev/null cd - >/dev/null
fi fi
} }
run_microbenchmarks_tests() run_microbenchmarks_tests() {
{
message_split message_split
if [[ $RUN_BENCHMARKS = false ]] ; then if [[ $RUN_BENCHMARKS = false ]]; then
warn "Skipping microbenchmarks" warn "Skipping microbenchmarks"
else else
message "Runnning microbenchmarks" message "Runnning microbenchmarks"
cd $MDB_SOURCE_PATH cd $MDB_SOURCE_PATH
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
cd - > /dev/null cd - >/dev/null
fi fi
} }
disable_plugins_for_bootstrap() disable_plugins_for_bootstrap() {
{
find /etc -type f -exec sed -i 's/plugin-load-add=auth_gssapi.so//g' {} + find /etc -type f -exec sed -i 's/plugin-load-add=auth_gssapi.so//g' {} +
find /etc -type f -exec sed -i 's/plugin-load-add=ha_columnstore.so//g' {} + find /etc -type f -exec sed -i 's/plugin-load-add=ha_columnstore.so//g' {} +
} }
enable_columnstore_back() enable_columnstore_back() {
{ echo plugin-load-add=ha_columnstore.so >>$CONFIG_DIR/columnstore.cnf
echo plugin-load-add=ha_columnstore.so >> $CONFIG_DIR/columnstore.cnf
sed -i '/\[mysqld\]/a\plugin-load-add=ha_columnstore.so' $CONFIG_DIR/columnstore.cnf sed -i '/\[mysqld\]/a\plugin-load-add=ha_columnstore.so' $CONFIG_DIR/columnstore.cnf
} }
fix_config_files() fix_config_files() {
{
message Fixing config files message Fixing config files
THREAD_STACK_SIZE="20M" THREAD_STACK_SIZE="20M"
@ -522,54 +508,52 @@ fix_config_files()
MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service
COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf
if [[ $ASAN = true ]] ; then if [[ $ASAN = true ]]; then
if grep -q thread_stack $COLUMNSTORE_CONFIG; then if grep -q thread_stack $COLUMNSTORE_CONFIG; then
warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN" warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN"
else else
echo "thread_stack = ${THREAD_STACK_SIZE}" >> $COLUMNSTORE_CONFIG echo "thread_stack = ${THREAD_STACK_SIZE}" >>$COLUMNSTORE_CONFIG
message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG" message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG"
fi fi
if grep -q ASAN $MDB_SERVICE_FILE; then if grep -q ASAN $MDB_SERVICE_FILE; then
warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility" warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility"
else else
echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${REPORT_PATH}/asan.mariadb'" >> $MDB_SERVICE_FILE echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${REPORT_PATH}/asan.mariadb'" >>$MDB_SERVICE_FILE
message "ASAN options were added to $MDB_SERVICE_FILE" message "ASAN options were added to $MDB_SERVICE_FILE"
fi fi
fi fi
if [[ $TSAN = true ]] ; then if [[ $TSAN = true ]]; then
if grep -q TSAN $MDB_SERVICE_FILE; then if grep -q TSAN $MDB_SERVICE_FILE; then
warn "MDB Server has TSAN options in $MDB_SERVICE_FILE, check it's compatibility" warn "MDB Server has TSAN options in $MDB_SERVICE_FILE, check it's compatibility"
else else
echo Environment="'TSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/tsan.mariadb'" >> $MDB_SERVICE_FILE echo Environment="'TSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/tsan.mariadb'" >>$MDB_SERVICE_FILE
message "TSAN options were added to $MDB_SERVICE_FILE" message "TSAN options were added to $MDB_SERVICE_FILE"
fi fi
fi fi
if [[ $UBSAN = true ]] ; then if [[ $UBSAN = true ]]; then
if grep -q UBSAN $MDB_SERVICE_FILE; then if grep -q UBSAN $MDB_SERVICE_FILE; then
warn "MDB Server has UBSAN options in $MDB_SERVICE_FILE, check it's compatibility" warn "MDB Server has UBSAN options in $MDB_SERVICE_FILE, check it's compatibility"
else else
echo Environment="'UBSAN_OPTIONS=abort_on_error=0,print_stacktrace=true,log_path=${REPORT_PATH}/ubsan.mariadb'" >> $MDB_SERVICE_FILE echo Environment="'UBSAN_OPTIONS=abort_on_error=0,print_stacktrace=true,log_path=${REPORT_PATH}/ubsan.mariadb'" >>$MDB_SERVICE_FILE
message "UBSAN options were added to $MDB_SERVICE_FILE" message "UBSAN options were added to $MDB_SERVICE_FILE"
fi fi
fi fi
if [[ $RECOMPILE_ONLY = false ]] ; then if [[ $RECOMPILE_ONLY = false ]]; then
message Reloading systemd message Reloading systemd
systemctl daemon-reload systemctl daemon-reload
fi fi
} }
make_dir() make_dir() {
{
mkdir -p $1 mkdir -p $1
chown mysql:mysql $1 chown mysql:mysql $1
} }
install() install() {
{ if [[ $RECOMPILE_ONLY = false ]]; then
if [[ $RECOMPILE_ONLY = false ]] ; then
message_split message_split
message "Installing MariaDB" message "Installing MariaDB"
disable_plugins_for_bootstrap disable_plugins_for_bootstrap
@ -580,17 +564,16 @@ install()
check_user_and_group mysql check_user_and_group mysql
check_user_and_group syslog check_user_and_group syslog
make_dir $CONFIG_DIR make_dir $CONFIG_DIR
echo "[client-server] echo "[client-server]
socket=/run/mysqld/mysqld.sock" > $CONFIG_DIR/socket.cnf socket=/run/mysqld/mysqld.sock" >$CONFIG_DIR/socket.cnf
mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so
make_dir /var/lib/mysql make_dir /var/lib/mysql
message "Running mysql_install_db" message "Running mysql_install_db"
sudo -u mysql mysql_install_db --rpm --user=mysql > /dev/null sudo -u mysql mysql_install_db --rpm --user=mysql >/dev/null
mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so
enable_columnstore_back enable_columnstore_back
@ -608,7 +591,7 @@ install()
make_dir /etc/mysql/ make_dir /etc/mysql/
cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start
> /etc/mysql/debian.cnf >/etc/mysql/debian.cnf
fi fi
fix_config_files fix_config_files
@ -648,16 +631,14 @@ install()
chmod 777 /var/log/mariadb/columnstore chmod 777 /var/log/mariadb/columnstore
} }
smoke() {
smoke() if [[ $SKIP_SMOKE = false ]]; then
{
if [[ $SKIP_SMOKE = false ]] ; then
message_split message_split
message "Creating test database" message "Creating test database"
mariadb -e "create database if not exists test;" mariadb -e "create database if not exists test;"
message "Selecting magic numbers" message "Selecting magic numbers"
MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql` MAGIC=$(mysql -N test <$MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql)
if [[ $MAGIC == '42' ]] ; then if [[ $MAGIC == '42' ]]; then
message "Great answer correct!" message "Great answer correct!"
else else
warn "Smoke failed, answer is '$MAGIC'" warn "Smoke failed, answer is '$MAGIC'"
@ -665,47 +646,45 @@ smoke()
fi fi
} }
generate_svgs() {
generate_svgs() if [[ $DRAW_DEPS = true ]]; then
{ message_split
if [[ $DRAW_DEPS = true ]] ; then warn "Generating svgs with dependency graph to $REPORT_PATH"
message_split for f in $MDB_SOURCE_PATH/mariadb.dot.*; do
warn "Generating svgs with dependency graph to $REPORT_PATH" dot -Tsvg -o $REPORT_PATH/$(basename $f).svg $f
for f in $MDB_SOURCE_PATH/mariadb.dot.*;
do dot -Tsvg -o $REPORT_PATH/`basename $f`.svg $f;
done done
fi fi
} }
if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]] ; then if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]]; then
install_deps install_deps
fi fi
if [[ $DO_NOT_FREEZE_REVISION = false ]] ; then if [[ $DO_NOT_FREEZE_REVISION = false ]]; then
disable_git_restore_frozen_revision disable_git_restore_frozen_revision
fi fi
construct_cmake_flags construct_cmake_flags
init_submodules init_submodules
if [[ $BUILD_PACKAGES = false ]] ; then if [[ $BUILD_PACKAGES = false ]]; then
select_branch select_branch
stop_service stop_service
if [[ $NO_CLEAN = false ]] ; then if [[ $NO_CLEAN = false ]]; then
clean_old_installation clean_old_installation
fi fi
build_binary build_binary
run_unit_tests run_unit_tests
run_microbenchmarks_tests run_microbenchmarks_tests
install install
if [[ $RESTART_SERVICES = true ]] ; then if [[ $RESTART_SERVICES = true ]]; then
start_service start_service
smoke smoke
generate_svgs generate_svgs
fi fi
else else
build_package build_package
fi fi
message_splitted "FINISHED" message_splitted "FINISHED"