You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-08-01 06:46:55 +03:00
CI from develop + cmapi [develop-23.02] (#2863)
* MCOL-5496: Merge CMAPI code to engine repo. [add] cmapi code to engine * MCOL-5496: Fix CI adding CMAPI steps. [fix] deb packages deps commands [add] several additional local variables [fix] packages url [fix] smoke step [fix] mtr step [fix] regression step [add] cmapipython, cmapibuild, cmapitest and cmapilog steps [fix] dockerfile step [fix] build step to include cmapi package on repodata creating [fix] multi_node_mtr step [fix] pkg step [add] cmapi steps to pipelines [fix] cmapi/CMakeLists.txt to prevent cmake byte-compile .py files for rpm packages [add] setup-repo.sh file [fix] now use packages from the repos in tests steps * Adding color 2 build script * Build script and logging scripts from develop * Remove test 222 from full regression, it is missing in 23.02 regression set --------- Co-authored-by: mariadb-AlanMologorsky <alan.mologorsky@mariadb.com> Co-authored-by: mariadb-RomanNavrotskiy <roman.navrotskiy@mariadb.com>
This commit is contained in:
Leonid Fedorov
GitHub
@ -22,7 +22,7 @@ DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky")
|
||||
cd $SCRIPT_LOCATION
|
||||
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo))
|
||||
cd -
|
||||
cd - > /dev/null
|
||||
|
||||
|
||||
optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
|
||||
@ -34,10 +34,17 @@ optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP
|
||||
optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
|
||||
optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH
|
||||
optparse.define short=D long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
|
||||
optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
|
||||
optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
|
||||
optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
|
||||
optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true
|
||||
optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
|
||||
optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core"
|
||||
optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true
|
||||
optparse.define short=G long=draw-deps desc="Draw dependencies graph" variable=DRAW_DEPS default=false value=true
|
||||
optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true
|
||||
optparse.define short=n long=no-clean-install desc="Do not perform a clean install (keep existing db files)" variable=NO_CLEAN default=false value=true
|
||||
optparse.define short=j long=parallel desc="Number of paralles for build" variable=CPUS default=$(getconf _NPROCESSORS_ONLN)
|
||||
optparse.define short=F long=show-build-flags desc="Print CMake flags, while build" variable=PRINT_CMAKE_FLAGS default=false
|
||||
|
||||
source $( optparse.build )
|
||||
|
||||
@ -79,15 +86,16 @@ select_branch()
|
||||
message "Turning off Columnstore submodule auto update via gitconfig"
|
||||
cd $MDB_SOURCE_PATH
|
||||
git config submodule.storage/columnstore/columnstore.update none
|
||||
cd -
|
||||
cd - > /dev/null
|
||||
fi
|
||||
|
||||
cd -
|
||||
cd - > /dev/null
|
||||
message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch"
|
||||
}
|
||||
|
||||
install_deps()
|
||||
{
|
||||
message_split
|
||||
message "Installing deps"
|
||||
if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
|
||||
apt-get -y update
|
||||
@ -95,9 +103,9 @@ install_deps()
|
||||
libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \
|
||||
libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \
|
||||
libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev \
|
||||
liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev
|
||||
liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev graphviz
|
||||
|
||||
elif [[ $OS = 'CentOS' || $OS = 'Rocky' ]]; then
|
||||
elif [[ $OS = 'CentOS' || $OS = 'Rocky' || $OS = 'Fedora' ]]; then
|
||||
if [[ "$OS_VERSION" == "7" ]]; then
|
||||
yum -y install cmake3 epel-release centos-release-scl
|
||||
CMAKE_BIN_NAME=cmake3
|
||||
@ -109,16 +117,23 @@ install_deps()
|
||||
yum -y groupinstall "Development Tools" && yum config-manager --set-enabled powertools
|
||||
yum install -y checkpolicy
|
||||
fi
|
||||
yum -y install epel-release \
|
||||
&& yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \
|
||||
&& yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts \
|
||||
&& yum -y install jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel install lzo-devel xz-devel lz4-devel bzip2-devel \
|
||||
&& yum -y install pcre2-devel
|
||||
if [[ $OS != 'Fedora' ]]; then
|
||||
yum -y install epel-release
|
||||
fi
|
||||
|
||||
yum install -y bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \
|
||||
vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts \
|
||||
jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel lzo-devel xz-devel lz4-devel bzip2-devel \
|
||||
pcre2-devel flex graphviz libaio-devel openssl-devel flex
|
||||
else
|
||||
error "Unsupported OS $OS"
|
||||
exit 17
|
||||
fi
|
||||
}
|
||||
|
||||
stop_service()
|
||||
{
|
||||
message_split
|
||||
message "Stopping MariaDB services"
|
||||
systemctl stop mariadb
|
||||
systemctl stop mariadb-columnstore
|
||||
@ -127,25 +142,33 @@ stop_service()
|
||||
check_service()
|
||||
{
|
||||
if systemctl is-active --quiet $1; then
|
||||
message "$1 service started$color_green OK $color_normal"
|
||||
message "$1 $color_normal[$color_green OK $color_normal]"
|
||||
else
|
||||
error "$1 service failed"
|
||||
message "$1 $color_normal[$color_red Fail $color_normal]"
|
||||
service $1 status
|
||||
fi
|
||||
}
|
||||
|
||||
start_service()
|
||||
{
|
||||
message_split
|
||||
message "Starting MariaDB services"
|
||||
systemctl start mariadb-columnstore
|
||||
systemctl start mariadb
|
||||
|
||||
check_service mariadb-columnstore
|
||||
check_service mariadb
|
||||
check_service mariadb-columnstore
|
||||
check_service mcs-controllernode
|
||||
check_service mcs-ddlproc
|
||||
check_service mcs-dmlproc
|
||||
check_service mcs-primproc
|
||||
check_service mcs-workernode@1
|
||||
check_service mcs-writeengineserver
|
||||
}
|
||||
|
||||
clean_old_installation()
|
||||
{
|
||||
message_split
|
||||
message "Cleaning old installation"
|
||||
rm -rf /var/lib/columnstore/data1/*
|
||||
rm -rf /var/lib/columnstore/data/
|
||||
@ -163,6 +186,7 @@ clean_old_installation()
|
||||
|
||||
build()
|
||||
{
|
||||
message_split
|
||||
message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode"
|
||||
|
||||
local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes
|
||||
@ -191,16 +215,37 @@ build()
|
||||
message "Buiding with unittests"
|
||||
fi
|
||||
|
||||
if [[ $DRAW_DEPS = true ]] ; then
|
||||
warn "Generating dependendies graph to mariadb.dot"
|
||||
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} --graphviz=mariadb.dot"
|
||||
fi
|
||||
|
||||
if [[ $USE_NINJA = true ]] ; then
|
||||
warn "Using Ninja instead of Makefiles"
|
||||
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja"
|
||||
fi
|
||||
|
||||
if [[ $ASAN = true ]] ; then
|
||||
warn "Building with ASAN"
|
||||
warn "Building with Address Sanitizer "
|
||||
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
|
||||
fi
|
||||
|
||||
if [[ $TSAN = true ]] ; then
|
||||
warn "Building with Thread Sanitizer"
|
||||
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
|
||||
fi
|
||||
|
||||
if [[ $UBSAN = true ]] ; then
|
||||
warn "Building with UB Sanitizer"
|
||||
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
|
||||
fi
|
||||
|
||||
if [[ $WITHOUT_COREDUMPS = true ]] ; then
|
||||
warn "Cores are not dumped"
|
||||
else
|
||||
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON"
|
||||
echo "${REPORT_PATH}/core_%e.%p" | sudo tee /proc/sys/kernel/core_pattern
|
||||
warn Building with CoreDumps: /proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p
|
||||
echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern
|
||||
fi
|
||||
|
||||
if [[ $MAKEFILE_VERBOSE = true ]] ; then
|
||||
@ -231,10 +276,10 @@ build()
|
||||
if [[ $SKIP_SUBMODULES = true ]] ; then
|
||||
warn "Skipping initialization of columnstore submodules"
|
||||
else
|
||||
message "Initialization of columnstore submodules"
|
||||
cd storage/columnstore/columnstore
|
||||
git submodule update --init
|
||||
cd -
|
||||
message "Initialization of columnstore submodules"
|
||||
cd storage/columnstore/columnstore
|
||||
git submodule update --init
|
||||
cd - > /dev/null
|
||||
fi
|
||||
|
||||
if [[ $FORCE_CMAKE_CONFIG = true ]] ; then
|
||||
@ -253,19 +298,25 @@ build()
|
||||
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15"
|
||||
fi
|
||||
|
||||
message "building with flags $MDB_CMAKE_FLAGS"
|
||||
if [[ $PRINT_CMAKE_FLAGS = true ]] ; then
|
||||
message "Building with flags"
|
||||
newline_array ${MDB_CMAKE_FLAGS[@]}
|
||||
fi
|
||||
|
||||
local CPUS=$(getconf _NPROCESSORS_ONLN)
|
||||
${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ |
|
||||
make -j $CPUS
|
||||
message "Installing silently"
|
||||
make -j $CPUS install > /dev/null
|
||||
message "Configuring cmake silently"
|
||||
${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS . | spinner
|
||||
message_split
|
||||
${CMAKE_BIN_NAME} --build . -j $CPUS && \
|
||||
message "Installing silently" &&
|
||||
${CMAKE_BIN_NAME} --install . | spinner 30
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
message_split
|
||||
error "!!!! BUILD FAILED !!!!"
|
||||
message_split
|
||||
exit 1
|
||||
fi
|
||||
cd -
|
||||
cd - > /dev/null
|
||||
}
|
||||
|
||||
check_user_and_group()
|
||||
@ -284,25 +335,27 @@ check_user_and_group()
|
||||
|
||||
run_unit_tests()
|
||||
{
|
||||
message_split
|
||||
if [[ $SKIP_UNIT_TESTS = true ]] ; then
|
||||
warn "Skipping unittests"
|
||||
else
|
||||
message "Running unittests"
|
||||
cd $MDB_SOURCE_PATH
|
||||
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress
|
||||
cd -
|
||||
cd - > /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
run_microbenchmarks_tests()
|
||||
{
|
||||
message_split
|
||||
if [[ $RUN_BENCHMARKS = false ]] ; then
|
||||
warn "Skipping microbenchmarks"
|
||||
else
|
||||
message "Runnning microbenchmarks"
|
||||
cd $MDB_SOURCE_PATH
|
||||
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
|
||||
cd -
|
||||
cd - > /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
@ -324,8 +377,10 @@ fix_config_files()
|
||||
THREAD_STACK_SIZE="20M"
|
||||
|
||||
SYSTEMD_SERVICE_DIR="/usr/lib/systemd/system"
|
||||
MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service
|
||||
COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf
|
||||
|
||||
if [[ $ASAN = true ]] ; then
|
||||
COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf
|
||||
if grep -q thread_stack $COLUMNSTORE_CONFIG; then
|
||||
warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN"
|
||||
else
|
||||
@ -333,19 +388,39 @@ fix_config_files()
|
||||
message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG"
|
||||
fi
|
||||
|
||||
MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service
|
||||
if grep -q ASAN $MDB_SERVICE_FILE; then
|
||||
warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility"
|
||||
else
|
||||
echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}'" >> $MDB_SERVICE_FILE
|
||||
echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${REPORT_PATH}/asan.mariadb'" >> $MDB_SERVICE_FILE
|
||||
message "ASAN options were added to $MDB_SERVICE_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $TSAN = true ]] ; then
|
||||
if grep -q TSAN $MDB_SERVICE_FILE; then
|
||||
warn "MDB Server has TSAN options in $MDB_SERVICE_FILE, check it's compatibility"
|
||||
else
|
||||
echo Environment="'TSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/tsan.mariadb'" >> $MDB_SERVICE_FILE
|
||||
message "TSAN options were added to $MDB_SERVICE_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $UBSAN = true ]] ; then
|
||||
if grep -q UBSAN $MDB_SERVICE_FILE; then
|
||||
warn "MDB Server has UBSAN options in $MDB_SERVICE_FILE, check it's compatibility"
|
||||
else
|
||||
echo Environment="'UBSAN_OPTIONS=abort_on_error=0,print_stacktrace=true,log_path=${REPORT_PATH}/ubsan.mariadb'" >> $MDB_SERVICE_FILE
|
||||
message "UBSAN options were added to $MDB_SERVICE_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
message Reloading systemd
|
||||
systemctl daemon-reload
|
||||
}
|
||||
|
||||
install()
|
||||
{
|
||||
message_split
|
||||
message "Installing MariaDB"
|
||||
disable_plugins_for_bootstrap
|
||||
|
||||
@ -425,14 +500,29 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
|
||||
|
||||
smoke()
|
||||
{
|
||||
message "Creating test database"
|
||||
mariadb -e "create database if not exists test;"
|
||||
message "Selecting magic numbers"
|
||||
MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql`
|
||||
if [[ $MAGIC == '42' ]] ; then
|
||||
message "Great answer correct"
|
||||
else
|
||||
warn "Smoke failed, answer is '$MAGIC'"
|
||||
if [[ $SKIP_SMOKE = false ]] ; then
|
||||
message_split
|
||||
message "Creating test database"
|
||||
mariadb -e "create database if not exists test;"
|
||||
message "Selecting magic numbers"
|
||||
MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql`
|
||||
if [[ $MAGIC == '42' ]] ; then
|
||||
message "Great answer correct!"
|
||||
else
|
||||
warn "Smoke failed, answer is '$MAGIC'"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
generate_svgs()
|
||||
{
|
||||
if [[ $DRAW_DEPS = true ]] ; then
|
||||
message_split
|
||||
warn "Generating svgs with dependency graph to $REPORT_PATH"
|
||||
for f in $MDB_SOURCE_PATH/mariadb.dot.*;
|
||||
do dot -Tsvg -o $REPORT_PATH/`basename $f`.svg $f;
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
@ -454,5 +544,6 @@ run_microbenchmarks_tests
|
||||
install
|
||||
start_service
|
||||
smoke
|
||||
generate_svgs
|
||||
|
||||
message "$color_green FINISHED $color_normal"
|
||||
message_splitted "FINISHED"
|
||||
|
||||
Reference in New Issue
Block a user