1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00

Make build scripts color brighter (#2803)

* Make build scripts color brighter

* better colors, draw deps and ninja generator options

* Add color spinner for configure and install make changed to cmake --build and cmake --install

* Clean more builds log garbage
This commit is contained in:
Leonid Fedorov
2023-04-11 03:35:05 +03:00
committed by GitHub
parent c38d98a510
commit 105cf2da22
2 changed files with 132 additions and 20 deletions

View File

@ -22,7 +22,7 @@ DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky")
cd $SCRIPT_LOCATION cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo))
cd - cd - > /dev/null
optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
@ -37,6 +37,8 @@ optparse.define short=D long=without-core-dumps desc="Do not produce core dumps"
optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core"
optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true
optparse.define short=T long=draw-deps desc="Draw dependencies graph" variable=DRAW_DEPS default=false value=true
source $( optparse.build ) source $( optparse.build )
@ -78,15 +80,16 @@ select_branch()
message "Turning off Columnstore submodule auto update via gitconfig" message "Turning off Columnstore submodule auto update via gitconfig"
cd $MDB_SOURCE_PATH cd $MDB_SOURCE_PATH
git config submodule.storage/columnstore/columnstore.update none git config submodule.storage/columnstore/columnstore.update none
cd - cd - > /dev/null
fi fi
cd - cd - > /dev/null
message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch" message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch"
} }
install_deps() install_deps()
{ {
message_split
message "Installing deps" message "Installing deps"
if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
apt-get -y update apt-get -y update
@ -94,7 +97,7 @@ install_deps()
libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \ libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \
libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \ libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \
libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev \ libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev \
liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev graphviz
elif [[ $OS = 'CentOS' || $OS = 'Rocky' ]]; then elif [[ $OS = 'CentOS' || $OS = 'Rocky' ]]; then
if [[ "$OS_VERSION" == "7" ]]; then if [[ "$OS_VERSION" == "7" ]]; then
@ -112,12 +115,13 @@ install_deps()
&& yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \ && yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \
&& yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts \ && yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts \
&& yum -y install jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel lzo-devel xz-devel lz4-devel bzip2-devel \ && yum -y install jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel lzo-devel xz-devel lz4-devel bzip2-devel \
&& yum -y install pcre2-devel && yum -y install pcre2-devel flex graphviz
fi fi
} }
stop_service() stop_service()
{ {
message_split
message "Stopping MariaDB services" message "Stopping MariaDB services"
systemctl stop mariadb systemctl stop mariadb
systemctl stop mariadb-columnstore systemctl stop mariadb-columnstore
@ -135,6 +139,7 @@ check_service()
start_service() start_service()
{ {
message_split
message "Starting MariaDB services" message "Starting MariaDB services"
systemctl start mariadb-columnstore systemctl start mariadb-columnstore
systemctl start mariadb systemctl start mariadb
@ -145,6 +150,7 @@ start_service()
clean_old_installation() clean_old_installation()
{ {
message_split
message "Cleaning old installation" message "Cleaning old installation"
rm -rf /var/lib/columnstore/data1/* rm -rf /var/lib/columnstore/data1/*
rm -rf /var/lib/columnstore/data/ rm -rf /var/lib/columnstore/data/
@ -162,6 +168,7 @@ clean_old_installation()
build() build()
{ {
message_split
message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode" message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode"
local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes
@ -190,6 +197,17 @@ build()
message "Buiding with unittests" message "Buiding with unittests"
fi fi
if [[ $DRAW_DEPS = true ]] ; then
warn "Generating dependendies graph to mariadb.dot"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} --graphviz=mariadb.dot"
fi
if [[ $USE_NINJA = true ]] ; then
warn "Using Ninja instead of Makefiles"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja"
fi
if [[ $ASAN = true ]] ; then if [[ $ASAN = true ]] ; then
warn "Building with ASAN" warn "Building with ASAN"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
@ -199,7 +217,9 @@ build()
warn "Cores are not dumped" warn "Cores are not dumped"
else else
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON"
echo "${REPORT_PATH}/core_%e.%p" | sudo tee /proc/sys/kernel/core_pattern
warn Builds with boreDumps CoreDump pattern changed to ${REPORT_PATH}/core_%e.%p
echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern
fi fi
if [[ $MAKEFILE_VERBOSE = true ]] ; then if [[ $MAKEFILE_VERBOSE = true ]] ; then
@ -233,7 +253,7 @@ build()
message "Initialization of columnstore submodules" message "Initialization of columnstore submodules"
cd storage/columnstore/columnstore cd storage/columnstore/columnstore
git submodule update --init git submodule update --init
cd - cd - > /dev/null
fi fi
if [[ $FORCE_CMAKE_CONFIG = true ]] ; then if [[ $FORCE_CMAKE_CONFIG = true ]] ; then
@ -252,19 +272,24 @@ build()
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15"
fi fi
message "building with flags $MDB_CMAKE_FLAGS" message "Building with flags"
newline_array ${MDB_CMAKE_FLAGS[@]}
local CPUS=$(getconf _NPROCESSORS_ONLN) local CPUS=$(getconf _NPROCESSORS_ONLN)
${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ | message "Configuring cmake silently"
make -j $CPUS ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS . | spinner
message "Installing silently" message_split
make -j $CPUS install > /dev/null ${CMAKE_BIN_NAME} --build . -j $CPUS && \
message "Installing silently" &&
${CMAKE_BIN_NAME} --install . | spinner 30
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
message_split
error "!!!! BUILD FAILED !!!!" error "!!!! BUILD FAILED !!!!"
message_split
exit 1 exit 1
fi fi
cd - cd - > /dev/null
} }
check_user_and_group() check_user_and_group()
@ -283,25 +308,27 @@ check_user_and_group()
run_unit_tests() run_unit_tests()
{ {
message_split
if [[ $SKIP_UNIT_TESTS = true ]] ; then if [[ $SKIP_UNIT_TESTS = true ]] ; then
warn "Skipping unittests" warn "Skipping unittests"
else else
message "Running unittests" message "Running unittests"
cd $MDB_SOURCE_PATH cd $MDB_SOURCE_PATH
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress
cd - cd - > /dev/null
fi fi
} }
run_microbenchmarks_tests() run_microbenchmarks_tests()
{ {
message_split
if [[ $RUN_BENCHMARKS = false ]] ; then if [[ $RUN_BENCHMARKS = false ]] ; then
warn "Skipping microbenchmarks" warn "Skipping microbenchmarks"
else else
message "Runnning microbenchmarks" message "Runnning microbenchmarks"
cd $MDB_SOURCE_PATH cd $MDB_SOURCE_PATH
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
cd - cd - > /dev/null
fi fi
} }
@ -345,6 +372,7 @@ fix_config_files()
install() install()
{ {
message_split
message "Installing MariaDB" message "Installing MariaDB"
disable_plugins_for_bootstrap disable_plugins_for_bootstrap
@ -435,6 +463,18 @@ smoke()
fi fi
} }
generate_svgs()
{
if [[ $DRAW_DEPS = true ]] ; then
message_split
warn "Generating svgs with dependency graph to $REPORT_PATH"
for f in $MDB_SOURCE_PATH/mariadb.dot.*;
do dot -Tsvg -o $REPORT_PATH/`basename $f`.svg $f;
done
fi
}
select_branch select_branch
if [[ $INSTALL_DEPS = true ]] ; then if [[ $INSTALL_DEPS = true ]] ; then
@ -449,5 +489,8 @@ run_microbenchmarks_tests
install install
start_service start_service
smoke smoke
generate_svgs
message "$color_green FINISHED $color_normal"
message_splitted "FINISHED"

View File

@ -1,10 +1,11 @@
color_normal=$(tput sgr0) color_normal=$(tput sgr0)
color_bold=$(tput bold) color_bold=$(tput bold)
color_red="$color_bold$(tput setaf 1)" color_red="$color_bold$(tput setaf 1)"
color_green=$(tput setaf 2) color_green="$color_bold$(tput setaf 2)"
color_fawn=$(tput setaf 3); color_beige="$color_fawn" color_fawn=$(tput setaf 3); color_beige="$color_fawn"
color_yellow="$color_bold$color_fawn" color_yellow="$color_bold$color_fawn"
color_darkblue=$(tput setaf 4) color_darkblue=$(tput setaf 4)
color_blue="$color_bold$color_darkblue" color_blue="$color_bold$color_darkblue"
color_purple=$(tput setaf 5); color_magenta="$color_purple" color_purple=$(tput setaf 5); color_magenta="$color_purple"
color_pink="$color_bold$color_purple" color_pink="$color_bold$color_purple"
@ -14,21 +15,89 @@ color_gray=$(tput setaf 7)
color_darkgray="$color_bold"$(tput setaf 0) color_darkgray="$color_bold"$(tput setaf 0)
color_white="$color_bold$color_gray" color_white="$color_bold$color_gray"
if [[ $(tput colors) == '256' ]]; then
color_red=$(tput setaf 196)
color_yellow=$(tput setaf 228)
color_cyan=$(tput setaf 87)
color_green=$(tput setaf 156)
color_darkgray=$(tput setaf 59)
fi
message() message()
{ {
echo $color_cyan -- $@$color_normal echo $color_cyan $@$color_normal
} }
warn() warn()
{ {
echo $color_yellow -- $@$color_normal echo $color_yellow $@$color_normal
} }
error() error()
{ {
echo $color_red -- $@$color_normal echo $color_red $@$color_normal
} }
message_split()
{
echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal
}
message_splitted()
{
message_split
echo $color_green$@$color_normal
message_split
}
colorify_array()
{
PROMT=""
for a in "$@"
do
i=$((((i+1) % (123-106)) + 106))
if [[ $(tput colors) == '256' ]]; then
PROMT="$PROMT $(tput setaf $i)$a$color_normal"
else
PROMT="$PROMT $a"
fi
done
echo $PROMT
}
newline_array()
{
PROMT=""
for a in "$@"
do
PROMT="$PROMT$a\n"
done
echo -e $PROMT
}
function spinner
{
freq=${1:-10}
points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷)
colored_points=($(colorify_array ${points[@]}))
len=${#points[@]}
point_num=0
line_num=0
while read data; do
line_num=$((line_num+1))
if [[ $((line_num % freq)) = 0 ]]; then
point_num=$(((point_num + 1) % len ))
echo -ne "\r${colored_points[point_num]}"
fi
done;
echo
}
detect_distro() detect_distro()
{ {
if [ -f /etc/os-release ]; then if [ -f /etc/os-release ]; then