You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-29 08:21:15 +03:00
chore(build): better build pipeline
This commit is contained in:
Leonid Fedorov
committed by
Leonid Fedorov
Leonid Fedorov
parent
ddfbeb192f
commit
2d8e16f08e
@ -97,7 +97,7 @@ install_deps() {
|
||||
bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
|
||||
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
|
||||
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
|
||||
graphviz devscripts ccache equivs eatmydata curl"
|
||||
graphviz devscripts ccache equivs eatmydata curl python3"
|
||||
|
||||
if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
|
||||
command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
|
||||
@ -144,6 +144,9 @@ if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
|
||||
fi
|
||||
|
||||
disable_git_restore_frozen_revision() {
|
||||
if [[ $DO_NOT_FREEZE_REVISION = true ]]; then
|
||||
return
|
||||
fi
|
||||
cd $MDB_SOURCE_PATH
|
||||
git config submodule.storage/columnstore/columnstore.update none
|
||||
cd - >/dev/null
|
||||
@ -152,6 +155,15 @@ disable_git_restore_frozen_revision() {
|
||||
DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot"
|
||||
|
||||
stop_service() {
|
||||
if [[ $RESTART_SERVICES = false || $RECOMPILE_ONLY = true ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
error "Please run script as root to be able to manage services"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
message_split
|
||||
message "Stopping MariaDB services"
|
||||
systemctl stop mariadb
|
||||
@ -169,6 +181,15 @@ check_service() {
|
||||
}
|
||||
|
||||
start_service() {
|
||||
if [[ $RESTART_SERVICES = false || $RECOMPILE_ONLY = true ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
error "Please run script as root to be able to manage services"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
message_split
|
||||
message "Starting MariaDB services"
|
||||
systemctl start mariadb-columnstore
|
||||
@ -185,16 +206,26 @@ start_service() {
|
||||
}
|
||||
|
||||
start_storage_manager_if_needed() {
|
||||
if [[ $CLOUD_STORAGE_ENABLED = true ]]; then
|
||||
export MCS_USE_S3_STORAGE=1
|
||||
message_split
|
||||
message "Starting Storage Manager service"
|
||||
systemctl start mcs-storagemanager
|
||||
check_service mcs-storagemanager
|
||||
if [[ $CLOUD_STORAGE_ENABLED = false ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
export MCS_USE_S3_STORAGE=1
|
||||
message_split
|
||||
message "Starting Storage Manager service"
|
||||
systemctl start mcs-storagemanager
|
||||
check_service mcs-storagemanager
|
||||
}
|
||||
|
||||
clean_old_installation() {
|
||||
if [[ $NO_CLEAN = true || $RECOMPILE_ONLY = true ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
error "Please run script as root to be able to clean installations"
|
||||
exit 1
|
||||
fi
|
||||
message_split
|
||||
message "Cleaning old installation"
|
||||
rm -rf /var/lib/columnstore/data1/*
|
||||
@ -479,19 +510,28 @@ build_binary() {
|
||||
message "Configuring cmake silently"
|
||||
${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
|
||||
message_split
|
||||
check_debian_install_file
|
||||
# check_debian_install_file // will be uncommented later
|
||||
generate_svgs
|
||||
|
||||
${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
|
||||
message "Installing silently" &&
|
||||
${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30
|
||||
|
||||
${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator
|
||||
check_errorcode
|
||||
|
||||
message "Adding symbol link to compile_commands.json to the source root"
|
||||
ln -sf "$MARIA_BUILD_PATH/compile_commands.json" "$MDB_SOURCE_PATH"
|
||||
}
|
||||
|
||||
install_binary() {
|
||||
if [[ $RECOMPILE_ONLY = true ]]; then
|
||||
warn "No binary installation done"
|
||||
return
|
||||
fi
|
||||
|
||||
message "Installing silently"
|
||||
cd $MDB_SOURCE_PATH
|
||||
${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30
|
||||
check_errorcode
|
||||
}
|
||||
|
||||
check_user_and_group() {
|
||||
user=$1
|
||||
if [ -z "$(grep "$user" /etc/passwd)" ]; then
|
||||
@ -510,24 +550,26 @@ run_unit_tests() {
|
||||
message_split
|
||||
if [[ $SKIP_UNIT_TESTS = true ]]; then
|
||||
warn "Skipping unittests"
|
||||
else
|
||||
message "Running unittests"
|
||||
cd $MARIA_BUILD_PATH
|
||||
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure
|
||||
cd - >/dev/null
|
||||
return
|
||||
fi
|
||||
|
||||
message "Running unittests"
|
||||
cd $MARIA_BUILD_PATH
|
||||
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure
|
||||
cd - >/dev/null
|
||||
}
|
||||
|
||||
run_microbenchmarks_tests() {
|
||||
message_split
|
||||
if [[ $RUN_BENCHMARKS = false ]]; then
|
||||
warn "Skipping microbenchmarks"
|
||||
else
|
||||
message "Runnning microbenchmarks"
|
||||
cd $MARIA_BUILD_PATH
|
||||
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
|
||||
cd - >/dev/null
|
||||
return
|
||||
fi
|
||||
|
||||
message "Runnning microbenchmarks"
|
||||
cd $MARIA_BUILD_PATH
|
||||
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
|
||||
cd - >/dev/null
|
||||
}
|
||||
|
||||
disable_plugins_for_bootstrap() {
|
||||
@ -593,95 +635,99 @@ make_dir() {
|
||||
}
|
||||
|
||||
install() {
|
||||
if [[ $RECOMPILE_ONLY = false ]]; then
|
||||
if [[ $RECOMPILE_ONLY = true ]]; then
|
||||
warn "No install configuration done"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
error "Please run script as root to install MariaDb to system paths"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
error "Please run script as root to install MariaDb to system paths"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
message_split
|
||||
message "Installing MariaDB"
|
||||
disable_plugins_for_bootstrap
|
||||
message_split
|
||||
message "Installing MariaDB"
|
||||
disable_plugins_for_bootstrap
|
||||
|
||||
make_dir "$REPORT_PATH"
|
||||
chmod 777 "$REPORT_PATH"
|
||||
make_dir "$REPORT_PATH"
|
||||
chmod 777 "$REPORT_PATH"
|
||||
|
||||
check_user_and_group mysql
|
||||
check_user_and_group syslog
|
||||
check_user_and_group mysql
|
||||
check_user_and_group syslog
|
||||
|
||||
make_dir $CONFIG_DIR
|
||||
make_dir $CONFIG_DIR
|
||||
|
||||
echo "[client-server]
|
||||
echo "[client-server]
|
||||
socket=/run/mysqld/mysqld.sock" >$CONFIG_DIR/socket.cnf
|
||||
|
||||
make_dir /var/lib/mysql
|
||||
make_dir /var/lib/mysql
|
||||
|
||||
message "Running mysql_install_db"
|
||||
sudo -u mysql mysql_install_db --rpm --user=mysql >/dev/null
|
||||
message "Running mariadb-install-db"
|
||||
sudo -u mysql mariadb-install-db --rpm --user=mysql >/dev/null
|
||||
|
||||
enable_columnstore_back
|
||||
enable_columnstore_back
|
||||
|
||||
make_dir /etc/columnstore
|
||||
make_dir /etc/columnstore
|
||||
|
||||
if [[ "$NO_CLEAN" == false ]]; then
|
||||
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
|
||||
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
|
||||
fi
|
||||
|
||||
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
|
||||
|
||||
if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then
|
||||
make_dir /usr/share/mysql
|
||||
make_dir /etc/mysql/
|
||||
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
|
||||
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start
|
||||
>/etc/mysql/debian.cnf
|
||||
fi
|
||||
|
||||
fix_config_files
|
||||
|
||||
if [ -d "$DEBCONFIG_DIR" ]; then
|
||||
message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR"
|
||||
cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR"
|
||||
fi
|
||||
|
||||
make_dir /var/lib/columnstore/data1
|
||||
make_dir /var/lib/columnstore/data1/systemFiles
|
||||
make_dir /var/lib/columnstore/data1/systemFiles/dbrm
|
||||
make_dir /run/mysqld/
|
||||
make_dir $DATA_DIR
|
||||
|
||||
chmod +x $INSTALL_PREFIX/bin/mariadb*
|
||||
|
||||
ldconfig
|
||||
|
||||
start_storage_manager_if_needed
|
||||
|
||||
message "Running columnstore-post-install"
|
||||
make_dir /var/lib/columnstore/local
|
||||
columnstore-post-install --rpmmode=install
|
||||
message "Running install_mcs_mysql"
|
||||
install_mcs_mysql.sh
|
||||
if [[ "$NO_CLEAN" == false ]]; then
|
||||
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
|
||||
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
|
||||
fi
|
||||
|
||||
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
|
||||
|
||||
if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then
|
||||
make_dir /usr/share/mysql
|
||||
make_dir /etc/mysql/
|
||||
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
|
||||
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start
|
||||
>/etc/mysql/debian.cnf
|
||||
fi
|
||||
|
||||
fix_config_files
|
||||
|
||||
if [ -d "$DEBCONFIG_DIR" ]; then
|
||||
message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR"
|
||||
cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR"
|
||||
fi
|
||||
|
||||
make_dir /var/lib/columnstore/data1
|
||||
make_dir /var/lib/columnstore/data1/systemFiles
|
||||
make_dir /var/lib/columnstore/data1/systemFiles/dbrm
|
||||
make_dir /run/mysqld/
|
||||
make_dir $DATA_DIR
|
||||
|
||||
chmod +x $INSTALL_PREFIX/bin/mariadb*
|
||||
|
||||
ldconfig
|
||||
|
||||
start_storage_manager_if_needed
|
||||
|
||||
message "Running columnstore-post-install"
|
||||
make_dir /var/lib/columnstore/local
|
||||
columnstore-post-install --rpmmode=install
|
||||
message "Running install_mcs_mysql"
|
||||
install_mcs_mysql.sh
|
||||
|
||||
chown -R syslog:syslog /var/log/mariadb/
|
||||
chmod 777 /var/log/mariadb/
|
||||
chmod 777 /var/log/mariadb/columnstore
|
||||
}
|
||||
|
||||
smoke() {
|
||||
if [[ $SKIP_SMOKE = false ]]; then
|
||||
message_split
|
||||
message "Creating test database"
|
||||
mariadb -e "create database if not exists test;"
|
||||
message "Selecting magic numbers"
|
||||
MAGIC=$(mysql -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql)
|
||||
if [[ $MAGIC == '42' ]]; then
|
||||
message "Great answer correct!"
|
||||
else
|
||||
warn "Smoke failed, answer is '$MAGIC'"
|
||||
fi
|
||||
if [[ $RECOMPILE_ONLY = true || $SKIP_SMOKE = true || $RESTART_SERVICES = false ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
message_split
|
||||
message "Creating test database"
|
||||
mariadb -e "create database if not exists test;"
|
||||
message "Selecting magic numbers"
|
||||
MAGIC=$(mariadb -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql)
|
||||
if [[ $MAGIC == '42' ]]; then
|
||||
message "Great answer correct!"
|
||||
else
|
||||
warn "Smoke failed, answer is '$MAGIC'"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -692,23 +738,22 @@ fi
|
||||
construct_cmake_flags
|
||||
init_submodules
|
||||
|
||||
if [[ $BUILD_PACKAGES = false ]]; then
|
||||
stop_service
|
||||
|
||||
if [[ $NO_CLEAN = false ]]; then
|
||||
clean_old_installation
|
||||
fi
|
||||
build_binary
|
||||
run_unit_tests
|
||||
run_microbenchmarks_tests
|
||||
install
|
||||
if [[ $RESTART_SERVICES = true ]]; then
|
||||
start_service
|
||||
smoke
|
||||
fi
|
||||
else
|
||||
if [[ $BUILD_PACKAGES = true ]]; then
|
||||
modify_packaging
|
||||
build_package
|
||||
message_splitted "PACKAGES BUILD FINISHED"
|
||||
|
||||
return 0
|
||||
fi
|
||||
|
||||
stop_service
|
||||
clean_old_installation
|
||||
build_binary
|
||||
install_binary
|
||||
run_unit_tests
|
||||
run_microbenchmarks_tests
|
||||
install
|
||||
start_service
|
||||
smoke
|
||||
|
||||
message_splitted "FINISHED"
|
||||
|
||||
Reference in New Issue
Block a user