1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00

chore(ci): add retries for the installations, fix colors for ci, clear apt cache, fix deps missing on CI stage by moving step to the beginnig

This commit is contained in:
Leonid Fedorov
2025-06-02 13:18:38 +00:00
committed by Leonid Fedorov
parent b2c0106c93
commit 7c7026253a
5 changed files with 430 additions and 385 deletions

View File

@ -67,6 +67,51 @@ source $(optparse.build)
message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
install_deps() {
if [[ $INSTALL_DEPS = false ]]; then
return
fi
message_split
prereq=""
RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \
snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \
rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \
cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
graphviz devscripts ccache equivs eatmydata curl"
if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \
${RPM_BUILD_DEPS}"
elif
[[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]
then
command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \
dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}"
elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then
prereq="apt-get clean && rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin"
command="${DEB_BUILD_DEPS}"
else
echo "Unsupported OS: $OS"
exit 17
fi
if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
command="${command} lto-disabled-list"
fi
eval "$prereq"
message "Installing dependencies for $OS"
retry_eval 5 "$command"
}
install_deps
cd $COLUMSNTORE_SOURCE_PATH
COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch"
@ -103,40 +148,6 @@ disable_git_restore_frozen_revision() {
DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot"
install_deps() {
message_split
RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \
snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \
rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \
cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
graphviz devscripts ccache equivs eatmydata curl"
if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \
${RPM_BUILD_DEPS}"
elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]; then
command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \
dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}"
elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then
command="${DEB_BUILD_DEPS}"
else
echo "Unsupported OS: $OS"
exit 17
fi
message "Installing dependencies for $OS"
eval "$command"
}
stop_service() {
message_split
message "Stopping MariaDB services"
@ -206,10 +217,9 @@ modify_packaging() {
#disable LTO for 22.04 for now
if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
apt install -y lto-disabled-list &&
for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do
echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list
done &&
for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do
echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list
done &&
grep mariadb /usr/share/lto-disabled-list/lto-disabled-list
fi
@ -269,7 +279,7 @@ construct_cmake_flags() {
if [[ $MAINTAINER_MODE = true ]]; then
MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES)
message "Columnstore mainteiner mode on"
message "Columnstore maintainer mode on"
else
warn "Maintainer mode is disabled, be careful, alien"
fi
@ -665,10 +675,6 @@ smoke() {
fi
}
if [[ $INSTALL_DEPS = true ]]; then
install_deps
fi
if [[ $DO_NOT_FREEZE_REVISION = false ]]; then
disable_git_restore_frozen_revision
fi

View File

@ -18,8 +18,8 @@ source $(optparse.build)
echo "Arguments received: $@"
if [ "$EUID" -ne 0 ]; then
error "Please run script as root"
exit 1
error "Please run script as root"
exit 1
fi
if [[ -z "${OS:-}" || -z "${ARCH:-}" ]]; then
@ -27,14 +27,13 @@ if [[ -z "${OS:-}" || -z "${ARCH:-}" ]]; then
exit 1
fi
pkg_format="deb"
if [[ "$OS" == *"rocky"* ]]; then
pkg_format="rpm"
pkg_format="rpm"
fi
if [[ "$ARCH" == "arm64" ]]; then
export CC=gcc #TODO: what it is for?
export CC=gcc #TODO: what it is for?
fi
on_exit() {
@ -52,24 +51,23 @@ install_deps() {
cd "$COLUMNSTORE_SOURCE_PATH"/cmapi
if [[ "$OS" == "rockylinux:9" ]]; then
dnf install -q -y libxcrypt-compat yum-utils
dnf config-manager --set-enabled devel && dnf update -q -y #to make redhat-lsb-core available for rocky 9
retry_eval 5 "dnf install -q -y libxcrypt-compat yum-utils"
retry_eval 5 "dnf config-manager --set-enabled devel && dnf update -q -y" #to make redhat-lsb-core available for rocky 9
fi
if [[ "$pkg_format" == "rpm" ]]; then
dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive
retry_eval 5 "dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive"
else
apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release
retry_eval 5 "apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release"
fi
if [ "$ARCH" == "amd64" ]; then
PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst"
PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst"
elif [ "$ARCH" == "arm64" ]; then
PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst"
PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst"
else
echo "Unsupported architecture: $ARCH"
exit 1
echo "Unsupported architecture: $ARCH"
exit 1
fi
rm -rf python pp

View File

@ -58,10 +58,10 @@ else
fi
if [[ $(compgen -G "./${RESULT}/*.rpm") ]]; then
dnf install -q -y createrepo
retry_eval 5 "dnf install -q -y createrepo"
createrepo "./${RESULT}"
else
apt update && apt install -y dpkg-dev
retry_eval 5 "apt update && apt install -y dpkg-dev"
dpkg-scanpackages "${RESULT}" | gzip >"./${RESULT}/Packages.gz"
fi

View File

@ -11,10 +11,10 @@ source "$SCRIPT_LOCATION"/utils.sh
echo "Arguments received: $@"
optparse.define short=c long=container-name desc="Name of the Docker container to run tests in" variable=CONTAINER_NAME
optparse.define short=i long=docker-image desc="Docker image name to start container from" variable=DOCKER_IMAGE
optparse.define short=r long=result-path desc="Name suffix used in core dump file path" variable=RESULT
optparse.define short=s long=do-setup desc="Run setup-repo.sh inside the container" variable=DO_SETUP
optparse.define short=u long=packages-url desc="Packages url" variable=PACKAGES_URL
optparse.define short=i long=docker-image desc="Docker image name to start container from" variable=DOCKER_IMAGE
optparse.define short=r long=result-path desc="Name suffix used in core dump file path" variable=RESULT
optparse.define short=s long=do-setup desc="Run setup-repo.sh inside the container" variable=DO_SETUP
optparse.define short=u long=packages-url desc="Packages url" variable=PACKAGES_URL
source $(optparse.build)
if [[ "$EUID" -ne 0 ]]; then
@ -23,8 +23,8 @@ if [[ "$EUID" -ne 0 ]]; then
fi
if [[ -z "${CONTAINER_NAME:-}" || -z "${DOCKER_IMAGE:-}" || -z "${RESULT:-}" || -z "${DO_SETUP:-}" || -z "${PACKAGES_URL:-}" ]]; then
echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true"
exit 1
echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true"
exit 1
fi
start_container() {
@ -48,21 +48,21 @@ start_container() {
)
if [[ "$CONTAINER_NAME" == *smoke* ]]; then
docker_run_args+=( --memory 3g )
docker_run_args+=(--memory 3g)
elif [[ "$CONTAINER_NAME" == *mtr* ]]; then
docker_run_args+=( --shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH" )
docker_run_args+=(--shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH")
elif [[ "$CONTAINER_NAME" == *cmapi* ]]; then
docker_run_args+=( --env PYTHONPATH="${PYTHONPATH}" )
docker_run_args+=(--env PYTHONPATH="${PYTHONPATH}")
elif [[ "$CONTAINER_NAME" == *upgrade* ]]; then
docker_run_args+=( --env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro )
docker_run_args+=(--env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro)
elif [[ "$CONTAINER_NAME" == *regression* ]]; then
docker_run_args+=( --shm-size=500m --memory 12g )
docker_run_args+=(--shm-size=500m --memory 12g)
else
echo "Unknown container type: $CONTAINER_NAME"
exit 1
fi
docker_run_args+=( "$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target )
docker_run_args+=("$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target)
docker run "${docker_run_args[@]}"
sleep 5
@ -85,31 +85,31 @@ fi
#list_cgroups
echo "Docker CGroups opts here"
ls -al /sys/fs/cgroup/cgroup.controllers || true
ls -al /sys/fs/cgroup/ || true
ls -al /sys/fs/cgroup/ || true
ls -al /sys/fs/cgroup/memory || true
execInnerDocker 'echo Inner Docker CGroups opts here' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/cgroup.controllers || true' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/ || true' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/memory || true' "$CONTAINER_NAME"
execInnerDocker 'echo Inner Docker CGroups opts here' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/cgroup.controllers || true' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/ || true' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/memory || true' "$CONTAINER_NAME"
# Prepare core dump directory inside container
execInnerDocker 'mkdir -p core && chmod 777 core' "$CONTAINER_NAME"
docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/. "$CONTAINER_NAME":/
docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh "$CONTAINER_NAME":/
docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh "$CONTAINER_NAME":/
execInnerDocker 'mkdir -p core && chmod 777 core' "$CONTAINER_NAME"
docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/. "$CONTAINER_NAME":/
docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh "$CONTAINER_NAME":/
docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh "$CONTAINER_NAME":/
if [[ "$DO_SETUP" == "true" ]]; then
execInnerDocker '/setup-repo.sh' "$CONTAINER_NAME"
execInnerDocker '/setup-repo.sh' "$CONTAINER_NAME"
fi
# install deps
if [[ "$RESULT" == *rocky* ]]; then
execInnerDocker 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' "$CONTAINER_NAME"
# sudo bypass
execInnerDocker "printf '%s\n' '#!/bin/sh' 'exec \"\$@\"' > /usr/bin/sudo && chmod +x /usr/bin/sudo" "$CONTAINER_NAME"
execInnerDockerWithRetry 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' "$CONTAINER_NAME"
else
execInnerDocker 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME"
execInnerDockerWithRetry 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME"
fi
# Configure core dump naming pattern
@ -118,9 +118,9 @@ execInnerDocker 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"
#Install columnstore in container
echo "Installing columnstore..."
if [[ "$RESULT" == *rocky* ]]; then
execInnerDocker 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME"
execInnerDockerWithRetry 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME"
else
execInnerDocker 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME"
execInnerDockerWithRetry 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME"
fi
sleep 5

View File

@ -1,199 +1,199 @@
#!/bin/bash
if [[ -n "$TERM" && "$TERM" != "dumb" && $(command -v tput) ]]; then
TPUT_AVAILABLE=true
TPUT_AVAILABLE=true
else
TPUT_AVAILABLE=false
TPUT_AVAILABLE=false
fi
if [[ $TPUT_AVAILABLE == true ]]; then
color_normal=$(tput sgr0)
color_bold=$(tput bold)
color_red="$color_bold$(tput setaf 1)"
color_green="$color_bold$(tput setaf 2)"
color_fawn=$(tput setaf 3); color_beige="$color_fawn"
color_yellow="$color_bold$color_fawn"
color_darkblue=$(tput setaf 4)
color_normal=$(tput sgr0)
color_bold=$(tput bold)
color_red="$color_bold$(tput setaf 1)"
color_green="$color_bold$(tput setaf 2)"
color_fawn=$(tput setaf 3)
color_beige="$color_fawn"
color_yellow="$color_bold$color_fawn"
color_darkblue=$(tput setaf 4)
color_blue="$color_bold$color_darkblue"
color_purple=$(tput setaf 5); color_magenta="$color_purple"
color_pink="$color_bold$color_purple"
color_darkcyan=$(tput setaf 6)
color_cyan="$color_bold$color_darkcyan"
color_gray=$(tput setaf 7)
color_darkgray="$color_bold"$(tput setaf 0)
color_white="$color_bold$color_gray"
color_blue="$color_bold$color_darkblue"
color_purple=$(tput setaf 5)
color_magenta="$color_purple"
color_pink="$color_bold$color_purple"
color_darkcyan=$(tput setaf 6)
color_cyan="$color_bold$color_darkcyan"
color_gray=$(tput setaf 7)
color_darkgray="$color_bold"$(tput setaf 0)
color_white="$color_bold$color_gray"
if [[ $(tput colors) == '256' ]]; then
color_red=$(tput setaf 196)
color_yellow=$(tput setaf 228)
color_cyan=$(tput setaf 87)
color_green=$(tput setaf 156)
color_darkgray=$(tput setaf 59)
fi
if [[ $(tput colors) == '256' ]]; then
color_red=$(tput setaf 196)
color_yellow=$(tput setaf 228)
color_cyan=$(tput setaf 87)
color_green=$(tput setaf 156)
color_darkgray=$(tput setaf 59)
fi
else
# Basic attributes
color_normal="\e[0m" # sgr0
color_bold="\e[1m" # bold
# Standard 8-color palette
color_red="\e[1;31m" # bold + setaf 1
color_green="\e[1;32m" # bold + setaf 2
color_fawn="\e[33m" # setaf 3
color_beige="$color_fawn" # alias
color_yellow="\e[1;33m" # bold + setaf 3
color_darkblue="\e[34m" # setaf 4
color_blue="\e[1;34m" # bold + setaf 4
color_purple="\e[35m" # setaf 5
color_magenta="$color_purple" # alias
color_pink="\e[1;35m" # bold + setaf 5
color_darkcyan="\e[36m" # setaf 6
color_cyan="\e[1;36m" # bold + setaf 6
color_gray="\e[37m" # setaf 7
color_darkgray="\e[1;30m" # bold + setaf 0
color_white="\e[1;37m" # bold + setaf 7
if [ "$TERM" = "xterm-256color" ] || [ "$COLORTERM" = "truecolor" ] || [ "$COLORTERM" = "24bit" ]; then
# Only set 256-color codes if actually in a 256-color terminal
color_red="\e[91m" # bright red
color_yellow="\e[93m" # light yellow
color_cyan="\e[96m" # bright cyan
color_green="\e[92m" # light green
color_darkgray="\e[90m" # dark gray
fi
fi
message()
{
message() {
echo -e $color_cyan$@$color_normal
}
warn() {
echo -e $color_yellow$@$color_normal
}
error() {
echo -e $color_red$@$color_normal
}
message_split() {
echo -e $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal
}
message_splitted() {
message_split
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_green$@$color_normal
else
echo "$@"
fi
message_split
}
colorify_array() {
PROMT=""
for a in "$@"; do
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_cyan$@$color_normal
i=$((((i + 1) % (123 - 106)) + 106))
if [[ $(tput colors) == '256' ]]; then
PROMT="$PROMT $(tput setaf $i)$a$color_normal"
else
PROMT="$PROMT $a"
fi
else
echo "$@"
PROMT="$PROMT $a"
fi
done
echo $PROMT
}
warn()
{
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_yellow$@$color_normal
else
echo "$@"
newline_array() {
PROMT=""
for a in "$@"; do
PROMT="$PROMT$a\n"
done
echo -e $PROMT
}
function spinner() {
freq=${1:-10}
points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷)
colored_points=($(colorify_array ${points[@]}))
len=${#points[@]}
point_num=0
line_num=0
while read data; do
line_num=$((line_num + 1))
if [[ $((line_num % freq)) = 0 ]]; then
point_num=$(((point_num + 1) % len))
if [[ $TPUT_AVAILABLE == true ]]; then
echo -ne "\r${colored_points[point_num]}"
else
echo -ne "\r${points[point_num]}"
fi
fi
done
echo
}
error()
{
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_red$@$color_normal
else
echo "$@"
fi
function onelinearizator() {
while read data; do
echo -ne "\r\e[K$data"
done
echo
}
message_split()
{
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal
else
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
fi
}
message_splitted()
{
message_split
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_green$@$color_normal
else
echo "$@"
fi
message_split
}
colorify_array()
{
PROMT=""
for a in "$@"
do
if [[ $TPUT_AVAILABLE == true ]]; then
i=$((((i+1) % (123-106)) + 106))
if [[ $(tput colors) == '256' ]]; then
PROMT="$PROMT $(tput setaf $i)$a$color_normal"
else
PROMT="$PROMT $a"
fi
else
PROMT="$PROMT $a"
fi
done
echo $PROMT
}
newline_array()
{
PROMT=""
for a in "$@"
do
PROMT="$PROMT$a\n"
done
echo -e $PROMT
}
function spinner
{
freq=${1:-10}
points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷)
colored_points=($(colorify_array ${points[@]}))
len=${#points[@]}
point_num=0
line_num=0
while read data; do
line_num=$((line_num+1))
if [[ $((line_num % freq)) = 0 ]]; then
point_num=$(((point_num + 1) % len ))
if [[ $TPUT_AVAILABLE == true ]]; then
echo -ne "\r${colored_points[point_num]}"
else
echo -ne "\r${points[point_num]}"
fi
fi
done;
echo
}
function onelinearizator
{
while read data; do
echo -ne "\r\033[K$data"
done;
echo
}
detect_distro()
{
if [ -f /etc/os-release ]; then
. /etc/os-release
export OS=$NAME
export OS_VERSION=$VERSION_ID
elif type lsb_release >/dev/null 2>&1; then
# linuxbase.org
export OS=$(lsb_release -si)
export OS_VERSION=$(lsb_release -sr)
elif [ -f /etc/lsb-release ]; then
# For some versions of Debian/Ubuntu without lsb_release command
. /etc/lsb-release
export OS=$DISTRIB_ID
OS_VERSION=$DISTRIB_RELEASE
elif [ -f /etc/debian_version ]; then
# Older Debian/Ubuntu/etc.
OS=Debian
OS_VERSION=$(cat /etc/debian_version)
else
# Fall back to uname, e.g. "Linux <version>", also works for BSD, etc.
OS=$(uname -s)
OS_VERSION=$(uname -r)
fi
OS=$(echo $OS | cut -f 1 -d " " | tr '[:upper:]' '[:lower:]')":"$OS_VERSION
message "Detected $color_yellow$OS $OS_VERSION$color_normal"
detect_distro() {
if [ -f /etc/os-release ]; then
. /etc/os-release
export OS=$NAME
export OS_VERSION=$VERSION_ID
elif type lsb_release >/dev/null 2>&1; then
# linuxbase.org
export OS=$(lsb_release -si)
export OS_VERSION=$(lsb_release -sr)
elif [ -f /etc/lsb-release ]; then
# For some versions of Debian/Ubuntu without lsb_release command
. /etc/lsb-release
export OS=$DISTRIB_ID
OS_VERSION=$DISTRIB_RELEASE
elif [ -f /etc/debian_version ]; then
# Older Debian/Ubuntu/etc.
OS=Debian
OS_VERSION=$(cat /etc/debian_version)
else
# Fall back to uname, e.g. "Linux <version>", also works for BSD, etc.
OS=$(uname -s)
OS_VERSION=$(uname -r)
fi
OS=$(echo $OS | cut -f 1 -d " " | tr '[:upper:]' '[:lower:]')":"$OS_VERSION
message "Detected $color_yellow$OS $OS_VERSION$color_normal"
}
menuStr=""
function hideCursor(){
printf "\033[?25l"
function hideCursor() {
printf "\e[?25l"
# capture CTRL+C so cursor can be reset
trap "showCursor && exit 0" 2
}
function showCursor(){
printf "\033[?25h"
function showCursor() {
printf "\e[?25h"
}
function clearLastMenu(){
function clearLastMenu() {
local msgLineCount=$(printf "$menuStr" | wc -l)
# moves the curser up N lines so the output overwrites it
echo -en "\033[${msgLineCount}A"
echo -en "\e[${msgLineCount}A"
# clear to end of screen to ensure there's no text left behind from previous input
[ $1 ] && tput ed
}
function renderMenu(){
function renderMenu() {
local start=0
local selector=""
local instruction="$1"
@ -205,8 +205,8 @@ function renderMenu(){
# Get the longest item from the list so that we know how many spaces to add
# to ensure there's no overlap from longer items when a list is scrolling up or down.
for (( i=0; i<$itemsLength; i++ )); do
if (( ${#menuItems[i]} > longest )); then
for ((i = 0; i < $itemsLength; i++)); do
if ((${#menuItems[i]} > longest)); then
longest=${#menuItems[i]}
fi
done
@ -216,12 +216,12 @@ function renderMenu(){
listLength=$3
if [ $selectedIndex -ge $listLength ]; then
start=$(($selectedIndex+1-$listLength))
listLength=$(($selectedIndex+1))
start=$(($selectedIndex + 1 - $listLength))
listLength=$(($selectedIndex + 1))
fi
fi
for (( i=$start; i<$listLength; i++ )); do
for ((i = $start; i < $listLength; i++)); do
local currItem="${menuItems[i]}"
currItemLength=${#currItem}
@ -246,9 +246,9 @@ function renderMenu(){
printf "${menuStr}"
}
function getChoice(){
local KEY__ARROW_UP=$(echo -e "\033[A")
local KEY__ARROW_DOWN=$(echo -e "\033[B")
function getChoice() {
local KEY__ARROW_UP=$(echo -e "\e[A")
local KEY__ARROW_DOWN=$(echo -e "\e[B")
local KEY__ENTER=$(echo -e "\n")
local captureInput=true
local displayHelp=false
@ -261,55 +261,55 @@ function getChoice(){
key="$1"
case $key in
-h|--help)
displayHelp=true
shift
;;
-i|--index)
selectedIndex=$2
shift 2
;;
-m|--max)
maxViewable=$2
shift 2
;;
-o|--options)
menuItems=$2[@]
menuItems=("${!menuItems}")
shift 2
;;
-q|--query)
instruction="$2"
shift 2
;;
*)
remainingArgs+=("$1")
shift
;;
-h | --help)
displayHelp=true
shift
;;
-i | --index)
selectedIndex=$2
shift 2
;;
-m | --max)
maxViewable=$2
shift 2
;;
-o | --options)
menuItems=$2[@]
menuItems=("${!menuItems}")
shift 2
;;
-q | --query)
instruction="$2"
shift 2
;;
*)
remainingArgs+=("$1")
shift
;;
esac
done
# just display help
if $displayHelp; then
echo;
echo
echo "Usage: getChoice [OPTION]..."
echo "Renders a keyboard navigable menu with a visual indicator of what's selected."
echo;
echo
echo " -h, --help Displays this message"
echo " -i, --index The initially selected index for the options"
echo " -m, --max Limit how many options are displayed"
echo " -o, --options An Array of options for a User to choose from"
echo " -q, --query Question or statement presented to the User"
echo;
echo
echo "Example:"
echo " foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")"
echo;
echo
echo " getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4"
echo " printf \"\\n First choice is '\${selectedChoice}'\\n\""
echo;
echo
echo " getChoice -q \"Select another option in case the first isn't available\" -o foodOptions"
echo " printf \"\\n Second choice is '\${selectedChoice}'\\n\""
echo;
echo
return 0
fi
@ -327,119 +327,104 @@ function getChoice(){
hideCursor
while $captureInput; do
read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters.
read -rsn3 key # `3` captures the escape (\e'), bracket ([), & type (A) characters.
case "$key" in
"$KEY__ARROW_UP")
selectedIndex=$((selectedIndex-1))
(( $selectedIndex < 0 )) && selectedIndex=$((itemsLength-1))
"$KEY__ARROW_UP")
selectedIndex=$((selectedIndex - 1))
(($selectedIndex < 0)) && selectedIndex=$((itemsLength - 1))
renderMenu "$instruction" $selectedIndex $maxViewable true
;;
renderMenu "$instruction" $selectedIndex $maxViewable true
;;
"$KEY__ARROW_DOWN")
selectedIndex=$((selectedIndex+1))
(( $selectedIndex == $itemsLength )) && selectedIndex=0
"$KEY__ARROW_DOWN")
selectedIndex=$((selectedIndex + 1))
(($selectedIndex == $itemsLength)) && selectedIndex=0
renderMenu "$instruction" $selectedIndex $maxViewable true
;;
renderMenu "$instruction" $selectedIndex $maxViewable true
;;
"$KEY__ENTER")
clearLastMenu true
showCursor
captureInput=false
;;
"$KEY__ENTER")
clearLastMenu true
showCursor
captureInput=false
;;
esac
done
}
function optparse.throw_error(){
function optparse.throw_error() {
local message="$1"
error "OPTPARSE: ERROR: $message"
exit 1
error "OPTPARSE: ERROR: $message"
exit 1
}
# -----------------------------------------------------------------------------------------------------------------------------
function optparse.define(){
if [ $# -lt 3 ]; then
optparse.throw_error "optparse.define <short> <long> <variable> [<desc>] [<default>] [<value>]"
fi
for option_id in $( seq 1 $# ) ; do
local option="$( eval "echo \$$option_id")"
local key="$( echo $option | awk -F "=" '{print $1}' )";
local value="$( echo $option | awk -F "=" '{print $2}' )";
function optparse.define() {
if [ $# -lt 3 ]; then
optparse.throw_error "optparse.define <short> <long> <variable> [<desc>] [<default>] [<value>]"
fi
for option_id in $(seq 1 $#); do
local option="$(eval "echo \$$option_id")"
local key="$(echo $option | awk -F "=" '{print $1}')"
local value="$(echo $option | awk -F "=" '{print $2}')"
#essentials: shortname, longname, description
if [ "$key" = "short" ]; then
local shortname="$value"
if [ ${#shortname} -ne 1 ]; then
optparse.throw_error "short name expected to be one character long"
fi
local short="-${shortname}"
elif [ "$key" = "long" ]; then
local longname="$value"
if [ ${#longname} -lt 2 ]; then
optparse.throw_error "long name expected to be atleast one character long"
fi
local long="--${longname}"
elif [ "$key" = "desc" ]; then
local desc="$value"
elif [ "$key" = "default" ]; then
local default="$value"
elif [ "$key" = "variable" ]; then
local variable="$value"
elif [ "$key" = "value" ]; then
local val="$value"
fi
done
if [ "$variable" = "" ]; then
optparse.throw_error "You must give a variable for option: ($short/$long)"
fi
if [ "$val" = "" ]; then
val="\$OPTARG"
fi
# build OPTIONS and help
optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")"
if [ "$default" != "" ]; then
optparse_usage="${optparse_usage} [default:$default]"
fi
optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;"
if [ "$default" != "" ]; then
optparse_defaults="${optparse_defaults}#NL${variable}=${default}"
fi
optparse_arguments_string="${optparse_arguments_string}${shortname}"
if [ "$val" = "\$OPTARG" ]; then
optparse_arguments_string="${optparse_arguments_string}:"
fi
optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;"
}
function execInnerDocker() {
local cmd_str="$1"
local img="$2"
local flags="${3:-}"
docker exec $flags -t "$img" bash -c "$cmd_str"
local dockerCommandExitCode=$?
if [[ $dockerCommandExitCode -ne 0 ]]; then
echo "Command \"$cmd_str\" failed in container \"$img\""
exit $dockerCommandExitCode
#essentials: shortname, longname, description
if [ "$key" = "short" ]; then
local shortname="$value"
if [ ${#shortname} -ne 1 ]; then
optparse.throw_error "short name expected to be one character long"
fi
local short="-${shortname}"
elif [ "$key" = "long" ]; then
local longname="$value"
if [ ${#longname} -lt 2 ]; then
optparse.throw_error "long name expected to be atleast one character long"
fi
local long="--${longname}"
elif [ "$key" = "desc" ]; then
local desc="$value"
elif [ "$key" = "default" ]; then
local default="$value"
elif [ "$key" = "variable" ]; then
local variable="$value"
elif [ "$key" = "value" ]; then
local val="$value"
fi
done
if [ "$variable" = "" ]; then
optparse.throw_error "You must give a variable for option: ($short/$long)"
fi
if [ "$val" = "" ]; then
val="\$OPTARG"
fi
# build OPTIONS and help
optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")"
if [ "$default" != "" ]; then
optparse_usage="${optparse_usage} [default:$default]"
fi
optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;"
if [ "$default" != "" ]; then
optparse_defaults="${optparse_defaults}#NL${variable}=${default}"
fi
optparse_arguments_string="${optparse_arguments_string}${shortname}"
if [ "$val" = "\$OPTARG" ]; then
optparse_arguments_string="${optparse_arguments_string}:"
fi
optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;"
}
# -----------------------------------------------------------------------------------------------------------------------------
function optparse.build(){
local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")"
function optparse.build() {
local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")"
# Building getopts header here
# Building getopts header here
# Function usage
cat << EOF > $build_file
# Function usage
cat <<EOF >$build_file
function usage(){
cat << XXX
usage: \$0 [OPTIONS]
@ -487,19 +472,75 @@ done
rm $build_file
EOF
local -A o=( ['#NL']='\n' ['#TB']='\t' )
local -A o=(['#NL']='\n' ['#TB']='\t')
for i in "${!o[@]}"; do
sed -i "s/${i}/${o[$i]}/g" $build_file
done
for i in "${!o[@]}"; do
sed -i "s/${i}/${o[$i]}/g" $build_file
done
# Unset global variables
unset optparse_usage
unset optparse_process
unset optparse_arguments_string
unset optparse_defaults
unset optparse_contractions
# Unset global variables
unset optparse_usage
unset optparse_process
unset optparse_arguments_string
unset optparse_defaults
unset optparse_contractions
# Return file name to parent
echo "$build_file"
# Return file name to parent
echo "$build_file"
}
function retry_eval() {
local max_retries=$1
local command=$2
local attempt=1
local initial_delay=1
while [ "$attempt" -le "$max_retries" ]; do
message_split
message "Attempt $attempt of $max_retries: $command"
if eval "$command"; then
message "Command '$command' done"
message_split
return 0
fi
if [ "$attempt" -lt "$max_retries" ]; then
delay=$((initial_delay * 2 ** (attempt - 1)))
message "Retrying command '$command' in $delay seconds..."
message_split
sleep "$delay"
fi
((attempt++))
done
error "Max retries reached for command: $command"
message_split
exit 13
}
function execInnerDocker() {
local cmd_str="$1"
local img="$2"
local flags="${3:-}"
docker exec $flags -t "$img" bash -c "$cmd_str"
local dockerCommandExitCode=$?
if [[ $dockerCommandExitCode -ne 0 ]]; then
error "Command \"$cmd_str\" failed in container \"$img\""
exit $dockerCommandExitCode
fi
}
function execInnerDockerWithRetry() {
local cmd_str="$1"
local img="$2"
local flags="${3:-}"
docker exec $flags -t "$img" bash -c " $(declare -f retry_eval); retry_eval 5 '$cmd_str'"
local dockerCommandExitCode=$?
if [[ $dockerCommandExitCode -ne 0 ]]; then
error "Command \"$cmd_str\" failed in container \"$img\""
exit $dockerCommandExitCode
fi
}