diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 722e69679..0a1b823e5 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -67,6 +67,51 @@ source $(optparse.build) message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal" +install_deps() { + if [[ $INSTALL_DEPS = false ]]; then + return + fi + message_split + prereq="" + RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \ + snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \ + rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \ + cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel" + + DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \ + bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ + libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \ + libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \ + graphviz devscripts ccache equivs eatmydata curl" + + if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then + command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \ + dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \ + ${RPM_BUILD_DEPS}" + elif + [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]] + then + command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \ + dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}" + + elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then + prereq="apt-get clean && rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin" + command="${DEB_BUILD_DEPS}" + else + echo "Unsupported OS: $OS" + exit 17 + fi + + if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then + command="${command} lto-disabled-list" + fi + eval "$prereq" + message "Installing dependencies for $OS" + retry_eval 5 "$command" +} + +install_deps + cd $COLUMSNTORE_SOURCE_PATH COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD) message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch" @@ -103,40 +148,6 @@ disable_git_restore_frozen_revision() { DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot" -install_deps() { - message_split - - RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \ - snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \ - rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \ - cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel" - - DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \ - bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ - libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \ - libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \ - graphviz devscripts ccache equivs eatmydata curl" - - if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then - command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \ - dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \ - ${RPM_BUILD_DEPS}" - - elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]; then - command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \ - dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}" - - elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then - command="${DEB_BUILD_DEPS}" - - else - echo "Unsupported OS: $OS" - exit 17 - fi - - message "Installing dependencies for $OS" - eval "$command" -} stop_service() { message_split message "Stopping MariaDB services" @@ -206,10 +217,9 @@ modify_packaging() { #disable LTO for 22.04 for now if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then - apt install -y lto-disabled-list && - for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do - echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list - done && + for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do + echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list + done && grep mariadb /usr/share/lto-disabled-list/lto-disabled-list fi @@ -269,7 +279,7 @@ construct_cmake_flags() { if [[ $MAINTAINER_MODE = true ]]; then MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES) - message "Columnstore mainteiner mode on" + message "Columnstore maintainer mode on" else warn "Maintainer mode is disabled, be careful, alien" fi @@ -665,10 +675,6 @@ smoke() { fi } -if [[ $INSTALL_DEPS = true ]]; then - install_deps -fi - if [[ $DO_NOT_FREEZE_REVISION = false ]]; then disable_git_restore_frozen_revision fi diff --git a/build/build_cmapi.sh b/build/build_cmapi.sh index d4f50cf46..241b3fc35 100755 --- a/build/build_cmapi.sh +++ b/build/build_cmapi.sh @@ -18,8 +18,8 @@ source $(optparse.build) echo "Arguments received: $@" if [ "$EUID" -ne 0 ]; then - error "Please run script as root" - exit 1 + error "Please run script as root" + exit 1 fi if [[ -z "${OS:-}" || -z "${ARCH:-}" ]]; then @@ -27,14 +27,13 @@ if [[ -z "${OS:-}" || -z "${ARCH:-}" ]]; then exit 1 fi - pkg_format="deb" if [[ "$OS" == *"rocky"* ]]; then - pkg_format="rpm" + pkg_format="rpm" fi if [[ "$ARCH" == "arm64" ]]; then - export CC=gcc #TODO: what it is for? + export CC=gcc #TODO: what it is for? fi on_exit() { @@ -52,24 +51,23 @@ install_deps() { cd "$COLUMNSTORE_SOURCE_PATH"/cmapi if [[ "$OS" == "rockylinux:9" ]]; then - dnf install -q -y libxcrypt-compat yum-utils - dnf config-manager --set-enabled devel && dnf update -q -y #to make redhat-lsb-core available for rocky 9 + retry_eval 5 "dnf install -q -y libxcrypt-compat yum-utils" + retry_eval 5 "dnf config-manager --set-enabled devel && dnf update -q -y" #to make redhat-lsb-core available for rocky 9 fi if [[ "$pkg_format" == "rpm" ]]; then - dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive + retry_eval 5 "dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive" else - apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release + retry_eval 5 "apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release" fi - if [ "$ARCH" == "amd64" ]; then - PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst" + PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst" elif [ "$ARCH" == "arm64" ]; then - PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst" + PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst" else - echo "Unsupported architecture: $ARCH" - exit 1 + echo "Unsupported architecture: $ARCH" + exit 1 fi rm -rf python pp diff --git a/build/createrepo.sh b/build/createrepo.sh index c326d2b5a..7bc2673c8 100755 --- a/build/createrepo.sh +++ b/build/createrepo.sh @@ -58,10 +58,10 @@ else fi if [[ $(compgen -G "./${RESULT}/*.rpm") ]]; then - dnf install -q -y createrepo + retry_eval 5 "dnf install -q -y createrepo" createrepo "./${RESULT}" else - apt update && apt install -y dpkg-dev + retry_eval 5 "apt update && apt install -y dpkg-dev" dpkg-scanpackages "${RESULT}" | gzip >"./${RESULT}/Packages.gz" fi diff --git a/build/prepare_test_stage.sh b/build/prepare_test_stage.sh index e48f18d3a..6bea2edd9 100755 --- a/build/prepare_test_stage.sh +++ b/build/prepare_test_stage.sh @@ -11,10 +11,10 @@ source "$SCRIPT_LOCATION"/utils.sh echo "Arguments received: $@" optparse.define short=c long=container-name desc="Name of the Docker container to run tests in" variable=CONTAINER_NAME -optparse.define short=i long=docker-image desc="Docker image name to start container from" variable=DOCKER_IMAGE -optparse.define short=r long=result-path desc="Name suffix used in core dump file path" variable=RESULT -optparse.define short=s long=do-setup desc="Run setup-repo.sh inside the container" variable=DO_SETUP -optparse.define short=u long=packages-url desc="Packages url" variable=PACKAGES_URL +optparse.define short=i long=docker-image desc="Docker image name to start container from" variable=DOCKER_IMAGE +optparse.define short=r long=result-path desc="Name suffix used in core dump file path" variable=RESULT +optparse.define short=s long=do-setup desc="Run setup-repo.sh inside the container" variable=DO_SETUP +optparse.define short=u long=packages-url desc="Packages url" variable=PACKAGES_URL source $(optparse.build) if [[ "$EUID" -ne 0 ]]; then @@ -23,8 +23,8 @@ if [[ "$EUID" -ne 0 ]]; then fi if [[ -z "${CONTAINER_NAME:-}" || -z "${DOCKER_IMAGE:-}" || -z "${RESULT:-}" || -z "${DO_SETUP:-}" || -z "${PACKAGES_URL:-}" ]]; then - echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true" - exit 1 + echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true" + exit 1 fi start_container() { @@ -48,21 +48,21 @@ start_container() { ) if [[ "$CONTAINER_NAME" == *smoke* ]]; then - docker_run_args+=( --memory 3g ) + docker_run_args+=(--memory 3g) elif [[ "$CONTAINER_NAME" == *mtr* ]]; then - docker_run_args+=( --shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH" ) + docker_run_args+=(--shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH") elif [[ "$CONTAINER_NAME" == *cmapi* ]]; then - docker_run_args+=( --env PYTHONPATH="${PYTHONPATH}" ) + docker_run_args+=(--env PYTHONPATH="${PYTHONPATH}") elif [[ "$CONTAINER_NAME" == *upgrade* ]]; then - docker_run_args+=( --env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro ) + docker_run_args+=(--env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro) elif [[ "$CONTAINER_NAME" == *regression* ]]; then - docker_run_args+=( --shm-size=500m --memory 12g ) + docker_run_args+=(--shm-size=500m --memory 12g) else echo "Unknown container type: $CONTAINER_NAME" exit 1 fi - docker_run_args+=( "$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target ) + docker_run_args+=("$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target) docker run "${docker_run_args[@]}" sleep 5 @@ -85,31 +85,31 @@ fi #list_cgroups echo "Docker CGroups opts here" ls -al /sys/fs/cgroup/cgroup.controllers || true -ls -al /sys/fs/cgroup/ || true +ls -al /sys/fs/cgroup/ || true ls -al /sys/fs/cgroup/memory || true -execInnerDocker 'echo Inner Docker CGroups opts here' "$CONTAINER_NAME" -execInnerDocker 'ls -al /sys/fs/cgroup/cgroup.controllers || true' "$CONTAINER_NAME" -execInnerDocker 'ls -al /sys/fs/cgroup/ || true' "$CONTAINER_NAME" -execInnerDocker 'ls -al /sys/fs/cgroup/memory || true' "$CONTAINER_NAME" +execInnerDocker 'echo Inner Docker CGroups opts here' "$CONTAINER_NAME" +execInnerDocker 'ls -al /sys/fs/cgroup/cgroup.controllers || true' "$CONTAINER_NAME" +execInnerDocker 'ls -al /sys/fs/cgroup/ || true' "$CONTAINER_NAME" +execInnerDocker 'ls -al /sys/fs/cgroup/memory || true' "$CONTAINER_NAME" # Prepare core dump directory inside container -execInnerDocker 'mkdir -p core && chmod 777 core' "$CONTAINER_NAME" -docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/. "$CONTAINER_NAME":/ -docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh "$CONTAINER_NAME":/ -docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh "$CONTAINER_NAME":/ +execInnerDocker 'mkdir -p core && chmod 777 core' "$CONTAINER_NAME" +docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/. "$CONTAINER_NAME":/ +docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh "$CONTAINER_NAME":/ +docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh "$CONTAINER_NAME":/ if [[ "$DO_SETUP" == "true" ]]; then - execInnerDocker '/setup-repo.sh' "$CONTAINER_NAME" + execInnerDocker '/setup-repo.sh' "$CONTAINER_NAME" fi # install deps if [[ "$RESULT" == *rocky* ]]; then - execInnerDocker 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' "$CONTAINER_NAME" # sudo bypass execInnerDocker "printf '%s\n' '#!/bin/sh' 'exec \"\$@\"' > /usr/bin/sudo && chmod +x /usr/bin/sudo" "$CONTAINER_NAME" + execInnerDockerWithRetry 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' "$CONTAINER_NAME" else - execInnerDocker 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME" + execInnerDockerWithRetry 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME" fi # Configure core dump naming pattern @@ -118,9 +118,9 @@ execInnerDocker 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p" #Install columnstore in container echo "Installing columnstore..." if [[ "$RESULT" == *rocky* ]]; then - execInnerDocker 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME" + execInnerDockerWithRetry 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME" else - execInnerDocker 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME" + execInnerDockerWithRetry 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME" fi sleep 5 diff --git a/build/utils.sh b/build/utils.sh index ebf3ac16d..caec203a2 100644 --- a/build/utils.sh +++ b/build/utils.sh @@ -1,199 +1,199 @@ #!/bin/bash if [[ -n "$TERM" && "$TERM" != "dumb" && $(command -v tput) ]]; then - TPUT_AVAILABLE=true + TPUT_AVAILABLE=true else - TPUT_AVAILABLE=false + TPUT_AVAILABLE=false fi if [[ $TPUT_AVAILABLE == true ]]; then - color_normal=$(tput sgr0) - color_bold=$(tput bold) - color_red="$color_bold$(tput setaf 1)" - color_green="$color_bold$(tput setaf 2)" - color_fawn=$(tput setaf 3); color_beige="$color_fawn" - color_yellow="$color_bold$color_fawn" - color_darkblue=$(tput setaf 4) + color_normal=$(tput sgr0) + color_bold=$(tput bold) + color_red="$color_bold$(tput setaf 1)" + color_green="$color_bold$(tput setaf 2)" + color_fawn=$(tput setaf 3) + color_beige="$color_fawn" + color_yellow="$color_bold$color_fawn" + color_darkblue=$(tput setaf 4) - color_blue="$color_bold$color_darkblue" - color_purple=$(tput setaf 5); color_magenta="$color_purple" - color_pink="$color_bold$color_purple" - color_darkcyan=$(tput setaf 6) - color_cyan="$color_bold$color_darkcyan" - color_gray=$(tput setaf 7) - color_darkgray="$color_bold"$(tput setaf 0) - color_white="$color_bold$color_gray" + color_blue="$color_bold$color_darkblue" + color_purple=$(tput setaf 5) + color_magenta="$color_purple" + color_pink="$color_bold$color_purple" + color_darkcyan=$(tput setaf 6) + color_cyan="$color_bold$color_darkcyan" + color_gray=$(tput setaf 7) + color_darkgray="$color_bold"$(tput setaf 0) + color_white="$color_bold$color_gray" + if [[ $(tput colors) == '256' ]]; then + color_red=$(tput setaf 196) + color_yellow=$(tput setaf 228) + color_cyan=$(tput setaf 87) + color_green=$(tput setaf 156) + color_darkgray=$(tput setaf 59) + fi - if [[ $(tput colors) == '256' ]]; then - color_red=$(tput setaf 196) - color_yellow=$(tput setaf 228) - color_cyan=$(tput setaf 87) - color_green=$(tput setaf 156) - color_darkgray=$(tput setaf 59) - fi +else + # Basic attributes + color_normal="\e[0m" # sgr0 + color_bold="\e[1m" # bold + + # Standard 8-color palette + color_red="\e[1;31m" # bold + setaf 1 + color_green="\e[1;32m" # bold + setaf 2 + color_fawn="\e[33m" # setaf 3 + color_beige="$color_fawn" # alias + color_yellow="\e[1;33m" # bold + setaf 3 + color_darkblue="\e[34m" # setaf 4 + color_blue="\e[1;34m" # bold + setaf 4 + color_purple="\e[35m" # setaf 5 + color_magenta="$color_purple" # alias + color_pink="\e[1;35m" # bold + setaf 5 + color_darkcyan="\e[36m" # setaf 6 + color_cyan="\e[1;36m" # bold + setaf 6 + color_gray="\e[37m" # setaf 7 + color_darkgray="\e[1;30m" # bold + setaf 0 + color_white="\e[1;37m" # bold + setaf 7 + if [ "$TERM" = "xterm-256color" ] || [ "$COLORTERM" = "truecolor" ] || [ "$COLORTERM" = "24bit" ]; then + # Only set 256-color codes if actually in a 256-color terminal + color_red="\e[91m" # bright red + color_yellow="\e[93m" # light yellow + color_cyan="\e[96m" # bright cyan + color_green="\e[92m" # light green + color_darkgray="\e[90m" # dark gray + fi fi -message() -{ +message() { + echo -e $color_cyan ・ $@$color_normal +} + +warn() { + echo -e $color_yellow ・ $@$color_normal +} + +error() { + echo -e $color_red ・ $@$color_normal +} + +message_split() { + echo -e $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal +} + +message_splitted() { + message_split + if [[ $TPUT_AVAILABLE == true ]]; then + echo $color_green ・ $@$color_normal + else + echo "$@" + fi + message_split +} + +colorify_array() { + PROMT="" + for a in "$@"; do if [[ $TPUT_AVAILABLE == true ]]; then - echo $color_cyan ・ $@$color_normal + i=$((((i + 1) % (123 - 106)) + 106)) + if [[ $(tput colors) == '256' ]]; then + PROMT="$PROMT $(tput setaf $i)$a$color_normal" + else + PROMT="$PROMT $a" + fi else - echo "$@" + PROMT="$PROMT $a" fi + done + echo $PROMT } -warn() -{ - if [[ $TPUT_AVAILABLE == true ]]; then - echo $color_yellow ・ $@$color_normal - else - echo "$@" +newline_array() { + PROMT="" + for a in "$@"; do + PROMT="$PROMT$a\n" + done + echo -e $PROMT +} + +function spinner() { + freq=${1:-10} + points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷) + colored_points=($(colorify_array ${points[@]})) + len=${#points[@]} + point_num=0 + line_num=0 + while read data; do + line_num=$((line_num + 1)) + if [[ $((line_num % freq)) = 0 ]]; then + point_num=$(((point_num + 1) % len)) + if [[ $TPUT_AVAILABLE == true ]]; then + echo -ne "\r${colored_points[point_num]}" + else + echo -ne "\r${points[point_num]}" + fi fi + done + echo } -error() -{ - if [[ $TPUT_AVAILABLE == true ]]; then - echo $color_red ・ $@$color_normal - else - echo "$@" - fi +function onelinearizator() { + while read data; do + echo -ne "\r\e[K$data" + done + echo } -message_split() -{ - if [[ $TPUT_AVAILABLE == true ]]; then - echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal - else - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - fi -} - -message_splitted() -{ - message_split - if [[ $TPUT_AVAILABLE == true ]]; then - echo $color_green ・ $@$color_normal - else - echo "$@" - fi - message_split -} - - -colorify_array() -{ - PROMT="" - for a in "$@" - do - if [[ $TPUT_AVAILABLE == true ]]; then - i=$((((i+1) % (123-106)) + 106)) - if [[ $(tput colors) == '256' ]]; then - PROMT="$PROMT $(tput setaf $i)$a$color_normal" - else - PROMT="$PROMT $a" - fi - else - PROMT="$PROMT $a" - fi - done - echo $PROMT -} - - -newline_array() -{ - PROMT="" - for a in "$@" - do - PROMT="$PROMT$a\n" - done - echo -e $PROMT -} - - -function spinner -{ - freq=${1:-10} - points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷) - colored_points=($(colorify_array ${points[@]})) - len=${#points[@]} - point_num=0 - line_num=0 - while read data; do - line_num=$((line_num+1)) - if [[ $((line_num % freq)) = 0 ]]; then - point_num=$(((point_num + 1) % len )) - if [[ $TPUT_AVAILABLE == true ]]; then - echo -ne "\r${colored_points[point_num]}" - else - echo -ne "\r${points[point_num]}" - fi - fi - done; - echo -} - -function onelinearizator -{ - while read data; do - echo -ne "\r\033[K$data" - done; - echo -} - -detect_distro() -{ - if [ -f /etc/os-release ]; then - . /etc/os-release - export OS=$NAME - export OS_VERSION=$VERSION_ID - elif type lsb_release >/dev/null 2>&1; then - # linuxbase.org - export OS=$(lsb_release -si) - export OS_VERSION=$(lsb_release -sr) - elif [ -f /etc/lsb-release ]; then - # For some versions of Debian/Ubuntu without lsb_release command - . /etc/lsb-release - export OS=$DISTRIB_ID - OS_VERSION=$DISTRIB_RELEASE - elif [ -f /etc/debian_version ]; then - # Older Debian/Ubuntu/etc. - OS=Debian - OS_VERSION=$(cat /etc/debian_version) - else - # Fall back to uname, e.g. "Linux ", also works for BSD, etc. - OS=$(uname -s) - OS_VERSION=$(uname -r) - fi - OS=$(echo $OS | cut -f 1 -d " " | tr '[:upper:]' '[:lower:]')":"$OS_VERSION - message "Detected $color_yellow$OS $OS_VERSION$color_normal" +detect_distro() { + if [ -f /etc/os-release ]; then + . /etc/os-release + export OS=$NAME + export OS_VERSION=$VERSION_ID + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + export OS=$(lsb_release -si) + export OS_VERSION=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + export OS=$DISTRIB_ID + OS_VERSION=$DISTRIB_RELEASE + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + OS_VERSION=$(cat /etc/debian_version) + else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + OS=$(uname -s) + OS_VERSION=$(uname -r) + fi + OS=$(echo $OS | cut -f 1 -d " " | tr '[:upper:]' '[:lower:]')":"$OS_VERSION + message "Detected $color_yellow$OS $OS_VERSION$color_normal" } menuStr="" -function hideCursor(){ - printf "\033[?25l" +function hideCursor() { + printf "\e[?25l" # capture CTRL+C so cursor can be reset trap "showCursor && exit 0" 2 } -function showCursor(){ - printf "\033[?25h" +function showCursor() { + printf "\e[?25h" } -function clearLastMenu(){ +function clearLastMenu() { local msgLineCount=$(printf "$menuStr" | wc -l) # moves the curser up N lines so the output overwrites it - echo -en "\033[${msgLineCount}A" + echo -en "\e[${msgLineCount}A" # clear to end of screen to ensure there's no text left behind from previous input [ $1 ] && tput ed } -function renderMenu(){ +function renderMenu() { local start=0 local selector="" local instruction="$1" @@ -205,8 +205,8 @@ function renderMenu(){ # Get the longest item from the list so that we know how many spaces to add # to ensure there's no overlap from longer items when a list is scrolling up or down. - for (( i=0; i<$itemsLength; i++ )); do - if (( ${#menuItems[i]} > longest )); then + for ((i = 0; i < $itemsLength; i++)); do + if ((${#menuItems[i]} > longest)); then longest=${#menuItems[i]} fi done @@ -216,12 +216,12 @@ function renderMenu(){ listLength=$3 if [ $selectedIndex -ge $listLength ]; then - start=$(($selectedIndex+1-$listLength)) - listLength=$(($selectedIndex+1)) + start=$(($selectedIndex + 1 - $listLength)) + listLength=$(($selectedIndex + 1)) fi fi - for (( i=$start; i<$listLength; i++ )); do + for ((i = $start; i < $listLength; i++)); do local currItem="${menuItems[i]}" currItemLength=${#currItem} @@ -246,9 +246,9 @@ function renderMenu(){ printf "${menuStr}" } -function getChoice(){ - local KEY__ARROW_UP=$(echo -e "\033[A") - local KEY__ARROW_DOWN=$(echo -e "\033[B") +function getChoice() { + local KEY__ARROW_UP=$(echo -e "\e[A") + local KEY__ARROW_DOWN=$(echo -e "\e[B") local KEY__ENTER=$(echo -e "\n") local captureInput=true local displayHelp=false @@ -261,55 +261,55 @@ function getChoice(){ key="$1" case $key in - -h|--help) - displayHelp=true - shift - ;; - -i|--index) - selectedIndex=$2 - shift 2 - ;; - -m|--max) - maxViewable=$2 - shift 2 - ;; - -o|--options) - menuItems=$2[@] - menuItems=("${!menuItems}") - shift 2 - ;; - -q|--query) - instruction="$2" - shift 2 - ;; - *) - remainingArgs+=("$1") - shift - ;; + -h | --help) + displayHelp=true + shift + ;; + -i | --index) + selectedIndex=$2 + shift 2 + ;; + -m | --max) + maxViewable=$2 + shift 2 + ;; + -o | --options) + menuItems=$2[@] + menuItems=("${!menuItems}") + shift 2 + ;; + -q | --query) + instruction="$2" + shift 2 + ;; + *) + remainingArgs+=("$1") + shift + ;; esac done # just display help if $displayHelp; then - echo; + echo echo "Usage: getChoice [OPTION]..." echo "Renders a keyboard navigable menu with a visual indicator of what's selected." - echo; + echo echo " -h, --help Displays this message" echo " -i, --index The initially selected index for the options" echo " -m, --max Limit how many options are displayed" echo " -o, --options An Array of options for a User to choose from" echo " -q, --query Question or statement presented to the User" - echo; + echo echo "Example:" echo " foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")" - echo; + echo echo " getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4" echo " printf \"\\n First choice is '\${selectedChoice}'\\n\"" - echo; + echo echo " getChoice -q \"Select another option in case the first isn't available\" -o foodOptions" echo " printf \"\\n Second choice is '\${selectedChoice}'\\n\"" - echo; + echo return 0 fi @@ -327,119 +327,104 @@ function getChoice(){ hideCursor while $captureInput; do - read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters. + read -rsn3 key # `3` captures the escape (\e'), bracket ([), & type (A) characters. case "$key" in - "$KEY__ARROW_UP") - selectedIndex=$((selectedIndex-1)) - (( $selectedIndex < 0 )) && selectedIndex=$((itemsLength-1)) + "$KEY__ARROW_UP") + selectedIndex=$((selectedIndex - 1)) + (($selectedIndex < 0)) && selectedIndex=$((itemsLength - 1)) - renderMenu "$instruction" $selectedIndex $maxViewable true - ;; + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; - "$KEY__ARROW_DOWN") - selectedIndex=$((selectedIndex+1)) - (( $selectedIndex == $itemsLength )) && selectedIndex=0 + "$KEY__ARROW_DOWN") + selectedIndex=$((selectedIndex + 1)) + (($selectedIndex == $itemsLength)) && selectedIndex=0 - renderMenu "$instruction" $selectedIndex $maxViewable true - ;; + renderMenu "$instruction" $selectedIndex $maxViewable true + ;; - "$KEY__ENTER") - clearLastMenu true - showCursor - captureInput=false - ;; + "$KEY__ENTER") + clearLastMenu true + showCursor + captureInput=false + ;; esac done } - -function optparse.throw_error(){ +function optparse.throw_error() { local message="$1" - error "OPTPARSE: ERROR: $message" - exit 1 + error "OPTPARSE: ERROR: $message" + exit 1 } # ----------------------------------------------------------------------------------------------------------------------------- -function optparse.define(){ - if [ $# -lt 3 ]; then - optparse.throw_error "optparse.define [] [] []" - fi - for option_id in $( seq 1 $# ) ; do - local option="$( eval "echo \$$option_id")" - local key="$( echo $option | awk -F "=" '{print $1}' )"; - local value="$( echo $option | awk -F "=" '{print $2}' )"; +function optparse.define() { + if [ $# -lt 3 ]; then + optparse.throw_error "optparse.define [] [] []" + fi + for option_id in $(seq 1 $#); do + local option="$(eval "echo \$$option_id")" + local key="$(echo $option | awk -F "=" '{print $1}')" + local value="$(echo $option | awk -F "=" '{print $2}')" - #essentials: shortname, longname, description - if [ "$key" = "short" ]; then - local shortname="$value" - if [ ${#shortname} -ne 1 ]; then - optparse.throw_error "short name expected to be one character long" - fi - local short="-${shortname}" - elif [ "$key" = "long" ]; then - local longname="$value" - if [ ${#longname} -lt 2 ]; then - optparse.throw_error "long name expected to be atleast one character long" - fi - local long="--${longname}" - elif [ "$key" = "desc" ]; then - local desc="$value" - elif [ "$key" = "default" ]; then - local default="$value" - elif [ "$key" = "variable" ]; then - local variable="$value" - elif [ "$key" = "value" ]; then - local val="$value" - fi - done - - if [ "$variable" = "" ]; then - optparse.throw_error "You must give a variable for option: ($short/$long)" - fi - - if [ "$val" = "" ]; then - val="\$OPTARG" - fi - - # build OPTIONS and help - optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")" - if [ "$default" != "" ]; then - optparse_usage="${optparse_usage} [default:$default]" - fi - optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;" - if [ "$default" != "" ]; then - optparse_defaults="${optparse_defaults}#NL${variable}=${default}" - fi - optparse_arguments_string="${optparse_arguments_string}${shortname}" - if [ "$val" = "\$OPTARG" ]; then - optparse_arguments_string="${optparse_arguments_string}:" - fi - optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;" -} - -function execInnerDocker() { - local cmd_str="$1" - local img="$2" - local flags="${3:-}" - - docker exec $flags -t "$img" bash -c "$cmd_str" - local dockerCommandExitCode=$? - - if [[ $dockerCommandExitCode -ne 0 ]]; then - echo "Command \"$cmd_str\" failed in container \"$img\"" - exit $dockerCommandExitCode + #essentials: shortname, longname, description + if [ "$key" = "short" ]; then + local shortname="$value" + if [ ${#shortname} -ne 1 ]; then + optparse.throw_error "short name expected to be one character long" + fi + local short="-${shortname}" + elif [ "$key" = "long" ]; then + local longname="$value" + if [ ${#longname} -lt 2 ]; then + optparse.throw_error "long name expected to be atleast one character long" + fi + local long="--${longname}" + elif [ "$key" = "desc" ]; then + local desc="$value" + elif [ "$key" = "default" ]; then + local default="$value" + elif [ "$key" = "variable" ]; then + local variable="$value" + elif [ "$key" = "value" ]; then + local val="$value" fi + done + + if [ "$variable" = "" ]; then + optparse.throw_error "You must give a variable for option: ($short/$long)" + fi + + if [ "$val" = "" ]; then + val="\$OPTARG" + fi + + # build OPTIONS and help + optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")" + if [ "$default" != "" ]; then + optparse_usage="${optparse_usage} [default:$default]" + fi + optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;" + if [ "$default" != "" ]; then + optparse_defaults="${optparse_defaults}#NL${variable}=${default}" + fi + optparse_arguments_string="${optparse_arguments_string}${shortname}" + if [ "$val" = "\$OPTARG" ]; then + optparse_arguments_string="${optparse_arguments_string}:" + fi + optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;" } # ----------------------------------------------------------------------------------------------------------------------------- -function optparse.build(){ - local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")" +function optparse.build() { + local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")" - # Building getopts header here + # Building getopts header here - # Function usage - cat << EOF > $build_file + # Function usage + cat <$build_file function usage(){ cat << XXX usage: \$0 [OPTIONS] @@ -487,19 +472,75 @@ done rm $build_file EOF - local -A o=( ['#NL']='\n' ['#TB']='\t' ) + local -A o=(['#NL']='\n' ['#TB']='\t') - for i in "${!o[@]}"; do - sed -i "s/${i}/${o[$i]}/g" $build_file - done + for i in "${!o[@]}"; do + sed -i "s/${i}/${o[$i]}/g" $build_file + done - # Unset global variables - unset optparse_usage - unset optparse_process - unset optparse_arguments_string - unset optparse_defaults - unset optparse_contractions + # Unset global variables + unset optparse_usage + unset optparse_process + unset optparse_arguments_string + unset optparse_defaults + unset optparse_contractions - # Return file name to parent - echo "$build_file" -} \ No newline at end of file + # Return file name to parent + echo "$build_file" +} + +function retry_eval() { + local max_retries=$1 + local command=$2 + local attempt=1 + local initial_delay=1 + + while [ "$attempt" -le "$max_retries" ]; do + message_split + message "Attempt $attempt of $max_retries: $command" + if eval "$command"; then + message "Command '$command' done" + message_split + return 0 + fi + if [ "$attempt" -lt "$max_retries" ]; then + delay=$((initial_delay * 2 ** (attempt - 1))) + message "Retrying command '$command' in $delay seconds..." + message_split + sleep "$delay" + fi + ((attempt++)) + done + + error "Max retries reached for command: $command" + message_split + exit 13 +} + +function execInnerDocker() { + local cmd_str="$1" + local img="$2" + local flags="${3:-}" + + docker exec $flags -t "$img" bash -c "$cmd_str" + local dockerCommandExitCode=$? + + if [[ $dockerCommandExitCode -ne 0 ]]; then + error "Command \"$cmd_str\" failed in container \"$img\"" + exit $dockerCommandExitCode + fi +} + +function execInnerDockerWithRetry() { + local cmd_str="$1" + local img="$2" + local flags="${3:-}" + + docker exec $flags -t "$img" bash -c " $(declare -f retry_eval); retry_eval 5 '$cmd_str'" + local dockerCommandExitCode=$? + + if [[ $dockerCommandExitCode -ne 0 ]]; then + error "Command \"$cmd_str\" failed in container \"$img\"" + exit $dockerCommandExitCode + fi +}