chore(ci): add retries for the installations, fix colors for ci, clear apt cache, fix deps missing on CI stage by moving step to the beginnig

build/bootstrap_mcs.sh

@@ -67,6 +67,51 @@ source $(optparse.build)
 
 message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
 
+install_deps() {
+    if [[ $INSTALL_DEPS = false ]]; then
+        return
+    fi
+    message_split
+    prereq=""
+    RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \
+      snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \
+      rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \
+      cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
+
+    DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
+      bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
+      libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
+      libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
+      graphviz devscripts ccache equivs eatmydata curl"
+
+    if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
+        command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
+      dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \
+      ${RPM_BUILD_DEPS}"
+    elif
+        [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]
+    then
+        command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \
+      dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}"
+
+    elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then
+        prereq="apt-get clean && rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin"
+        command="${DEB_BUILD_DEPS}"
+    else
+        echo "Unsupported OS: $OS"
+        exit 17
+    fi
+
+    if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
+        command="${command} lto-disabled-list"
+    fi
+    eval "$prereq"
+    message "Installing dependencies for $OS"
+    retry_eval 5 "$command"
+}
+
+install_deps
+
 cd $COLUMSNTORE_SOURCE_PATH
 COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch"
@@ -103,40 +148,6 @@ disable_git_restore_frozen_revision() {
 
 DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot"
 
-install_deps() {
-    message_split
-
-    RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \
-      snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \
-      rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \
-      cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
-
-    DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
-      bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
-      libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
-      libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
-      graphviz devscripts ccache equivs eatmydata curl"
-
-    if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
-        command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
-      dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \
-      ${RPM_BUILD_DEPS}"
-
-    elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]; then
-        command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \
-      dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}"
-
-    elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then
-        command="${DEB_BUILD_DEPS}"
-
-    else
-        echo "Unsupported OS: $OS"
-        exit 17
-    fi
-
-    message "Installing dependencies for $OS"
-    eval "$command"
-}
 stop_service() {
     message_split
     message "Stopping MariaDB services"
@@ -206,7 +217,6 @@ modify_packaging() {
 
     #disable LTO for 22.04 for now
     if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
-        apt install -y lto-disabled-list &&
         for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do
             echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list
         done &&
@@ -269,7 +279,7 @@ construct_cmake_flags() {
 
     if [[ $MAINTAINER_MODE = true ]]; then
         MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES)
-        message "Columnstore mainteiner mode on"
+        message "Columnstore maintainer mode on"
     else
         warn "Maintainer mode is disabled, be careful, alien"
     fi
@@ -665,10 +675,6 @@ smoke() {
     fi
 }
 
-if [[ $INSTALL_DEPS = true ]]; then
-    install_deps
-fi
-
 if [[ $DO_NOT_FREEZE_REVISION = false ]]; then
     disable_git_restore_frozen_revision
 fi

build/build_cmapi.sh

@@ -27,7 +27,6 @@ if [[ -z "${OS:-}" || -z "${ARCH:-}" ]]; then
   exit 1
 fi
 
-
 pkg_format="deb"
 if [[ "$OS" == *"rocky"* ]]; then
   pkg_format="rpm"
@@ -52,17 +51,16 @@ install_deps() {
   cd "$COLUMNSTORE_SOURCE_PATH"/cmapi
 
   if [[ "$OS" == "rockylinux:9" ]]; then
-     dnf install -q -y libxcrypt-compat yum-utils
+    retry_eval 5 "dnf install -q -y libxcrypt-compat yum-utils"
-     dnf config-manager --set-enabled devel && dnf update -q -y  #to make redhat-lsb-core available for rocky 9
+    retry_eval 5 "dnf config-manager --set-enabled devel && dnf update -q -y" #to make redhat-lsb-core available for rocky 9
   fi
 
   if [[ "$pkg_format" == "rpm" ]]; then
-    dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive
+    retry_eval 5 "dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive"
   else
-    apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release
+    retry_eval 5 "apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release"
   fi
 
-
   if [ "$ARCH" == "amd64" ]; then
     PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst"
   elif [ "$ARCH" == "arm64" ]; then

build/createrepo.sh

@@ -58,10 +58,10 @@ else
 fi
 
 if [[ $(compgen -G "./${RESULT}/*.rpm") ]]; then
-  dnf install -q -y createrepo
+  retry_eval 5 "dnf install -q -y createrepo"
   createrepo "./${RESULT}"
 else
-  apt update && apt install -y dpkg-dev
+  retry_eval 5 "apt update && apt install -y dpkg-dev"
   dpkg-scanpackages "${RESULT}" | gzip >"./${RESULT}/Packages.gz"
 fi
 

build/prepare_test_stage.sh

@@ -48,21 +48,21 @@ start_container() {
     )
 
     if [[ "$CONTAINER_NAME" == *smoke* ]]; then
-        docker_run_args+=( --memory 3g )
+        docker_run_args+=(--memory 3g)
     elif [[ "$CONTAINER_NAME" == *mtr* ]]; then
-        docker_run_args+=( --shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH" )
+        docker_run_args+=(--shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH")
     elif [[ "$CONTAINER_NAME" == *cmapi* ]]; then
-        docker_run_args+=( --env PYTHONPATH="${PYTHONPATH}" )
+        docker_run_args+=(--env PYTHONPATH="${PYTHONPATH}")
     elif [[ "$CONTAINER_NAME" == *upgrade* ]]; then
-        docker_run_args+=( --env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro )
+        docker_run_args+=(--env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro)
     elif [[ "$CONTAINER_NAME" == *regression* ]]; then
-        docker_run_args+=( --shm-size=500m --memory 12g )
+        docker_run_args+=(--shm-size=500m --memory 12g)
     else
         echo "Unknown container type: $CONTAINER_NAME"
         exit 1
     fi
 
-    docker_run_args+=( "$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target )
+    docker_run_args+=("$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target)
 
     docker run "${docker_run_args[@]}"
     sleep 5
@@ -105,11 +105,11 @@ fi
 
 # install deps
 if [[ "$RESULT" == *rocky* ]]; then
-    execInnerDocker 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which'                   "$CONTAINER_NAME"
     # sudo bypass
     execInnerDocker "printf '%s\n' '#!/bin/sh' 'exec \"\$@\"' > /usr/bin/sudo && chmod +x /usr/bin/sudo" "$CONTAINER_NAME"
+    execInnerDockerWithRetry 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' "$CONTAINER_NAME"
 else
-    execInnerDocker 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME"
+    execInnerDockerWithRetry 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME"
 fi
 
 # Configure core dump naming pattern
@@ -118,9 +118,9 @@ execInnerDocker 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"
 #Install columnstore in container
 echo "Installing columnstore..."
 if [[ "$RESULT" == *rocky* ]]; then
-    execInnerDocker 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME"
+    execInnerDockerWithRetry 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME"
 else
-    execInnerDocker 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME"
+    execInnerDockerWithRetry 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME"
 fi
 
 sleep 5

build/utils.sh

@@ -11,12 +11,14 @@ if [[ $TPUT_AVAILABLE == true ]]; then
   color_bold=$(tput bold)
   color_red="$color_bold$(tput setaf 1)"
   color_green="$color_bold$(tput setaf 2)"
-    color_fawn=$(tput setaf 3); color_beige="$color_fawn"
+  color_fawn=$(tput setaf 3)
+  color_beige="$color_fawn"
   color_yellow="$color_bold$color_fawn"
   color_darkblue=$(tput setaf 4)
 
   color_blue="$color_bold$color_darkblue"
-    color_purple=$(tput setaf 5); color_magenta="$color_purple"
+  color_purple=$(tput setaf 5)
+  color_magenta="$color_purple"
   color_pink="$color_bold$color_purple"
   color_darkcyan=$(tput setaf 6)
   color_cyan="$color_bold$color_darkcyan"
@@ -24,7 +26,6 @@ if [[ $TPUT_AVAILABLE == true ]]; then
   color_darkgray="$color_bold"$(tput setaf 0)
   color_white="$color_bold$color_gray"
 
-
   if [[ $(tput colors) == '256' ]]; then
     color_red=$(tput setaf 196)
     color_yellow=$(tput setaf 228)
@@ -32,46 +33,55 @@ if [[ $TPUT_AVAILABLE == true ]]; then
     color_green=$(tput setaf 156)
     color_darkgray=$(tput setaf 59)
   fi
+
+else
+  # Basic attributes
+  color_normal="\e[0m" # sgr0
+  color_bold="\e[1m"   # bold
+
+  # Standard 8-color palette
+  color_red="\e[1;31m"          # bold + setaf 1
+  color_green="\e[1;32m"        # bold + setaf 2
+  color_fawn="\e[33m"           # setaf 3
+  color_beige="$color_fawn"     # alias
+  color_yellow="\e[1;33m"       # bold + setaf 3
+  color_darkblue="\e[34m"       # setaf 4
+  color_blue="\e[1;34m"         # bold + setaf 4
+  color_purple="\e[35m"         # setaf 5
+  color_magenta="$color_purple" # alias
+  color_pink="\e[1;35m"         # bold + setaf 5
+  color_darkcyan="\e[36m"       # setaf 6
+  color_cyan="\e[1;36m"         # bold + setaf 6
+  color_gray="\e[37m"           # setaf 7
+  color_darkgray="\e[1;30m"     # bold + setaf 0
+  color_white="\e[1;37m"        # bold + setaf 7
+  if [ "$TERM" = "xterm-256color" ] || [ "$COLORTERM" = "truecolor" ] || [ "$COLORTERM" = "24bit" ]; then
+    # Only set 256-color codes if actually in a 256-color terminal
+    color_red="\e[91m"      # bright red
+    color_yellow="\e[93m"   # light yellow
+    color_cyan="\e[96m"     # bright cyan
+    color_green="\e[92m"    # light green
+    color_darkgray="\e[90m" # dark gray
+  fi
 fi
 
-message()
+message() {
-{
+  echo -e $color_cyan ・ $@$color_normal
-    if [[ $TPUT_AVAILABLE == true ]]; then
-        echo $color_cyan ・ $@$color_normal
-    else
-        echo "$@"
-    fi
 }
 
-warn()
+warn() {
-{
+  echo -e $color_yellow ・ $@$color_normal
-    if [[ $TPUT_AVAILABLE == true ]]; then
-        echo $color_yellow ・ $@$color_normal
-    else
-        echo "$@"
-    fi
 }
 
-error()
+error() {
-{
+  echo -e $color_red ・ $@$color_normal
-    if [[ $TPUT_AVAILABLE == true ]]; then
-        echo $color_red ・ $@$color_normal
-    else
-        echo "$@"
-    fi
 }
 
-message_split()
+message_split() {
-{
+  echo -e $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal
-    if [[ $TPUT_AVAILABLE == true ]]; then
-        echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal
-    else
-        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-    fi
 }
 
-message_splitted()
+message_splitted() {
-{
   message_split
   if [[ $TPUT_AVAILABLE == true ]]; then
     echo $color_green ・ $@$color_normal
@@ -81,14 +91,11 @@ message_splitted()
   message_split
 }
 
-
+colorify_array() {
-colorify_array()
-{
   PROMT=""
-    for a in "$@"
+  for a in "$@"; do
-    do
     if [[ $TPUT_AVAILABLE == true ]]; then
-            i=$((((i+1) % (123-106)) + 106))
+      i=$((((i + 1) % (123 - 106)) + 106))
       if [[ $(tput colors) == '256' ]]; then
         PROMT="$PROMT $(tput setaf $i)$a$color_normal"
       else
@@ -101,20 +108,15 @@ colorify_array()
   echo $PROMT
 }
 
-
+newline_array() {
-newline_array()
-{
   PROMT=""
-    for a in "$@"
+  for a in "$@"; do
-    do
     PROMT="$PROMT$a\n"
   done
   echo -e $PROMT
 }
 
-
+function spinner() {
-function spinner
-{
   freq=${1:-10}
   points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷)
   colored_points=($(colorify_array ${points[@]}))
@@ -122,29 +124,27 @@ function spinner
   point_num=0
   line_num=0
   while read data; do
-        line_num=$((line_num+1))
+    line_num=$((line_num + 1))
     if [[ $((line_num % freq)) = 0 ]]; then
-            point_num=$(((point_num + 1) % len ))
+      point_num=$(((point_num + 1) % len))
       if [[ $TPUT_AVAILABLE == true ]]; then
         echo -ne "\r${colored_points[point_num]}"
       else
         echo -ne "\r${points[point_num]}"
       fi
     fi
-    done;
+  done
   echo
 }
 
-function onelinearizator
+function onelinearizator() {
-{
   while read data; do
-        echo -ne "\r\033[K$data"
+    echo -ne "\r\e[K$data"
-    done;
+  done
   echo
 }
 
-detect_distro()
+detect_distro() {
-{
   if [ -f /etc/os-release ]; then
     . /etc/os-release
     export OS=$NAME
@@ -173,27 +173,27 @@ detect_distro()
 
 menuStr=""
 
-function hideCursor(){
+function hideCursor() {
-  printf "\033[?25l"
+  printf "\e[?25l"
 
   # capture CTRL+C so cursor can be reset
   trap "showCursor && exit 0" 2
 }
 
-function showCursor(){
+function showCursor() {
-  printf "\033[?25h"
+  printf "\e[?25h"
 }
 
-function clearLastMenu(){
+function clearLastMenu() {
   local msgLineCount=$(printf "$menuStr" | wc -l)
   # moves the curser up N lines so the output overwrites it
-  echo -en "\033[${msgLineCount}A"
+  echo -en "\e[${msgLineCount}A"
 
   # clear to end of screen to ensure there's no text left behind from previous input
   [ $1 ] && tput ed
 }
 
-function renderMenu(){
+function renderMenu() {
   local start=0
   local selector=""
   local instruction="$1"
@@ -205,8 +205,8 @@ function renderMenu(){
 
   # Get the longest item from the list so that we know how many spaces to add
   # to ensure there's no overlap from longer items when a list is scrolling up or down.
-  for (( i=0; i<$itemsLength; i++ )); do
+  for ((i = 0; i < $itemsLength; i++)); do
-    if (( ${#menuItems[i]} > longest )); then
+    if ((${#menuItems[i]} > longest)); then
       longest=${#menuItems[i]}
     fi
   done
@@ -216,12 +216,12 @@ function renderMenu(){
     listLength=$3
 
     if [ $selectedIndex -ge $listLength ]; then
-      start=$(($selectedIndex+1-$listLength))
+      start=$(($selectedIndex + 1 - $listLength))
-      listLength=$(($selectedIndex+1))
+      listLength=$(($selectedIndex + 1))
     fi
   fi
 
-  for (( i=$start; i<$listLength; i++ )); do
+  for ((i = $start; i < $listLength; i++)); do
     local currItem="${menuItems[i]}"
     currItemLength=${#currItem}
 
@@ -246,9 +246,9 @@ function renderMenu(){
   printf "${menuStr}"
 }
 
-function getChoice(){
+function getChoice() {
-  local KEY__ARROW_UP=$(echo -e "\033[A")
+  local KEY__ARROW_UP=$(echo -e "\e[A")
-  local KEY__ARROW_DOWN=$(echo -e "\033[B")
+  local KEY__ARROW_DOWN=$(echo -e "\e[B")
   local KEY__ENTER=$(echo -e "\n")
   local captureInput=true
   local displayHelp=false
@@ -261,24 +261,24 @@ function getChoice(){
     key="$1"
 
     case $key in
-      -h|--help)
+    -h | --help)
       displayHelp=true
       shift
       ;;
-      -i|--index)
+    -i | --index)
       selectedIndex=$2
       shift 2
       ;;
-      -m|--max)
+    -m | --max)
       maxViewable=$2
       shift 2
       ;;
-      -o|--options)
+    -o | --options)
       menuItems=$2[@]
       menuItems=("${!menuItems}")
       shift 2
       ;;
-      -q|--query)
+    -q | --query)
       instruction="$2"
       shift 2
       ;;
@@ -291,25 +291,25 @@ function getChoice(){
 
   # just display help
   if $displayHelp; then
-    echo;
+    echo
     echo "Usage: getChoice [OPTION]..."
     echo "Renders a keyboard navigable menu with a visual indicator of what's selected."
-    echo;
+    echo
     echo "  -h, --help     Displays this message"
     echo "  -i, --index    The initially selected index for the options"
     echo "  -m, --max      Limit how many options are displayed"
     echo "  -o, --options  An Array of options for a User to choose from"
     echo "  -q, --query    Question or statement presented to the User"
-    echo;
+    echo
     echo "Example:"
     echo "  foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")"
-    echo;
+    echo
     echo "  getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4"
     echo "  printf \"\\n First choice is '\${selectedChoice}'\\n\""
-    echo;
+    echo
     echo "  getChoice -q \"Select another option in case the first isn't available\" -o foodOptions"
     echo "  printf \"\\n Second choice is '\${selectedChoice}'\\n\""
-    echo;
+    echo
 
     return 0
   fi
@@ -327,19 +327,19 @@ function getChoice(){
   hideCursor
 
   while $captureInput; do
-    read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters.
+    read -rsn3 key # `3` captures the escape (\e'), bracket ([), & type (A) characters.
 
     case "$key" in
     "$KEY__ARROW_UP")
-        selectedIndex=$((selectedIndex-1))
+      selectedIndex=$((selectedIndex - 1))
-        (( $selectedIndex < 0 )) && selectedIndex=$((itemsLength-1))
+      (($selectedIndex < 0)) && selectedIndex=$((itemsLength - 1))
 
       renderMenu "$instruction" $selectedIndex $maxViewable true
       ;;
 
     "$KEY__ARROW_DOWN")
-        selectedIndex=$((selectedIndex+1))
+      selectedIndex=$((selectedIndex + 1))
-        (( $selectedIndex == $itemsLength )) && selectedIndex=0
+      (($selectedIndex == $itemsLength)) && selectedIndex=0
 
       renderMenu "$instruction" $selectedIndex $maxViewable true
       ;;
@@ -353,22 +353,21 @@ function getChoice(){
   done
 }
 
-
+function optparse.throw_error() {
-function optparse.throw_error(){
   local message="$1"
   error "OPTPARSE: ERROR: $message"
   exit 1
 }
 
 # -----------------------------------------------------------------------------------------------------------------------------
-function optparse.define(){
+function optparse.define() {
   if [ $# -lt 3 ]; then
     optparse.throw_error "optparse.define <short> <long> <variable> [<desc>] [<default>] [<value>]"
   fi
-        for option_id in $( seq 1 $# ) ; do
+  for option_id in $(seq 1 $#); do
-                local option="$( eval "echo \$$option_id")"
+    local option="$(eval "echo \$$option_id")"
-                local key="$( echo $option | awk -F "=" '{print $1}' )";
+    local key="$(echo $option | awk -F "=" '{print $1}')"
-                local value="$( echo $option | awk -F "=" '{print $2}' )";
+    local value="$(echo $option | awk -F "=" '{print $2}')"
 
     #essentials: shortname, longname, description
     if [ "$key" = "short" ]; then
@@ -418,28 +417,14 @@ function optparse.define(){
   optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;"
 }
 
-function execInnerDocker() {
-    local cmd_str="$1"
-    local img="$2"
-    local flags="${3:-}"
-
-    docker exec $flags -t "$img" bash -c "$cmd_str"
-    local dockerCommandExitCode=$?
-
-    if [[ $dockerCommandExitCode -ne 0 ]]; then
-        echo "Command \"$cmd_str\" failed in container \"$img\""
-        exit $dockerCommandExitCode
-    fi
-}
-
 # -----------------------------------------------------------------------------------------------------------------------------
-function optparse.build(){
+function optparse.build() {
   local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")"
 
   # Building getopts header here
 
   # Function usage
-        cat << EOF > $build_file
+  cat <<EOF >$build_file
 function usage(){
 cat << XXX
 usage: \$0 [OPTIONS]
@@ -487,7 +472,7 @@ done
 rm $build_file
 EOF
 
-        local -A o=( ['#NL']='\n' ['#TB']='\t' )
+  local -A o=(['#NL']='\n' ['#TB']='\t')
 
   for i in "${!o[@]}"; do
     sed -i "s/${i}/${o[$i]}/g" $build_file
@@ -503,3 +488,59 @@ EOF
   # Return file name to parent
   echo "$build_file"
 }
+
+function retry_eval() {
+  local max_retries=$1
+  local command=$2
+  local attempt=1
+  local initial_delay=1
+
+  while [ "$attempt" -le "$max_retries" ]; do
+    message_split
+    message "Attempt $attempt of $max_retries: $command"
+    if eval "$command"; then
+      message "Command '$command' done"
+      message_split
+      return 0
+    fi
+    if [ "$attempt" -lt "$max_retries" ]; then
+      delay=$((initial_delay * 2 ** (attempt - 1)))
+      message "Retrying command '$command' in $delay seconds..."
+      message_split
+      sleep "$delay"
+    fi
+    ((attempt++))
+  done
+
+  error "Max retries reached for command: $command"
+  message_split
+  exit 13
+}
+
+function execInnerDocker() {
+  local cmd_str="$1"
+  local img="$2"
+  local flags="${3:-}"
+
+  docker exec $flags -t "$img" bash -c "$cmd_str"
+  local dockerCommandExitCode=$?
+
+  if [[ $dockerCommandExitCode -ne 0 ]]; then
+    error "Command \"$cmd_str\" failed in container \"$img\""
+    exit $dockerCommandExitCode
+  fi
+}
+
+function execInnerDockerWithRetry() {
+  local cmd_str="$1"
+  local img="$2"
+  local flags="${3:-}"
+
+  docker exec $flags -t "$img" bash -c " $(declare -f retry_eval); retry_eval 5 '$cmd_str'"
+  local dockerCommandExitCode=$?
+
+  if [[ $dockerCommandExitCode -ne 0 ]]; then
+    error "Command \"$cmd_str\" failed in container \"$img\""
+    exit $dockerCommandExitCode
+  fi
+}