diff --git a/cmapi/scripts/mcs_backup_manager.sh b/cmapi/scripts/mcs_backup_manager.sh index d140c1c94..feeb17a0b 100644 --- a/cmapi/scripts/mcs_backup_manager.sh +++ b/cmapi/scripts/mcs_backup_manager.sh @@ -13,8 +13,8 @@ # ######################################################################## # Documentation: bash mcs_backup_manager.sh help -# Version: 3.8 -# +# Version: 3.9 +# # Backup Example # LocalStorage: sudo ./mcs_backup_manager.sh backup # S3: sudo ./mcs_backup_manager.sh backup -bb s3://my-cs-backups @@ -23,17 +23,17 @@ # # Restore Example # LocalStorage: sudo ./mcs_backup_manager.sh restore -l -# S3: sudo ./mcs_backup_manager.sh restore -bb s3://my-cs-backups -l -# +# S3: sudo ./mcs_backup_manager.sh restore -bb s3://my-cs-backups -l +# ######################################################################## -mcs_bk_manager_version="3.8" +mcs_bk_manager_version="3.9" start=$(date +%s) action=$1 print_action_help_text() { echo " MariaDB Columnstore Backup Manager - + Actions: backup Full & Incremental columnstore backup with additional flags to augment the backup taken @@ -64,13 +64,12 @@ check_operating_system() { *) # unknown option printf "\ncheck_operating_system: unknown os & version: $OPERATING_SYSTEM\n" exit 2; - esac + esac } -load_default_backup_variables() -{ +load_default_backup_variables() { check_operating_system - + # What directory to store the backups on this machine or the target machine. # Consider write permissions of the scp user and the user running this script. # Mariadb-backup will use this location as a tmp dir for S3 and remote backups temporarily @@ -101,7 +100,7 @@ load_default_backup_variables() ;; *) # unknown option handle_failed_dependencies "\nload_default_backup_variables: unknown os & version: $OPERATING_SYSTEM\n"; - esac + esac # Fixed Paths CS_CONFIGS_PATH="/etc/columnstore" @@ -115,7 +114,7 @@ load_default_backup_variables() cs_cache=$(grep -A25 "\[Cache\]" $STORAGEMANGER_CNF | grep ^path | cut -d "=" -f 2 | tr -d " ") # What storage topogoly is being used by Columnstore - found in /etc/columnstore/storagemanager.cnf - # Options: "LocalStorage" or "S3" + # Options: "LocalStorage" or "S3" storage=$(grep -m 1 "^service = " $STORAGEMANGER_CNF | awk '{print $3}') # Name of the existing bucket used in the cluster - found in /etc/columnstore/storagemanager.cnf @@ -125,8 +124,8 @@ load_default_backup_variables() # modes ['direct','indirect'] - direct backups run on the columnstore nodes themselves. indirect run on another machine that has read-only mounts associated with columnstore/mariadb mode="direct" - # Name of the Configuration file to load variables from - config_file=".cs-backup-config" + # Name of the Configuration file to load variables from - relative or full path accepted + config_file="" # Track your write speed with "dstat --top-cpu --top-io" # Monitor individual rsyncs with ` watch -n0.25 "ps aux | grep 'rsync -a' | grep -vE 'color|grep' | wc -l; ps aux | grep 'rsync -a' | grep -vE 'color|grep' " ` @@ -152,10 +151,13 @@ load_default_backup_variables() if [ ! -f /var/lib/columnstore/local/module ]; then pm="pm1"; else pm=$(cat /var/lib/columnstore/local/module); fi; PM_NUMBER=$(echo "$pm" | tr -dc '0-9') if [[ -z $PM_NUMBER ]]; then PM_NUMBER=1; fi; - + #source_ips=$(grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}" /etc/columnstore/Columnstore.xml) #source_host_names=$(grep "" /etc/columnstore/Columnstore.xml) - cmapi_key="$(grep "x-api-key" $CS_CONFIGS_PATH/cmapi_server.conf | awk '{print $3}' | tr -d "'" )"; + cmapi_key="" + if [[ -f "$CS_CONFIGS_PATH/cmapi_server.conf" ]]; then + cmapi_key="$(grep "x-api-key" "$CS_CONFIGS_PATH/cmapi_server.conf" | awk '{print $3}' | tr -d "'")"; + fi final_message="Backup Complete" skip_save_brm=false skip_locks=false @@ -192,15 +194,14 @@ load_default_backup_variables() columnstore_online=false confirm_xmllint_installed - + # Number of DBroots # Integer usually 1 or 3 DBROOT_COUNT=$(xmllint --xpath "string(//DBRootCount)" $CS_CONFIGS_PATH/Columnstore.xml) ASSIGNED_DBROOT=$(xmllint --xpath "string(//ModuleDBRootID$PM_NUMBER-1-3)" $CS_CONFIGS_PATH/Columnstore.xml) } -parse_backup_variables() -{ +parse_backup_variables() { # Dynamic Arguments while [[ $# -gt 0 ]]; do key="$1" @@ -256,6 +257,9 @@ parse_backup_variables() ;; -f| --config-file) config_file="$2" + if [ "$config_file" == "" ] || [ ! -f "$config_file" ]; then + handle_early_exit_on_backup "\n[!!!] Invalid Configuration File: $config_file\n" true + fi shift # past argument shift # past value ;; @@ -299,7 +303,7 @@ parse_backup_variables() shift # past argument ;; -nv-ssl| --no-verify-ssl) - no_verify_ssl=true + no_verify_ssl=true shift # past argument ;; -pi| --poll-interval) @@ -327,16 +331,17 @@ parse_backup_variables() done # Load config file - if [ -f $config_file ]; then - source $config_file + if [ -n "$config_file" ] && [ -f "$config_file" ]; then + if ! source "$config_file"; then + handle_early_exit_on_backup "\n[!!!] Failed to source configuration file: $config_file\n" true + fi fi # Adjustment for indirect mode if [ $mode == "indirect" ]; then skip_locks=true; skip_save_brm=true; fi; } -print_backup_help_text() -{ +print_backup_help_text() { echo " Columnstore Backup @@ -349,7 +354,7 @@ print_backup_help_text() -s | --storage The storage used by columnstore data 'LocalStorage' or 'S3' -i | --incremental Adds columnstore deltas to an existing full backup [ , auto_most_recent ] -P | --parallel Number of parallel rsync/compression threads to run - -f | --config-file Path to backup configuration file to load variables from + -f | --config-file Path to backup configuration file to load variables from - see load_default_backup_variables() for defaults -sbrm | --skip-save-brm Skip saving brm prior to running a backup - ideal for dirty backups -slock | --skip-locks Skip issuing write locks - ideal for dirty backups -spoll | --skip-polls Skip sql checks confirming no write/cpimports running @@ -363,7 +368,7 @@ print_backup_help_text() -r | --retention-days Retain backups created within the last X days, the rest are deleted, default 0 = keep all backups -ha | --highavilability Hint wether shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) - HA S3 ( /var/lib/columnstore/storagemanager/ ) + HA S3 ( /var/lib/columnstore/storagemanager/ ) Local Storage Examples: ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage @@ -371,12 +376,12 @@ print_backup_help_text() ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage --incremental auto_most_recent ./$0 backup -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -s LocalStorage - S3 Examples: + S3 Examples: ./$0 backup -bb s3://my-cs-backups -s S3 ./$0 backup -bb s3://my-cs-backups -c pigz --quiet -sb ./$0 backup -bb gs://my-cs-backups -s S3 --incremental 12-18-2023 ./$0 backup -bb s3://my-onpremise-bucket -s S3 -url http://127.0.0.1:8000 - + Cron Example: */60 */24 * * * root bash /root/$0 -bb s3://my-cs-backups -s S3 >> /root/csBackup.log 2>&1 "; @@ -386,14 +391,13 @@ print_backup_help_text() } -print_backup_variables() -{ +print_backup_variables() { # Log Variables s1=20 s2=20 printf "\nBackup Variables\n" echo "--------------------------------------------------------------------------" - if [ -f $config_file ]; then + if [ -n "$config_file" ] && [ -f "$config_file" ]; then printf "%-${s1}s %-${s2}s\n" "Configuration File:" "$config_file"; fi echo "--------------------------------------------------------------------------" @@ -412,18 +416,18 @@ print_backup_variables() printf "%-${s1}s %-${s2}s\n" "Compression:" "true"; printf "%-${s1}s %-${s2}s\n" "Compression Format:" "$compress_format"; printf "%-${s1}s %-${s2}s\n" "Compression Threads:" "$PARALLEL_THREADS"; - else + else printf "%-${s1}s %-${s2}s\n" "Parallel Enabled:" "$parrallel_rsync"; if $parrallel_rsync ; then printf "%-${s1}s %-${s2}s\n" "Parallel Threads:" "$PARALLEL_THREADS"; fi; fi if [ $storage == "LocalStorage" ]; then printf "%-${s1}s %-${s2}s\n" "Backup Destination:" "$backup_destination"; - if [ $backup_destination == "Remote" ]; then printf "%-${s1}s %-${s2}s\n" "scp:" "$scp"; fi; + if [ $backup_destination == "Remote" ]; then printf "%-${s1}s %-${s2}s\n" "scp:" "$scp"; fi; printf "%-${s1}s %-${s2}s\n" "Backup Location:" "$backup_location"; fi - if [ $storage == "S3" ]; then + if [ $storage == "S3" ]; then printf "%-${s1}s %-${s2}s\n" "Active Bucket:" "$bucket"; printf "%-${s1}s %-${s2}s\n" "Backup Bucket:" "$backup_bucket"; fi @@ -431,7 +435,7 @@ print_backup_variables() } check_package_managers() { - + package_manager=''; if command -v apt &> /dev/null ; then if ! command -v dpkg-query &> /dev/null ; then @@ -443,21 +447,21 @@ check_package_managers() { if command -v yum &> /dev/null ; then package_manager="yum"; - fi + fi - if [ $package_manager == '' ]; then + if [ $package_manager == '' ]; then handle_failed_dependencies "[!!] No package manager found: yum or apt must be installed" exit 1; fi; } confirm_xmllint_installed() { - + if ! command -v xmllint > /dev/null; then printf "[!] xmllint not installed ... attempting auto install\n\n" check_package_managers case $package_manager in - yum ) + yum ) install_command="yum install libxml2 -y"; ;; apt ) @@ -481,7 +485,7 @@ confirm_rsync_installed() { printf "[!] rsync not installed ... attempting auto install\n\n" check_package_managers case $package_manager in - yum ) + yum ) install_command="yum install rsync -y"; ;; apt ) @@ -507,7 +511,7 @@ confirm_mariadb_backup_installed() { printf "[!] mariadb-backup not installed ... attempting auto install\n\n" check_package_managers case $package_manager in - yum ) + yum ) install_command="yum install MariaDB-backup -y"; ;; apt ) @@ -531,7 +535,7 @@ confirm_pigz_installed() { printf "[!] pigz not installed ... attempting auto install\n\n" check_package_managers case $package_manager in - yum ) + yum ) install_command="yum install pigz -y"; ;; apt ) @@ -549,13 +553,12 @@ confirm_pigz_installed() { fi } -check_for_dependancies() -{ +check_for_dependancies() { # Check pidof works if [ $mode != "indirect" ] && ! command -v pidof > /dev/null; then handle_failed_dependencies "\n\n[!] Please make sure pidof is installed and executable\n\n" fi - + # used for save_brm and defining columnstore_user if ! command -v stat > /dev/null; then handle_failed_dependencies "\n\n[!] Please make sure stat is installed and executable\n\n" @@ -564,22 +567,22 @@ check_for_dependancies() confirm_rsync_installed confirm_mariadb_backup_installed - if [ $1 == "backup" ] && [ $mode != "indirect" ] && ! command -v dbrmctl > /dev/null; then + if [ $1 == "backup" ] && [ $mode != "indirect" ] && ! command -v dbrmctl > /dev/null; then handle_failed_dependencies "\n\n[!] dbrmctl unreachable to issue lock \n\n" fi if [ $storage == "S3" ]; then - + # Default cloud cloud="aws" - + # Critical argument for S3 - determine which cloud if [ -z "$backup_bucket" ]; then handle_failed_dependencies "\n\n[!] Undefined --backup-bucket: $backup_bucket \nfor examples see: ./$0 backup --help\n\n"; fi if [[ $backup_bucket == gs://* ]]; then cloud="gcp"; protocol="gs"; elif [[ $backup_bucket == s3://* ]]; then cloud="aws"; protocol="s3"; - else + else handle_failed_dependencies "\n\n[!] Invalid --backup-bucket - doesnt lead with gs:// or s3:// - $backup_bucket\n\n"; fi @@ -591,7 +594,7 @@ check_for_dependancies() gsutil=$(which mcs_gsutil 2>/dev/null) elif ! command -v gsutil > /dev/null; then which gsutil - echo "Hints: + echo "Hints: curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-443.0.0-linux-x86_64.tar.gz tar -xvf google-cloud-cli-443.0.0-linux-x86_64.tar.gz ./google-cloud-sdk/install.sh -q @@ -603,13 +606,13 @@ check_for_dependancies() " handle_failed_dependencies "\n\n[!] Please make sure gsutil cli is installed configured and executable\n\n" - else + else gsutil=$(which gsutil 2>/dev/null) fi - + # gsutil sytax for silent if $quiet; then xtra_s3_args+="-q"; fi - else + else # on prem S3 will use aws cli # If AWS - Check aws-cli installed @@ -619,7 +622,7 @@ check_for_dependancies() awscli=$(which mcs_aws 2>/dev/null) elif ! command -v aws > /dev/null; then which aws - echo "Hints: + echo "Hints: curl \"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip\" -o \"awscliv2.zip\" unzip awscliv2.zip sudo ./aws/install @@ -629,15 +632,14 @@ check_for_dependancies() else awscli=$(which aws 2>/dev/null) fi - + # aws sytax for silent if $quiet; then xtra_s3_args+="--quiet"; fi fi; fi } -validation_prechecks_for_backup() -{ +validation_prechecks_for_backup() { echo "Prechecks" # Adjust rsync for incremental copy @@ -655,19 +657,19 @@ validation_prechecks_for_backup() # Detect if columnstore online if [ $mode == "direct" ]; then - if [ -z $(pidof PrimProc) ] || [ -z $(pidof WriteEngineServer) ]; then - printf " - Columnstore is OFFLINE \n"; - export columnstore_online=false; - else - printf " - Columnstore is ONLINE - safer if offline \n"; - export columnstore_online=true; + if [ -z $(pidof PrimProc) ] || [ -z $(pidof WriteEngineServer) ]; then + printf " - Columnstore is OFFLINE \n"; + export columnstore_online=false; + else + printf " - Columnstore is ONLINE - safer if offline \n"; + export columnstore_online=true; fi fi; # Validate compression option if [[ -n "$compress_format" ]]; then case $compress_format in - pigz) + pigz) confirm_pigz_installed ;; *) # unknown option @@ -685,66 +687,66 @@ validation_prechecks_for_backup() fi # Storage Based Checks - if [ $storage == "LocalStorage" ]; then + if [ $storage == "LocalStorage" ]; then # Incremental Job checks if $incremental; then # $backup_location must exist to find an existing full back to add to if [ ! -d $backup_location ]; then - handle_early_exit_on_backup "[X] Backup directory ($backup_location) DOES NOT exist ( -bl ) \n\n" true; + handle_early_exit_on_backup "[X] Backup directory ($backup_location) DOES NOT exist ( -bl ) \n\n" true; fi - if [ "$today" == "auto_most_recent" ]; then + if [ "$today" == "auto_most_recent" ]; then auto_select_most_recent_backup_for_incremental fi - + # Validate $today is a non-empty value - if [ -z "$today" ]; then + if [ -z "$today" ]; then handle_early_exit_on_backup "\nUndefined folder to increment on ($backup_location$today)\nTry --incremental or --incremental auto_most_recent \n" true fi # Cant continue if this folder (which represents a full backup) doesnt exists - if [ $backup_destination == "Local" ]; then - if [ -d $backup_location$today ]; then - printf " - Full backup directory exists\n"; - else - handle_early_exit_on_backup "[X] Full backup directory ($backup_location$today) DOES NOT exist \n\n" true; + if [ $backup_destination == "Local" ]; then + if [ -d $backup_location$today ]; then + printf " - Full backup directory exists\n"; + else + handle_early_exit_on_backup "[X] Full backup directory ($backup_location$today) DOES NOT exist \n\n" true; fi; elif [ $backup_destination == "Remote" ]; then - if [[ $(ssh $scp test -d $backup_location$today && echo exists) ]]; then - printf " - Full backup directory exists\n"; - else - handle_early_exit_on_backup "[X] Full backup directory ($backup_location$today) DOES NOT exist on remote $scp \n\n" true; + if [[ $(ssh $scp test -d $backup_location$today && echo exists) ]]; then + printf " - Full backup directory exists\n"; + else + handle_early_exit_on_backup "[X] Full backup directory ($backup_location$today) DOES NOT exist on remote $scp \n\n" true; fi fi fi elif [ $storage == "S3" ]; then - + # Adjust s3api flags for onpremise/custom endpoints add_s3_api_flags="" if [ -n "$s3_url" ]; then add_s3_api_flags+=" --endpoint-url $s3_url"; fi; if $no_verify_ssl; then add_s3_api_flags+=" --no-verify-ssl"; fi; - + # Validate addtional relevant arguments for S3 if [ -z "$backup_bucket" ]; then echo "Invalid --backup_bucket: $backup_bucket - is empty"; exit 1; fi # Check cli access to bucket if [ $cloud == "gcp" ]; then - + if $gsutil ls $backup_bucket > /dev/null ; then printf " - Success listing backup bucket\n" - else + else printf "\n[X] Failed to list bucket contents... \nCheck $gsutil credentials: $gsutil config -a" handle_early_exit_on_backup "\n$gsutil ls $backup_bucket \n\n" true fi - else + else # Check aws cli access to bucket if $( $awscli $add_s3_api_flags s3 ls $backup_bucket > /dev/null ) ; then printf " - Success listing backup bucket\n" - else + else printf "\n[X] Failed to list bucket contents... \nCheck aws cli credentials: aws configure" handle_early_exit_on_backup "\naws $add_s3_api_flags s3 ls $backup_bucket \n" true fi @@ -752,31 +754,31 @@ validation_prechecks_for_backup() # Incremental Job checks if $incremental; then - if [ "$today" == "auto_most_recent" ]; then + if [ "$today" == "auto_most_recent" ]; then auto_select_most_recent_backup_for_incremental fi - + # Validate $today is a non-empty value - if [ -z "$today" ]; then + if [ -z "$today" ]; then handle_early_exit_on_backup "\nUndefined folder to increment on ($backup_bucket/$today)\nTry --incremental or --incremental auto_most_recent \n" true fi # Cant continue if this folder (which represents a full backup) doesnt exists if [ $cloud == "gcp" ]; then - if [[ $( $gsutil ls $backup_bucket/$today | head ) ]]; then - printf " - Full backup directory exists\n"; - else - handle_early_exit_on_backup "[X] Full backup directory ($backup_bucket/$today) DOES NOT exist in GCS \nCheck - $gsutil ls $backup_bucket/$today | head \n\n" true; - fi + if [[ $( $gsutil ls $backup_bucket/$today | head ) ]]; then + printf " - Full backup directory exists\n"; + else + handle_early_exit_on_backup "[X] Full backup directory ($backup_bucket/$today) DOES NOT exist in GCS \nCheck - $gsutil ls $backup_bucket/$today | head \n\n" true; + fi else - if [[ $( $awscli $add_s3_api_flags s3 ls $backup_bucket/$today/ | head ) ]]; then - printf " - Full backup directory exists\n"; - else - handle_early_exit_on_backup "[X] Full backup directory ($backup_bucket/$today) DOES NOT exist in S3 \nCheck - aws $add_s3_api_flags s3 ls $backup_bucket/$today | head \n\n" true; - fi + if [[ $( $awscli $add_s3_api_flags s3 ls $backup_bucket/$today/ | head ) ]]; then + printf " - Full backup directory exists\n"; + else + handle_early_exit_on_backup "[X] Full backup directory ($backup_bucket/$today) DOES NOT exist in S3 \nCheck - aws $add_s3_api_flags s3 ls $backup_bucket/$today | head \n\n" true; + fi fi; fi - else + else handle_early_exit_on_backup "Invalid Variable storage: $storage" true fi } @@ -788,11 +790,11 @@ validation_prechecks_for_backup() auto_select_most_recent_backup_for_incremental() { printf " - Searching for most recent backup ...." - if [ $storage == "LocalStorage" ]; then + if [ $storage == "LocalStorage" ]; then most_recent_backup=$(ls -td "${backup_location}"* 2>/dev/null | head -n 1) if [[ -z "$most_recent_backup" ]]; then handle_early_exit_on_backup "\n[!!!] No backup found to increment in '$backup_location', please run a full backup or define a folder that exists --incremental \n" true - else + else today=$(basename $most_recent_backup 2>/dev/null) fi @@ -803,7 +805,7 @@ auto_select_most_recent_backup_for_incremental() { most_recent_backup_time_diff=$((2**63 - 1)); while IFS= read -r line; do - + folder=$(echo "$line" | awk '{print substr($2, 1, length($2)-1)}') date_time=$(s3ls "${backup_bucket}/${folder}/restore --recursive" | awk '{print $1,$2}') @@ -811,9 +813,9 @@ auto_select_most_recent_backup_for_incremental() { # Parse the date backup_date=$(date -d "$date_time" +%s) - + # Calculate the difference in days - time_diff=$(( (current_date - backup_date) )) + time_diff=$(( (current_date - backup_date) )) # echo "date_time: $date_time" # echo "backup_date: $backup_date" # echo "time_diff: $time_diff" @@ -826,14 +828,14 @@ auto_select_most_recent_backup_for_incremental() { fi printf "." done <<< "$backups" - + # printf "\n\nMost Recent: $most_recent_backup \n" # printf "Time Diff: $most_recent_backup_time_diff \n" if [[ -z "$most_recent_backup" ]]; then handle_early_exit_on_backup "\n[!!!] No backup found to increment, please run a full backup or define a folder that exists --incremental \n" true exit 1; - else + else today=$most_recent_backup fi fi @@ -841,7 +843,7 @@ auto_select_most_recent_backup_for_incremental() { } apply_backup_retention_policy() { - + if [ $retention_days -eq 0 ]; then printf " - Skipping Backup Rentention Policy\n" return 0; @@ -851,25 +853,25 @@ apply_backup_retention_policy() { if [ $storage == "LocalStorage" ]; then # example: find /tmp/backups/ -mindepth 1 -maxdepth 1 -type d -name "*" -amin +0 find "$backup_location" -mindepth 1 -maxdepth 1 -type d -name "*" -mtime +$retention_days -exec rm -r {} \; - - elif [ $storage == "S3" ]; then + + elif [ $storage == "S3" ]; then current_date=$(date +%s) backups=$(s3ls $backup_bucket) - + while IFS= read -r line; do delete_backup=false folder=$(echo "$line" | awk '{print substr($2, 1, length($2)-1)}') date_time=$(s3ls "${backup_bucket}/${folder}/restore --recursive" | awk '{print $1,$2}') - + if [[ -n "$date_time" ]]; then # Parse the date backup_date=$(date -d "$date_time" +%s) - + # Calculate the difference in days - days_diff=$(( (current_date - backup_date) / (60*60*24) )) + days_diff=$(( (current_date - backup_date) / (60*60*24) )) # echo "line: $line" # echo "date_time: $date_time" # echo "backup_date: $backup_date" @@ -905,7 +907,7 @@ cs_read_only_wait_loop() { printf "." current_status=$(dbrmctl status); if [ $? -ne 0 ]; then - handle_early_exit_on_backup "\n[!] Failed to get dbrmctl status\n\n" + handle_early_exit_on_backup "\n[!] Failed to get dbrmctl status\n\n" fi if [ $retry_counter -ge $retry_limit ]; then handle_early_exit_on_backup "\n[!] Set columnstore readonly wait retry limit exceeded: $retry_counter \n\n" @@ -913,21 +915,22 @@ cs_read_only_wait_loop() { ((retry_counter++)) done - printf "Done\n" + printf "Done\n\n" } -# Having columnstore offline is best for non-volatile backups +# Having columnstore offline is best for non-volatile backups # If online - issue a flush tables with read lock and set DBRM to readonly -issue_write_locks() +issue_write_locks() { if [ $mode == "indirect" ] || $skip_locks; then printf "\n"; return; fi; - if ! $skip_mdb && ! pidof mariadbd > /dev/null; then - handle_early_exit_on_backup "\n[X] MariaDB is offline ... Needs to be online to issue read only lock and to run mariadb-backup \n\n" true; + if ! $skip_mdb && ! pidof mariadbd > /dev/null; then + handle_early_exit_on_backup "\n[X] MariaDB is offline ... Needs to be online to issue read only lock and to run mariadb-backup \n\n" true; fi; - printf "\nLocks \n"; + printf "\nLocks \n"; # Pre 23.10.2 CS startreadonly doesnt exist - so poll cpimports added to protect them - if ! $skip_polls; then + # Consider removing when 23.10.x is EOL + if ! $skip_polls; then poll_check_no_active_sql_writes poll_check_no_active_cpimports fi; @@ -939,41 +942,42 @@ issue_write_locks() if mariadb -e "FLUSH TABLES WITH READ LOCK;"; then mariadb -e "set global read_only=ON;" read_lock=true - printf " Done\n"; - else - handle_early_exit_on_backup "\n[X] Failed issuing read-only lock\n" - fi + printf " Done\n"; + else + handle_early_exit_on_backup "\n[X] Failed issuing read-only lock\n" + fi fi - - if [ $pm == "pm1" ]; then - + + if [ $pm == "pm1" ]; then + # Set Columnstore ReadOnly Mode startreadonly_exists=$(dbrmctl -h 2>&1 | grep "startreadonly") printf " - Issuing read-only lock to Columnstore Engine ... "; - if ! $columnstore_online; then + if ! $columnstore_online; then printf "Skip since offline\n"; elif [ $DBROOT_COUNT == "1" ] && [[ -n "$startreadonly_exists" ]]; then if dbrmctl startreadonly ; then - cs_read_only_wait_loop - printf " \n"; + if ! $skip_polls; then + cs_read_only_wait_loop + fi else - handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via dbrmctl startreadonly\n" + handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via dbrmctl startreadonly\n" fi; - elif [ $DBROOT_COUNT == "1" ]; then + elif [ $DBROOT_COUNT == "1" ]; then if ! dbrmctl readonly ; then - handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via dbrmctl readonly \n" + handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via dbrmctl readonly \n" fi - else + else if command -v mcs &> /dev/null && command -v jq &> /dev/null ; then if ! mcs cluster set mode --mode readonly | jq -r tostring ; then - handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n" + handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n" fi else # Older CS versions dont have mcs cli cmapiResponse=$(curl -s -X PUT https://127.0.0.1:8640/cmapi/0.4.0/cluster/mode-set --header 'Content-Type:application/json' --header "x-api-key:$cmapi_key" --data '{"timeout":20, "mode": "readonly"}' -k); - if [[ $cmapiResponse == '{"error":'* ]] ; then - handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n" + if [[ $cmapiResponse == '{"error":'* ]] ; then + handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n" else printf "$cmapiResponse \n"; fi @@ -981,7 +985,6 @@ issue_write_locks() fi else cs_read_only_wait_loop - printf " \n"; fi } @@ -1023,7 +1026,7 @@ poll_check_no_active_cpimports() { printf "Done\n" no_cpimports=true break - else + else printf "." if ! $quiet; then printf "\n$active_cpimports"; fi; sleep "$poll_interval" @@ -1036,8 +1039,7 @@ poll_check_no_active_cpimports() { fi; } -run_save_brm() -{ +run_save_brm() { if $skip_save_brm || [ $pm != "pm1" ] || ! $columnstore_online || [ $mode == "indirect" ]; then return; fi; @@ -1047,46 +1049,45 @@ run_save_brm() # Copy extent map locally just in case save_brm fails if [ $storage == "LocalStorage" ]; then - printf " - Backing up DBRMs @ $tmp_dir ... " + printf " - Backing up DBRMs @ $tmp_dir ... " cp -R $DBRM_PATH/* $tmp_dir - printf " Done \n" - elif [ $storage == "S3" ]; then + printf " Done \n" + elif [ $storage == "S3" ]; then - printf " - Backing up minimal DBRMs @ $tmp_dir ... " + printf " - Backing up minimal DBRMs @ $tmp_dir ... " # Base Set - eval "smcat /data1/systemFiles/dbrm/BRM_saves_current 2>/dev/null > $tmp_dir/BRM_saves_current $xtra_cmd_args" - eval "smcat /data1/systemFiles/dbrm/BRM_saves_journal 2>/dev/null > $tmp_dir/BRM_saves_journal $xtra_cmd_args" - eval "smcat /data1/systemFiles/dbrm/BRM_saves_em 2>/dev/null > $tmp_dir/BRM_saves_em $xtra_cmd_args" - eval "smcat /data1/systemFiles/dbrm/BRM_saves_vbbm 2>/dev/null > $tmp_dir/BRM_saves_vbbm $xtra_cmd_args" - eval "smcat /data1/systemFiles/dbrm/BRM_saves_vss 2>/dev/null > $tmp_dir/BRM_saves_vss $xtra_cmd_args" + smcat /data1/systemFiles/dbrm/BRM_saves_current 2>/dev/null > $tmp_dir/BRM_saves_current + smcat /data1/systemFiles/dbrm/BRM_saves_journal 2>/dev/null > $tmp_dir/BRM_saves_journal + smcat /data1/systemFiles/dbrm/BRM_saves_em 2>/dev/null > $tmp_dir/BRM_saves_em + smcat /data1/systemFiles/dbrm/BRM_saves_vbbm 2>/dev/null > $tmp_dir/BRM_saves_vbbm + smcat /data1/systemFiles/dbrm/BRM_saves_vss 2>/dev/null > $tmp_dir/BRM_saves_vss # A Set - eval "smcat /data1/systemFiles/dbrm/BRM_savesA_em 2>/dev/null > $tmp_dir/BRM_savesA_em $xtra_cmd_args" - eval "smcat /data1/systemFiles/dbrm/BRM_savesA_vbbm 2>/dev/null > $tmp_dir/BRM_savesA_vbbm $xtra_cmd_args" - eval "smcat /data1/systemFiles/dbrm/BRM_savesA_vss 2>/dev/null > $tmp_dir/BRM_savesA_vss $xtra_cmd_args" - + smcat /data1/systemFiles/dbrm/BRM_savesA_em 2>/dev/null > $tmp_dir/BRM_savesA_em + smcat /data1/systemFiles/dbrm/BRM_savesA_vbbm 2>/dev/null > $tmp_dir/BRM_savesA_vbbm + smcat /data1/systemFiles/dbrm/BRM_savesA_vss 2>/dev/null > $tmp_dir/BRM_savesA_vss + # B Set - eval "smcat /data1/systemFiles/dbrm/BRM_savesB_em 2>/dev/null > $tmp_dir/BRM_savesB_em $xtra_cmd_args" - eval "smcat /data1/systemFiles/dbrm/BRM_savesB_vbbm 2>/dev/null > $tmp_dir/BRM_savesB_vbbm $xtra_cmd_args" - eval "smcat /data1/systemFiles/dbrm/BRM_savesB_vss 2>/dev/null > $tmp_dir/BRM_savesB_vss $xtra_cmd_args" - printf " Done \n" + smcat /data1/systemFiles/dbrm/BRM_savesB_em 2>/dev/null > $tmp_dir/BRM_savesB_em + smcat /data1/systemFiles/dbrm/BRM_savesB_vbbm 2>/dev/null > $tmp_dir/BRM_savesB_vbbm + smcat /data1/systemFiles/dbrm/BRM_savesB_vss 2>/dev/null > $tmp_dir/BRM_savesB_vss + printf " Done \n" fi - + printf " - Saving BRMs... \n" brm_owner=$(stat -c "%U" $DBRM_PATH/) - if sudo -u $brm_owner /usr/bin/save_brm ; then + if sudo -u $brm_owner /usr/bin/save_brm ; then rm -rf $tmp_dir - else - printf "\n Failed: save_brm error - see /var/log/messages - backup @ $tmpDir \n\n"; - handle_early_exit_on_backup + else + printf "\n Failed: save_brm error - see /var/log/messages - backup @ $tmpDir \n\n"; + handle_early_exit_on_backup fi - + } # Example: s3sync # Example: s3sync s3://$bucket $backup_bucket/$today/columnstoreData "Done - Columnstore data sync complete" -s3sync() -{ +s3sync() { local from=$1 local to=$2 @@ -1094,14 +1095,14 @@ s3sync() local failed_message=$4 local retries=${5:-0} local cmd="" - + if [ $cloud == "gcp" ]; then cmd="$gsutil $xtra_s3_args -m rsync -r -d $from $to" # gsutil throws WARNINGS if not directed to /dev/null if $quiet; then cmd+=" 2>/dev/null"; fi eval "$cmd" - else + else # Default AWS cmd="$awscli $xtra_s3_args $add_s3_api_flags s3 sync $from $to" $cmd @@ -1112,7 +1113,7 @@ s3sync() if [ $exit_code -eq 0 ]; then if [ -n "$success_message" ]; then printf "$success_message"; fi; else - if [ $retries -lt $RETRY_LIMIT ]; then + if [ $retries -lt $RETRY_LIMIT ]; then echo "$cmd" echo "Retrying: $retries" sleep 1; @@ -1127,15 +1128,14 @@ s3sync() } # Example: s3cp -s3cp() -{ +s3cp() { local from=$1 local to=$2 local cmd="" - + if [ $cloud == "gcp" ]; then cmd="$gsutil $xtra_s3_args cp $from $to" - else + else # Default AWS cmd="$awscli $xtra_s3_args $add_s3_api_flags s3 cp $from $to" fi; @@ -1144,15 +1144,14 @@ s3cp() } -# Example: s3rm -s3rm() -{ +# Example: s3rm +s3rm() { local path=$1 local cmd="" - + if [ $cloud == "gcp" ]; then cmd="$gsutil $xtra_s3_args -m rm -r $path" - else + else # Default AWS cmd="$awscli $xtra_s3_args $add_s3_api_flags s3 rm $path --recursive" fi; @@ -1160,15 +1159,14 @@ s3rm() $cmd } -# Example: s3ls -s3ls() -{ +# Example: s3ls +s3ls() { local path=$1 local cmd="" - + if [ $cloud == "gcp" ]; then cmd="$gsutil ls $path" - else + else # Default AWS cmd="$awscli $add_s3_api_flags s3 ls $path" fi; @@ -1176,19 +1174,18 @@ s3ls() $cmd } -clear_read_lock() -{ +clear_read_lock() { if [ $mode == "indirect" ] || $skip_locks; then return; fi; - printf "\nClearing Locks\n"; - # Clear MDB Lock + printf "\nClearing Locks\n"; + # Clear MDB Lock if pidof mariadbd > /dev/null && [ $read_lock ]; then printf " - Clearing read-only lock on MariaDB Server ... "; if mariadb -e "UNLOCK TABLES;" && mariadb -qsNe "set global read_only=$ORIGINAL_READONLY_STATUS;"; then read_lock=false; printf " Done\n" - else + else handle_early_exit_on_backup "\n[X] Failed clearing readLock\n" true fi fi @@ -1196,31 +1193,30 @@ clear_read_lock() if [ $pm == "pm1" ]; then # Clear CS Lock - printf " - Clearing read-only lock on Columnstore Engine ... "; - if ! $columnstore_online; then + printf " - Clearing read-only lock on Columnstore Engine ... "; + if ! $columnstore_online; then printf "Skip since offline\n" - elif [ $DBROOT_COUNT == "1" ]; then + elif [ $DBROOT_COUNT == "1" ]; then if dbrmctl readwrite ; then printf " "; - else + else handle_early_exit_on_backup "\n[X] Failed clearing columnstore BRM lock via dbrmctl\n" true; fi elif command -v mcs &> /dev/null && command -v jq &> /dev/null ; then if ! mcs cluster set mode --mode readwrite | jq -r tostring ;then - handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n"; + handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n"; fi - elif curl -s -X PUT https://127.0.0.1:8640/cmapi/0.4.0/cluster/mode-set --header 'Content-Type:application/json' --header "x-api-key:$cmapi_key" --data '{"timeout":20, "mode": "readwrite"}' -k ; then + elif curl -s -X PUT https://127.0.0.1:8640/cmapi/0.4.0/cluster/mode-set --header 'Content-Type:application/json' --header "x-api-key:$cmapi_key" --data '{"timeout":20, "mode": "readwrite"}' -k ; then printf " \n" else handle_early_exit_on_backup "\n[X] Failed clearing columnstore BRM lock\n" true; fi fi - -} + +} # handle_ called when certain checks/functionality fails -handle_failed_dependencies() -{ +handle_failed_dependencies() { printf "\nDependency Checks Failed: $1" alert "$1" exit 1; @@ -1228,8 +1224,7 @@ handle_failed_dependencies() # first argument is the error message # 2nd argument true = skip clear_read_lock, false= dont skip -handle_early_exit_on_backup() -{ +handle_early_exit_on_backup() { skip_clear_locks=${2:-false} if ! $skip_clear_locks; then clear_read_lock; fi; printf "\nBackup Failed: $1\n" @@ -1237,8 +1232,7 @@ handle_early_exit_on_backup() exit 1; } -handle_early_exit_on_restore() -{ +handle_early_exit_on_restore() { printf "\nRestore Failed: $1\n" alert "$1" exit 1; @@ -1267,11 +1261,11 @@ wait_on_rsync() local total=0 local w=0; - while [ $rsyncInProgress -gt "$concurrentThreshHold" ]; do - if ! $quiet && $visualize && [ $(($w % 10)) -eq 0 ]; then - if [[ $total == 0 ]]; then - total=$(du -sh /var/lib/columnstore/data$dbrootToSync/) - else + while [ $rsyncInProgress -gt "$concurrentThreshHold" ]; do + if ! $quiet && $visualize && [ $(($w % 10)) -eq 0 ]; then + if [[ $total == 0 ]]; then + total=$(du -sh /var/lib/columnstore/data$dbrootToSync/) + else echo -E "$rsyncInProgress rsync processes running ... seconds: $w" echo "Status: $(du -sh $backup_location$today/data$dbrootToSync/)" echo "Goal: $total" @@ -1285,12 +1279,11 @@ wait_on_rsync() done } -initiate_rsyncs() -{ +initiate_rsyncs() { local dbrootToSync=$1 parallel_rysnc_flags=" -a " if $incremental ; then parallel_rysnc_flags+=" --inplace --no-whole-file --delete"; fi; - + deepParallelRsync /var/lib/columnstore/data$dbrootToSync 1 $DEPTH data$dbrootToSync & sleep 2; #jobs @@ -1298,19 +1291,18 @@ initiate_rsyncs() wait } -# A recursive function that increments depthCurrent+1 each directory it goes deeper and issuing rsync on each directory remaing at the target depth -# Example values: +# A recursive function that increments depthCurrent+1 each directory it goes deeper and issuing rsync on each directory remaing at the target depth +# Example values: # path: /var/lib/columnstore/data1 # depthCurrent: 1 # depthTarget: 3 # depthCurrent: data1 -deepParallelRsync() -{ +deepParallelRsync() { path=$1 depthCurrent=$2 depthTarget=$3 relativePath=$4 - # echo "DEBUG: + # echo "DEBUG: # path=$1 # depthCurrent=$2 # depthTarget=$3 @@ -1330,15 +1322,15 @@ deepParallelRsync() #echo "DEBUG - copy to relative: $backup_location$today/$relativePath/" if ls $fullFilePath | xargs -P $PARALLEL_THREADS -I {} rsync $parallel_rysnc_flags $fullFilePath/{} $backup_location$today/$relativePath/$fileName ; then echo " + Completed: $backup_location$today/$relativePath/$fileName" - else + else echo "Failed: $backup_location$today/$relativePath/$fileName" exit 1; fi - + else - # echo "DEBUG - Fork Deeper - $fullFilePath " + # echo "DEBUG - Fork Deeper - $fullFilePath " wait_on_rsync false "0.5" $PARALLEL_FOLDERS - # Since target depth not reached, recursively call for each directory + # Since target depth not reached, recursively call for each directory deepParallelRsync $fullFilePath "$((depthCurrent+1))" $depthTarget "$relativePath/$fileName" & fi @@ -1346,8 +1338,8 @@ deepParallelRsync() elif [ -f $fullFilePath ]; then rsync $additional_rysnc_flags $fullFilePath $backup_location$today/$relativePath/ - # If filename is * then the directory is empty - elif [ "$fileName" == "*" ]; then + # If filename is * then the directory is empty + elif [ "$fileName" == "*" ]; then # echo "DEBUG - Skipping $relativePath - empty"; continue else @@ -1357,9 +1349,8 @@ deepParallelRsync() wait } -run_backup() -{ - if [ $storage == "LocalStorage" ]; then +run_backup() { + if [ $storage == "LocalStorage" ]; then if [ $backup_destination == "Local" ]; then printf "\nLocal Storage Backup\n" @@ -1370,34 +1361,34 @@ run_backup() mkdir -p $backup_location$today else mkdir -p $backup_location$today/mysql - mkdir -p $backup_location$today/configs + mkdir -p $backup_location$today/configs mkdir -p $backup_location$today/configs/mysql - + # Check/Create CS Data Directories i=1 while [ $i -le $DBROOT_COUNT ]; do if [[ $ASSIGNED_DBROOT == "$i" || $HA == true ]]; then mkdir -p $backup_location$today/data$i ; fi ((i++)) - done + done fi - printf " Done\n" - + printf " Done\n" + # Backup Columnstore data i=1 while [ $i -le $DBROOT_COUNT ]; do - if [[ $ASSIGNED_DBROOT == "$i" || $HA == true ]]; then + if [[ $ASSIGNED_DBROOT == "$i" || $HA == true ]]; then if [[ -n "$compress_format" ]]; then # For compression keep track of dirs & files to include and compress/stream in the end - $compress_paths - compress_paths="/var/lib/columnstore/data$i/* " + compress_paths="/var/lib/columnstore/data$i/* " - elif $parrallel_rsync ; then + elif $parrallel_rsync ; then printf " - Parallel Rsync CS Data$i... \n" initiate_rsyncs $i - printf " Done\n" + printf " Done\n" else printf " - Syncing Columnstore Data$i... " eval "rsync $additional_rysnc_flags /var/lib/columnstore/data$i/* $backup_location$today/data$i/ $xtra_cmd_args"; - printf " Done\n" + printf " Done\n" fi; fi ((i++)) @@ -1405,7 +1396,7 @@ run_backup() # Backup MariaDB data if [ $ASSIGNED_DBROOT == "1" ]; then - + # logic to increment mysql and keep count if we want to backup incremental mysql data # i=1 # latestMysqlIncrement=0 @@ -1414,7 +1405,7 @@ run_backup() # done # MariaDB backup wont rerun if folder exists - so clear it before running mariadb-backup - if [ -d $backup_location$today/mysql ]; then + if [ -d $backup_location$today/mysql ]; then rm -rf $backup_location$today/mysql fi @@ -1425,20 +1416,20 @@ run_backup() mbd_prefix="$backup_location$today/$split_file_mdb_prefix.$compress_format" case $compress_format in pigz) - # Handle Cloud + # Handle Cloud if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $mbd_prefix 2>> $logfile; then handle_early_exit_on_backup "\nFailed mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $mbd_prefix 2>> $logfile \n" fi printf " Done @ $mbd_prefix\n" ;; *) # unknown option - handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" esac else printf " - Copying MariaDB Data... " if eval "mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root $xtra_cmd_args" ; then printf " Done \n"; else printf "\n Failed: mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root\n"; handle_early_exit_on_backup; fi fi - else + else echo "[!] Skipping mariadb-backup" fi @@ -1454,7 +1445,7 @@ run_backup() rm -rf $CS_CONFIGS_PATH/$col_xml_backup rm -rf $CS_CONFIGS_PATH/$cmapi_backup cp $CS_CONFIGS_PATH/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup - cp $CS_CONFIGS_PATH/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup + if [ -f $CS_CONFIGS_PATH/cmapi_server.conf ]; then cp $CS_CONFIGS_PATH/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup; fi; compress_paths+=" $CS_CONFIGS_PATH/$col_xml_backup $STORAGEMANGER_CNF $CS_CONFIGS_PATH/$cmapi_backup" fi; fi @@ -1470,13 +1461,13 @@ run_backup() compress_paths+=" $MARIADB_SERVER_CONFIGS_PATH/*" fi fi - + # Handle compression for Columnstore Data & Configs if [[ -n "$compress_format" ]]; then cs_prefix="$backup_location$today/$split_file_cs_prefix.$compress_format" compressed_split_size="250M"; case $compress_format in - pigz) + pigz) printf " - Compressing CS Data & Configs... " if ! eval "tar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $cs_prefix 2>> $logfile"; then handle_early_exit_on_backup "[!] - Compression Failed \ntar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $cs_prefix 2>> $logfile \n" @@ -1484,26 +1475,26 @@ run_backup() printf " Done @ $cs_prefix\n" ;; *) # unknown option - handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" esac fi - if $incremental ; then + if $incremental ; then # Log each incremental run now=$(date "+%m-%d-%Y %H:%M:%S"); echo "$pm updated on $now" >> $backup_location$today/incrementallyUpdated.txt final_message="Incremental Backup Complete" - else + else # Create restore job file extra_flags="" - if [[ -n "$compress_format" ]]; then extra_flags+=" -c $compress_format"; fi; - if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; + if [[ -n "$compress_format" ]]; then extra_flags+=" -c $compress_format"; fi; + if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; echo "./$0 restore -l $today -bl $backup_location -bd $backup_destination -s $storage --dbroots $DBROOT_COUNT -m $mode $extra_flags --quiet" > $backup_location$today/restore.job fi final_message+=" @ $backup_location$today" - + elif [ $backup_destination == "Remote" ]; then - + # Check/Create Directories on remote server printf "[+] Checking Remote Directories... " i=1 @@ -1515,16 +1506,16 @@ run_backup() echo $makeDirectories ssh $scp " mkdir -p $backup_location$today/mysql ; $makeDataDirectories - mkdir -p $backup_location$today/configs ; - mkdir -p $backup_location$today/configs/mysql; + mkdir -p $backup_location$today/configs ; + mkdir -p $backup_location$today/configs/mysql; mkdir -p $backup_location$today/configs/mysql/$pm " printf " Done\n" - + printf "[~] Rsync Remote Columnstore Data... \n" i=1 while [ $i -le $DBROOT_COUNT ]; do #if [ $pm == "pm$i" ]; then rsync -a $additional_rysnc_flags /var/lib/columnstore/data$i/* $scp:$backup_location$today/data$i/ ; fi - if [ $pm == "pm$i" ]; then + if [ $pm == "pm$i" ]; then find /var/lib/columnstore/data$i/* -mindepth 1 -maxdepth 2 | xargs -P $PARALLEL_THREADS -I {} rsync $additional_rysnc_flags /var/lib/columnstore/data$i/* $scp:$backup_location$today/data$i/ fi ((i++)) @@ -1543,27 +1534,27 @@ run_backup() if ! $skip_mdb; then printf "[~] Backing up mysql... \n" mkdir -p $backup_location$today - if mariadb-backup --backup --target-dir=$backup_location$today --user=root ; then + if mariadb-backup --backup --target-dir=$backup_location$today --user=root ; then rsync -a $backup_location$today/* $scp:$backup_location$today/mysql rm -rf $backup_location$today - else - printf "\n Failed: mariadb-backup --backup --target-dir=$backup_location$today --user=root\n\n"; - handle_early_exit_on_backup + else + printf "\n Failed: mariadb-backup --backup --target-dir=$backup_location$today --user=root\n\n"; + handle_early_exit_on_backup fi - else + else echo "[!] Skipping mariadb-backup" fi; - + # Backup CS configurations - rsync $additional_rysnc_flags $CS_CONFIGS_PATH/Columnstore.xml $scp:$backup_location$today/configs/ + rsync $additional_rysnc_flags $CS_CONFIGS_PATH/Columnstore.xml $scp:$backup_location$today/configs/ rsync $additional_rysnc_flags $STORAGEMANGER_CNF $scp:$backup_location$today/configs/ - if [ -f $CS_CONFIGS_PATH/cmapi_server.conf ]; then - rsync $additional_rysnc_flags $CS_CONFIGS_PATH/cmapi_server.conf $scp:$backup_location$today/configs/; + if [ -f $CS_CONFIGS_PATH/cmapi_server.conf ]; then + rsync $additional_rysnc_flags $CS_CONFIGS_PATH/cmapi_server.conf $scp:$backup_location$today/configs/; fi fi - - - if $incremental ; then + + + if $incremental ; then now=$(date "+%m-%d-%Y +%H:%M:%S") ssh $scp "echo \"$pm updated on $now\" >> $backup_location$today/incrementallyUpdated.txt" final_message="Incremental Backup Complete" @@ -1584,8 +1575,20 @@ run_backup() max_wait=180 printf " - Checking storagemanager/journal/data$ASSIGNED_DBROOT/* " while [[ $j_counts -gt 0 ]]; do - if [ $i -gt $max_wait ]; then printf "[!] Maybe you have orphaned journal files, active writes or an unreachable bucket \n"; handle_early_exit_on_backup "\n[!] max_wait exceeded for $cs_journal/data$ASSIGNED_DBROOT/* to sync with bucket "; ls -la $cs_journal/data$ASSIGNED_DBROOT/*; fi; - if (( $i%10 == 0 )); then printf "\n[!] Not empty yet - found $j_counts files @ $cs_journal/data$ASSIGNED_DBROOT/*\n"; printf " - Checking storagemanager/journal/data$ASSIGNED_DBROOT/* "; fi; + + # Handle when max_wait exceeded + if [ $i -gt $max_wait ]; then + printf "[!] Maybe you have orphaned journal files, active writes or an unreachable bucket \n"; + handle_early_exit_on_backup "\n[!] max_wait exceeded for $cs_journal/data$ASSIGNED_DBROOT/* to sync with bucket "; + ls -la $cs_journal/data$ASSIGNED_DBROOT/*; + fi; + + # Print every 10 seconds + if (( $i%10 == 0 )); then + printf "\n[!] Not empty yet - found $j_counts files @ $cs_journal/data$ASSIGNED_DBROOT/*\n"; + printf " - Checking storagemanager/journal/data$ASSIGNED_DBROOT/* "; + fi; + sleep 1 i=$(($i+1)) j_counts=$(find $cs_journal/data$ASSIGNED_DBROOT/* -type f 2>/dev/null | wc -l) @@ -1597,14 +1600,14 @@ run_backup() if [[ -z "$compress_format" ]]; then printf " - Syncing Columnstore Metadata \n" - if $HA; then + if $HA; then s3sync $STORAGEMANAGER_PATH $backup_bucket/$today/storagemanager " - Done storagemanager/*\n" "\n\n[!!!] sync failed - storagemanager/*\n\n"; - else + else s3sync $cs_cache/data$ASSIGNED_DBROOT $backup_bucket/$today/storagemanager/cache/data$ASSIGNED_DBROOT " + cache/data$ASSIGNED_DBROOT\n" "\n\n[!!!] sync failed - cache/data$ASSIGNED_DBROOT\n\n"; s3sync $cs_metadata/data$ASSIGNED_DBROOT $backup_bucket/$today/storagemanager/metadata/data$ASSIGNED_DBROOT " + metadata/data$ASSIGNED_DBROOT\n" "\n\n[!!!] sync failed - metadata/data$ASSIGNED_DBROOT\n\n" s3sync $cs_journal/data$ASSIGNED_DBROOT $backup_bucket/$today/storagemanager/journal/data$ASSIGNED_DBROOT " + journal/data$ASSIGNED_DBROOT\n" "\n\n[!!!] sync failed - journal/data$ASSIGNED_DBROOT\n\n" fi; - + else # For compression keep track of dirs & files to include and compress/stream in the end - $compress_paths compress_paths="$cs_cache/data$ASSIGNED_DBROOT $cs_metadata/data$ASSIGNED_DBROOT $cs_journal/data$ASSIGNED_DBROOT " @@ -1612,7 +1615,7 @@ run_backup() # PM1 mostly backups everything else if [ $ASSIGNED_DBROOT == "1" ]; then - + # Backup CS configurations if [[ -z "$compress_format" ]]; then s3sync $CS_CONFIGS_PATH/ $backup_bucket/$today/configs/ " + $CS_CONFIGS_PATH/\n" "\n\n[!!!] sync failed - $CS_CONFIGS_PATH/\n\n"; @@ -1624,7 +1627,7 @@ run_backup() if ! $skip_bucket_data; then printf " - Saving Columnstore data ... " s3sync $protocol://$bucket $backup_bucket/$today/columnstoreData " Done \n" - else + else printf " - [!] Skipping columnstore bucket data \n" fi @@ -1636,12 +1639,12 @@ run_backup() mbd_prefix="$backup_bucket/$today/$split_file_mdb_prefix.$compress_format" case $compress_format in pigz) - # Handle Cloud + # Handle Cloud if [ $cloud == "gcp" ]; then if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter="gsutil cp - ${mbd_prefix}_\$FILE 2>$logfile" - chunk 2>$logfile; then handle_early_exit_on_backup "\nFailed mariadb-backup --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter=\"gsutil cp - ${mbd_prefix}_\$FILE 2>$logfile\" - chunk 2>$logfile \n" fi - + elif [ $cloud == "aws" ]; then if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter="aws s3 cp - ${mbd_prefix}_\$FILE 2>$logfile 1>&2" - chunk 2>$logfile; then handle_early_exit_on_backup "\nFailed mariadb-backup --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter=\"aws s3 cp - ${mbd_prefix}_\$FILE 2>$logfile 1>&2\" - chunk 2>$logfile\n" @@ -1650,7 +1653,7 @@ run_backup() printf " Done @ $mbd_prefix\n" ;; *) # unknown option - handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac else @@ -1667,11 +1670,11 @@ run_backup() s3rm $backup_bucket/$today/mysql/ s3sync $backup_location$today/mysql $backup_bucket/$today/mysql/ " + mariadb-backup to bucket @ $backup_bucket/$today/mysql/ \n" rm -rf $backup_location$today/mysql - else + else handle_early_exit_on_backup "\nFailed making directory for MariaDB backup\ncommand: mkdir -p $backup_location$today/mysql\n" fi fi - else + else echo "[!] Skipping mariadb-backup" fi fi @@ -1691,13 +1694,13 @@ run_backup() cs_prefix="$backup_bucket/$today/$split_file_cs_prefix.$compress_format" compressed_split_size="250M"; case $compress_format in - pigz) + pigz) printf " - Compressing CS local files -> bucket ... " if [ $cloud == "gcp" ]; then if ! eval "tar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b $compressed_split_size --filter=\"gsutil cp - ${cs_prefix}_\\\$FILE 2>$logfile 1>&2\" - chunk 2>$logfile"; then handle_early_exit_on_backup "[!] - Compression/Split/Upload Failed \n" fi - + elif [ $cloud == "aws" ]; then if ! eval "tar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b $compressed_split_size --filter=\"aws s3 cp - ${cs_prefix}_\\\$FILE 2>$logfile 1>&2\" - chunk 2>$logfile"; then handle_early_exit_on_backup "[!] - Compression/Split/Upload Failed \n" @@ -1706,15 +1709,15 @@ run_backup() printf " Done @ $cs_prefix\n" ;; *) # unknown option - handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac fi # Create restore job file & update incremental log - if $incremental ; then + if $incremental ; then now=$(date "+%m-%d-%Y +%H:%M:%S") increment_file="incremental_history.txt" - + # download S3 file, append to it and reupload if [[ $(s3ls "$backup_bucket/$today/$increment_file") ]]; then s3cp $backup_bucket/$today/$increment_file $increment_file @@ -1727,18 +1730,18 @@ run_backup() else # Create restore job file extra_flags="" - if [[ -n "$compress_format" ]]; then extra_flags+=" -c $compress_format"; fi; - if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; - if $skip_bucket_data; then extra_flags+=" --skip-bucket-data"; fi; + if [[ -n "$compress_format" ]]; then extra_flags+=" -c $compress_format"; fi; + if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; + if $skip_bucket_data; then extra_flags+=" --skip-bucket-data"; fi; if [ -n "$s3_url" ]; then extra_flags+=" -url $s3_url"; fi; echo "./$0 restore -l $today -s $storage -bb $backup_bucket -dbs $DBROOT_COUNT -m $mode -nb $protocol://$bucket $extra_flags --quiet --continue" > restoreS3.job s3cp restoreS3.job $backup_bucket/$today/restoreS3.job rm -rf restoreS3.job fi final_message+=" @ $backup_bucket/$today" - + fi - + clear_read_lock end=$(date +%s) runtime=$((end-start)) @@ -1746,8 +1749,7 @@ run_backup() printf "$final_message\n\n" } -load_default_restore_variables() -{ +load_default_restore_variables() { # What date folder to load from the backup_location load_date='' @@ -1765,20 +1767,20 @@ load_default_restore_variables() scp="" # What storage topogoly was used by Columnstore in the backup - found in storagemanager.cnf - # Options: "LocalStorage" or "S3" + # Options: "LocalStorage" or "S3" storage="LocalStorage" # Flag for high available systems (meaning shared storage exists supporting the topology so that each node sees all data) HA=false # When set to true skips the enforcement that new_bucket should be empty prior to starting a restore - continue=false + continue=false # modes ['direct','indirect'] - direct backups run on the columnstore nodes themselves. indirect run on another machine that has read-only mounts associated with columnstore/mariadb mode="direct" - # Name of the Configuration file to load variables from - config_file=".cs-backup-config" + # Name of the Configuration file to load variables from - relative or full path accepted + config_file="" # Only used if storage=S3 # Name of the bucket to copy into to store the backup S3 data @@ -1842,8 +1844,7 @@ load_default_restore_variables() rsync_flags=" -av"; } -parse_restore_variables() -{ +parse_restore_variables() { # Dynamic Arguments while [[ $# -gt 0 ]]; do key="$1" @@ -1933,6 +1934,9 @@ parse_restore_variables() ;; -f|--config-file) config_file="$2" + if [ "$config_file" == "" ] || [ ! -f "$config_file" ]; then + handle_early_exit_on_restore "\n[!!!] Invalid Configuration File: $config_file\n" + fi shift # past argument shift # past value ;; @@ -1959,7 +1963,7 @@ parse_restore_variables() shift # past argument ;; -nv-ssl| --no-verify-ssl) - no_verify_ssl=true + no_verify_ssl=true shift # past argument ;; -h|--help|-help|help) @@ -1972,8 +1976,10 @@ parse_restore_variables() done # Load config file - if [ -f $config_file ]; then - source $config_file + if [ -n "$config_file" ] && [ -f "$config_file" ]; then + if ! source "$config_file"; then + handle_early_exit_on_restore "\n[!!!] Failed to source configuration file: $config_file\n" true + fi fi } @@ -1992,12 +1998,12 @@ print_restore_help_text() -nv-ssl| --no-verify-ssl) Skips verifying ssl certs, useful for onpremise s3 storage -s | --storage The storage used by columnstore data 'LocalStorage' or 'S3' -pm | --nodeid Forces the handling of the restore as this node as opposed to whats detected on disk - -nb | --new-bucket Defines the new bucket to copy the s3 data to from the backup bucket. + -nb | --new-bucket Defines the new bucket to copy the s3 data to from the backup bucket. Use -nb if the new restored cluster should use a different bucket than the backup bucket itself - -nr | --new-region Defines the region of the new bucket to copy the s3 data to from the backup bucket - -nk | --new-key Defines the aws key to connect to the new_bucket - -ns | --new-secret Defines the aws secret of the aws key to connect to the new_bucket - -f | --config-file Path to backup configuration file to load variables from + -nr | --new-region Defines the region of the new bucket to copy the s3 data to from the backup bucket + -nk | --new-key Defines the aws key to connect to the new_bucket + -ns | --new-secret Defines the aws secret of the aws key to connect to the new_bucket + -f | --config-file Path to backup configuration file to load variables from - see load_default_restore_variables() for defaults -cont| --continue This acknowledges data in your --new_bucket is ok to delete when restoring S3 -smdb| --skip-mariadb-backup Skip restoring mariadb server via mariadb-backup - ideal for only restoring columnstore -sb | --skip-bucket-data Skip restoring columnstore data in the bucket - ideal if looking to only restore mariadb server @@ -2006,12 +2012,12 @@ print_restore_help_text() -P | --parallel Number of parallel decompression and mbstream threads to run -ha | --highavilability Hint for if shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) - HA S3 ( /var/lib/columnstore/storagemanager/ ) + HA S3 ( /var/lib/columnstore/storagemanager/ ) Local Storage Examples: ./$0 restore -s LocalStorage -bl /tmp/backups/ -bd Local -l 12-29-2021 ./$0 restore -s LocalStorage -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -l 12-29-2021 - + S3 Storage Examples: ./$0 restore -s S3 -bb s3://my-cs-backups -l 12-29-2021 ./$0 restore -s S3 -bb gs://on-premise-bucket -l 12-29-2021 -url http://127.0.0.1:8000 @@ -2027,11 +2033,10 @@ print_restore_variables() { printf "\nRestore Variables\n" - if [ -f $config_file ]; then + if [ -n "$config_file" ] && [ -f "$config_file" ]; then printf "%-${s1}s %-${s2}s\n" "Configuration File:" "$config_file"; - source $config_file fi - if [ $mode == "indirect" ] || $skip_mdb || $skip_bucket_data; then + if [ $mode == "indirect" ] || $skip_mdb || $skip_bucket_data; then echo "------------------------------------------------" echo "Skips: MariaDB($skip_mdb) Bucket($skip_bucket_data)"; fi; @@ -2043,7 +2048,7 @@ print_restore_variables() else echo "Compression: false" fi - if [ $storage == "LocalStorage" ]; then + if [ $storage == "LocalStorage" ]; then echo "Backup Location: $backup_location" echo "Backup Destination: $backup_destination" echo "Scp: $scp" @@ -2055,7 +2060,7 @@ print_restore_variables() echo "PM Number: $pm_number" elif [ $storage == "S3" ]; then - echo "Backup Location: $backup_location" + echo "Backup Location: $backup_location" echo "Storage: $storage" echo "Load Date: $load_date" echo "timestamp: $(date +%m-%d-%Y-%H%M%S)" @@ -2078,7 +2083,7 @@ print_restore_variables() validation_prechecks_for_restore() { echo "Prechecks ..." - if [ $storage != "LocalStorage" ] && [ $storage != "S3" ]; then handle_early_exit_on_restore "Invalid script variable storage: $storage\n"; fi + if [ $storage != "LocalStorage" ] && [ $storage != "S3" ]; then handle_early_exit_on_restore "Invalid script variable storage: $storage\n"; fi if [ -z "$load_date" ]; then handle_early_exit_on_restore "\n[!!!] Required field --load: $load_date - is empty\n" ; fi if [ -z "$mode" ]; then handle_early_exit_on_restore "\n[!!!] Required field --mode: $mode - is empty\n" ; fi if [ "$mode" != "direct" ] && [ "$mode" != "indirect" ] ; then handle_early_exit_on_restore "\n[!!!] Invalid field --mode: $mode\n"; fi @@ -2090,7 +2095,7 @@ validation_prechecks_for_restore() { if [ -n "$s3_url" ]; then add_s3_api_flags+=" --endpoint-url $s3_url"; fi if $no_verify_ssl; then add_s3_api_flags+=" --no-verify-ssl"; fi; - # If remote backup - Validate that scp works + # If remote backup - Validate that scp works if [ $backup_destination == "Remote" ]; then if ssh $scp echo ok 2>&1 ;then printf 'SSH Works\n\n' @@ -2100,17 +2105,17 @@ validation_prechecks_for_restore() { fi # Make sure the database is offline - if [ "$mode" == "direct" ]; then - if [ -z $(pidof PrimProc) ]; then - printf " - Columnstore Status ... Offline\n"; - else - handle_early_exit_on_restore "\n[X] Columnstore is ONLINE - please turn off \n\n"; + if [ "$mode" == "direct" ]; then + if [ -z $(pidof PrimProc) ]; then + printf " - Columnstore Status ... Offline\n"; + else + handle_early_exit_on_restore "\n[X] Columnstore is ONLINE - please turn off \n\n"; fi - - if [ -z $(pidof mariadbd) ]; then - printf " - MariaDB Server Status ... Offline\n"; - else - handle_early_exit_on_restore "\n[X] MariaDB is ONLINE - please turn off \n\n"; + + if [ -z $(pidof mariadbd) ]; then + printf " - MariaDB Server Status ... Offline\n"; + else + handle_early_exit_on_restore "\n[X] MariaDB is ONLINE - please turn off \n\n"; fi fi @@ -2118,7 +2123,7 @@ validation_prechecks_for_restore() { check_package_managers cmapi_installed_command="" case $package_manager in - yum ) + yum ) cmapi_installed_command="yum list installed MariaDB-columnstore-cmapi &> /dev/null;"; ;; apt ) @@ -2130,33 +2135,33 @@ validation_prechecks_for_restore() { esac if eval $cmapi_installed_command ; then - if ! sudo mcs cmapi is-ready ; then - printf " - Columnstore Management API Status .. Offline\n"; - else - handle_early_exit_on_restore "\n[X] Cmapi is ONLINE - please turn off \n\n"; + if ! sudo mcs cmapi is-ready ; then + printf " - Columnstore Management API Status .. Offline\n"; + else + handle_early_exit_on_restore "\n[X] Cmapi is ONLINE - please turn off \n\n"; fi else - printf " - Columnstore Management API Status .. Not Installed - Skipping\n"; + printf " - Columnstore Management API Status .. Not Installed - Skipping\n"; fi # Validate addtional relevant arguments per storage option - if [ $storage == "LocalStorage" ]; then - + if [ $storage == "LocalStorage" ]; then + if [ -z "$backup_location" ]; then handle_early_exit_on_restore "Invalid --backup_location: $backup_location - is empty"; fi if [ -z "$backup_destination" ]; then handle_early_exit_on_restore "Invalid --backup_destination: $backup_destination - is empty"; fi if [ $backup_destination == "Remote" ] && [ -d $backup_location$load_date ]; then echo "Switching to '-bd Local'"; backup_destination="Local"; fi if [ $backup_destination == "Local" ]; then - + if [ ! -d $backup_location ]; then handle_early_exit_on_restore "Invalid directory --backup_location: $backup_location - doesnt exist"; fi if [ -z "$(ls -A "$backup_location")" ]; then echo "Invalid --backup_location: $backup_location - directory is empty."; fi; if [ ! -d $backup_location$load_date ]; then handle_early_exit_on_restore "Invalid directory --load: $backup_location$load_date - doesnt exist"; else printf " - Backup directory exists\n"; fi fi; local files=$(ls $backup_location$load_date) - if [[ -n "$compress_format" ]]; then + if [[ -n "$compress_format" ]]; then case "$compress_format" in - pigz) - flag_cs_local=false + pigz) + flag_cs_local=false flag_mysql=$skip_mdb while read -r line; do @@ -2177,17 +2182,17 @@ validation_prechecks_for_restore() { fi ;; *) # unknown option - handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" esac - else - + else + flag_configs=false flag_data=false flag_mysql=$skip_mdb files=$(ls $backup_location$load_date ) while read -r line; do folder_name=$(echo "$line" | awk '{ print $NF }') - + case "$folder_name" in *configs*) flag_configs=true ;; *data*) flag_data=true ;; @@ -2205,15 +2210,15 @@ validation_prechecks_for_restore() { handle_early_exit_on_restore " Backup is missing subdirectories... \nls $backup_location$load_date\n" fi fi - - elif [ $storage == "S3" ]; then - + + elif [ $storage == "S3" ]; then + # Check for concurrency settings - https://docs.aws.amazon.com/cli/latest/topic/s3-config.html if [ $cloud == "aws" ]; then cli_concurrency=$(sudo grep max_concurrent_requests ~/.aws/config 2>/dev/null ); if [ -z "$cli_concurrency" ]; then echo "[!!!] check: '~/.aws/config' - We recommend increasing s3 concurrency for better throughput to/from S3. This value should scale with avaialble CPU and networking capacity"; printf "example: aws configure set default.s3.max_concurrent_requests 200\n"; fi; fi; # Validate addtional relevant arguments if [ -z "$backup_bucket" ]; then handle_early_exit_on_restore "Invalid --backup_bucket: $backup_bucket - is empty"; fi - + # Prepare commands for each cloud if [ $cloud == "gcp" ]; then check1="$gsutil ls $backup_bucket"; @@ -2221,7 +2226,7 @@ validation_prechecks_for_restore() { check3="$gsutil ls $new_bucket"; check4="$gsutil ls $backup_bucket/$load_date"; check5="$gsutil ls $backup_bucket/$load_date/"; - else + else check1="$awscli $add_s3_api_flags s3 ls $backup_bucket" check2="$awscli $add_s3_api_flags s3 ls | grep $new_bucket" check3="$awscli $add_s3_api_flags s3 ls $new_bucket" @@ -2232,33 +2237,33 @@ validation_prechecks_for_restore() { # Check aws cli access to bucket if $check1 > /dev/null ; then echo -e " - Success listing backup bucket" - else + else handle_early_exit_on_restore "[X] Failed to list bucket contents...\n$check1\n" fi # New bucket exists and empty check - if [ "$new_bucket" ] && [ $pm == "pm1" ]; then + if [ "$new_bucket" ] && [ $pm == "pm1" ]; then # Removing as storage.buckets.get permission likely not needed - # new_bucket_exists=$($check2); if [ -z "$new_bucket_exists" ]; then handle_early_exit_on_restore "[!!!] Didnt find new bucket - Check: $check2\n"; fi; echo "[+] New Bucket exists"; + # new_bucket_exists=$($check2); if [ -z "$new_bucket_exists" ]; then handle_early_exit_on_restore "[!!!] Didnt find new bucket - Check: $check2\n"; fi; echo "[+] New Bucket exists"; # Throw warning if new bucket is NOT empty - if ! $continue; then nb_contents=$($check3 | wc -l); - if [ $nb_contents -lt 1 ]; then - echo " - New Bucket is empty"; - else - echo "[!!!] New bucket is NOT empty... $nb_contents files exist... exiting"; - echo "add "--continue" to skip this exit"; - echo -e "\nExample empty bucket command:\n aws s3 rm $new_bucket --recursive\n gsutil -m rm -r $new_bucket/* \n"; - handle_early_exit_on_restore "Please empty bucket or add --continue \n"; - fi; - else - echo " - [!] Skipping empty new_bucket check"; - fi; + if ! $continue; then nb_contents=$($check3 | wc -l); + if [ $nb_contents -lt 1 ]; then + echo " - New Bucket is empty"; + else + echo "[!!!] New bucket is NOT empty... $nb_contents files exist... exiting"; + echo "add "--continue" to skip this exit"; + echo -e "\nExample empty bucket command:\n aws s3 rm $new_bucket --recursive\n gsutil -m rm -r $new_bucket/* \n"; + handle_early_exit_on_restore "Please empty bucket or add --continue \n"; + fi; + else + echo " - [!] Skipping empty new_bucket check"; + fi; fi # Check if s3 bucket load date exists if $check4 > /dev/null ; then echo -e " - Backup directory exists" - else + else handle_early_exit_on_restore "\n[X] Backup directory load date ($backup_bucket/$load_date) DOES NOT exist in S3 \nCheck - $check4 \n\n"; fi @@ -2267,8 +2272,8 @@ validation_prechecks_for_restore() { if [[ -n "$compress_format" ]]; then # when compressed, storagemanager/configs wont exist but rather pigz_chunkXXX does. and mysql could be split up case "$compress_format" in - pigz) - flag_cs_local=false + pigz) + flag_cs_local=false flag_mysql=$skip_mdb flag_columnstoreData=$skip_bucket_data @@ -2292,7 +2297,7 @@ validation_prechecks_for_restore() { fi ;; *) # unknown option - handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac else flag_configs=false @@ -2310,7 +2315,7 @@ validation_prechecks_for_restore() { *configs*) flag_configs=true ;; #*) echo "unknown: $folder_name" ;; esac - done <<< "$folders" + done <<< "$folders" if $flag_storagemanager && $flag_mysql && $flag_columnstoreData && $flag_configs; then echo " - Backup subdirectories exist" @@ -2323,7 +2328,7 @@ validation_prechecks_for_restore() { handle_early_exit_on_restore " Backup is missing subdirectories... \n$check5\n" fi fi; - else + else handle_early_exit_on_restore " Failed to list backup contents...\n$check5\n" fi; fi; @@ -2331,10 +2336,10 @@ validation_prechecks_for_restore() { if $quiet; then xtra_cmd_args+=" 2> /dev/null"; rsync_flags=" -a"; fi } -# Restore Columnstore.xml by updating critical parameters +# Restore Columnstore.xml by updating critical parameters # Depends on $CS_CONFIGS_PATH/$col_xml_backup restore_columnstore_values() { - + printf "\nRestore Columnstore.xml Values" if ! $quiet; then printf "\n"; else printf "... "; fi; columnstore_config_pairs_to_transfer=( @@ -2355,20 +2360,20 @@ restore_columnstore_values() { for pair in "${columnstore_config_pairs_to_transfer[@]}"; do level_one=$(echo "$pair" | cut -d ' ' -f 1) level_two=$(echo "$pair" | cut -d ' ' -f 2) - + # Get the source value using mcsGetConfig -c source_value=$(mcsGetConfig -c "$CS_CONFIGS_PATH/$col_xml_backup" "$level_one" "$level_two") - + if [ -n "$source_value" ]; then # Set the value in the active Columnstore.xml using mcsSetConfig if mcsSetConfig "$level_one" "$level_two" "$source_value"; then # echo instead of printf to avoid escaping % from HashJoin TotalUmMemory - if ! $quiet; then echo " - Set $level_one $level_two $source_value"; fi; + if ! $quiet; then echo " - Set $level_one $level_two $source_value"; fi; else printf "\n[!] Failed to Set $level_one $level_two $source_value \n"; fi else - if ! $quiet; then printf " - N/A: $level_one $level_two \n"; fi; + if ! $quiet; then printf " - N/A: $level_one $level_two \n"; fi; fi done if $quiet; then printf " Done\n"; fi; @@ -2377,11 +2382,11 @@ restore_columnstore_values() { run_restore() { # Branch logic based on topology - if [ $storage == "LocalStorage" ]; then + if [ $storage == "LocalStorage" ]; then # Branch logic based on where the backup resides if [ $backup_destination == "Local" ]; then - + # MariaDB Columnstore Restore printf "\nRestore MariaDB Columnstore LocalStorage\n" @@ -2392,18 +2397,18 @@ run_restore() i=1; while [ $i -le $DBROOT_COUNT ]; do if [[ $pm == "pm$i" || $HA == true ]]; then - printf " - Deleting Columnstore Data$i ... "; - rm -rf /var/lib/columnstore/data$i/*; - printf " Done \n"; + printf " - Deleting Columnstore Data$i ... "; + rm -rf /var/lib/columnstore/data$i/*; + printf " Done \n"; fi ((i++)) - done + done tar_flags="xf" if ! $quiet; then tar_flags+="v"; fi; cs_prefix="$backup_location$load_date/$split_file_cs_prefix.$compress_format" case $compress_format in - pigz) + pigz) printf " - Decompressing CS Files -> local ... " cd / if ! eval "pigz -dc -p $PARALLEL_THREADS $cs_prefix | tar $tar_flags - "; then @@ -2412,16 +2417,16 @@ run_restore() printf " Done \n" ;; *) # unknown option - handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac # Set permissions after restoring files i=1; while [ $i -le $DBROOT_COUNT ]; do if [[ $pm == "pm$i" || $HA == true ]]; then - printf " - Chowning Columnstore Data$i ... "; - chown $columnstore_user:$columnstore_user -R /var/lib/columnstore/data$i/; - printf " Done \n"; + printf " - Chowning Columnstore Data$i ... "; + chown $columnstore_user:$columnstore_user -R /var/lib/columnstore/data$i/; + printf " Done \n"; fi ((i++)) done @@ -2433,25 +2438,25 @@ run_restore() while [ $i -le $DBROOT_COUNT ]; do if [[ $pm == "pm$i" || $HA == true ]]; then printf " - Restoring Columnstore Data$i ... "; - rm -rf /var/lib/columnstore/data$i/*; + rm -rf /var/lib/columnstore/data$i/*; rsync $rsync_flags $backup_location$load_date/data$i/ /var/lib/columnstore/data$i/; - chown $columnstore_user:$columnstore_user -R /var/lib/columnstore/data$i/; - printf " Done Data$i \n"; + chown $columnstore_user:$columnstore_user -R /var/lib/columnstore/data$i/; + printf " Done Data$i \n"; fi ((i++)) - done - - # Put configs in place + done + + # Put configs in place printf " - Columnstore Configs ... " rsync $rsync_flags $backup_location$load_date/configs/storagemanager.cnf $STORAGEMANGER_CNF rsync $rsync_flags $backup_location$load_date/configs/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup rsync $rsync_flags $backup_location$load_date/configs/mysql/$pm/ $MARIADB_SERVER_CONFIGS_PATH/ - if [ -f "$backup_location$load_date/configs/cmapi_server.conf" ]; then - rsync $rsync_flags $backup_location$load_date/configs/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup ; + if [ -f "$backup_location$load_date/configs/cmapi_server.conf" ]; then + rsync $rsync_flags $backup_location$load_date/configs/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup ; fi; - printf " Done\n" + printf " Done\n" fi - + elif [ $backup_destination == "Remote" ]; then printf "[~] Copy MySQL Data..." tmp="localscpcopy-$load_date" @@ -2463,58 +2468,63 @@ run_restore() # Loop through per dbroot printf "[~] Columnstore Data..."; i=1; while [ $i -le $DBROOT_COUNT ]; do - if [[ $pm == "pm$i" || $HA == true ]]; then - rm -rf /var/lib/columnstore/data$i/; - rsync -av $scp:$backup_location$load_date/data$i/ /var/lib/columnstore/data$i/ ; + if [[ $pm == "pm$i" || $HA == true ]]; then + rm -rf /var/lib/columnstore/data$i/; + rsync -av $scp:$backup_location$load_date/data$i/ /var/lib/columnstore/data$i/ ; chown $columnstore_user:$columnstore_user -R /var/lib/columnstore/data$i/; fi ((i++)) done - printf "[+] Columnstore Data... Done\n" + printf "[+] Columnstore Data... Done\n" # Put configs in place printf " - Columnstore Configs ... " rsync $rsync_flags $backup_location$load_date/configs/storagemanager.cnf $STORAGEMANGER_CNF - rsync $rsync_flags $backup_location$load_date/configs/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup - rsync $rsync_flags $backup_location$load_date/configs/mysql/$pm/ $MARIADB_SERVER_CONFIGS_PATH/ - if [ -f "$backup_location$load_date/configs/cmapi_server.conf" ]; then - rsync $rsync_flags $backup_location$load_date/configs/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup ; + rsync $rsync_flags $backup_location$load_date/configs/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup + rsync $rsync_flags $backup_location$load_date/configs/mysql/$pm/ $MARIADB_SERVER_CONFIGS_PATH/ + if [ -f "$backup_location$load_date/configs/cmapi_server.conf" ]; then + rsync $rsync_flags $backup_location$load_date/configs/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup ; fi; printf " Done\n" load_date=$tmp - else + else handle_early_exit_on_restore "Invalid Script Variable --backup_destination: $backup_destination" fi - # MariaDB Server Restore - printf "\nRestore MariaDB Server\n" - if [[ -n "$compress_format" ]]; then - # Handle compressed mariadb-backup restore - mbd_prefix="$backup_location$load_date/$split_file_mdb_prefix.$compress_format" - case $compress_format in - pigz) - printf " - Decompressing MariaDB Files -> $MARIADB_PATH ... " - rm -rf $MARIADB_PATH/* - cd / - if ! eval "pigz -dc -p $PARALLEL_THREADS $mbd_prefix | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH"; then - handle_early_exit_on_restore "Failed to decompress mariadb backup\ncommand:\npigz -dc -p $PARALLEL_THREADS $mbd_prefix | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH \n" - fi; - printf " Done \n" - printf " - Running mariabackup --prepare ... " - if eval "mariabackup --prepare --target-dir=$MARIADB_PATH $xtra_cmd_args"; then - printf " Done\n"; - else - handle_early_exit_on_restore "Failed to --prepare\nmariabackup --prepare --target-dir=$MARIADB_PATH"; - fi; - ;; - *) # unknown option - handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" - esac + if ! $skip_mdb; then + # MariaDB Server Restore + printf "\nRestore MariaDB Server\n" + if [[ -n "$compress_format" ]]; then + # Handle compressed mariadb-backup restore + mbd_prefix="$backup_location$load_date/$split_file_mdb_prefix.$compress_format" + case $compress_format in + pigz) + printf " - Decompressing MariaDB Files -> $MARIADB_PATH ... " + rm -rf $MARIADB_PATH/* + cd / + if ! eval "pigz -dc -p $PARALLEL_THREADS $mbd_prefix | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH"; then + handle_early_exit_on_restore "Failed to decompress mariadb backup\ncommand:\npigz -dc -p $PARALLEL_THREADS $mbd_prefix | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH \n" + fi; + printf " Done \n" + printf " - Running mariabackup --prepare ... " + if eval "mariabackup --prepare --target-dir=$MARIADB_PATH $xtra_cmd_args"; then + printf " Done\n"; + else + handle_early_exit_on_restore "Failed to --prepare\nmariabackup --prepare --target-dir=$MARIADB_PATH"; + fi; + ;; + *) # unknown option + handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" + esac + else + eval "mariabackup --prepare --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args" + rm -rf /var/lib/mysql/* + eval "mariabackup --copy-back --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args" + fi else - eval "mariabackup --prepare --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args" - rm -rf /var/lib/mysql/* - eval "mariabackup --copy-back --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args" - fi + echo "[!] Skipping mariadb server restore" + + fi; printf " - Chowning MariaDB Files ... "; chown -R $mariadb_user:$mariadb_user /var/lib/mysql/ chown -R $columnstore_user:$mariadb_user /var/log/mariadb/ @@ -2525,18 +2535,18 @@ run_restore() printf "\nRestore MariaDB Columnstore S3\n" # Sync the columnstore data from the backup bucket to the new bucket - if ! $skip_bucket_data && [ "$new_bucket" ] && [ $pm == "pm1" ]; then + if ! $skip_bucket_data && [ "$new_bucket" ] && [ $pm == "pm1" ]; then printf "[+] Starting Bucket Sync:\n - $backup_bucket/$load_date/columnstoreData to $new_bucket\n" s3sync $backup_bucket/$load_date/columnstoreData/ $new_bucket/ " - Done with S3 Bucket sync\n" - else + else printf "[!] Skipping Columnstore Bucket\n" fi; - + printf "[+] Starting Columnstore Configurations & Metadata: \n" if [[ -n "$compress_format" ]]; then cs_prefix="$backup_bucket/$load_date/$split_file_cs_prefix.$compress_format" case $compress_format in - pigz) + pigz) printf " - Decompressing CS Files bucket -> local ... " if [ $cloud == "gcp" ]; then # gsutil supports simple "cat prefix*" to standard out unlike awscli @@ -2545,11 +2555,11 @@ run_restore() handle_early_exit_on_restore "Failed to decompress and untar columnstore localfiles\ncommand:\n$gsutil cat $cs_prefix* | gzip -dc | tar xf - \n\n" fi elif [ $cloud == "aws" ]; then - + # List all the pigz compressed chunks in the S3 prefix - should I clear prior local files? chunk_list=$($awscli s3 ls "$cs_prefix" | awk '{print $NF}') cd / - + # Use process substitution to concatenate the compressed chunks and pipe to pigz and extract using tar if ! eval "pigz -dc -p $PARALLEL_THREADS <(for chunk in \$chunk_list; do $awscli s3 cp \"$backup_bucket/$load_date/\$chunk\" - ; done) | tar xf - "; then handle_early_exit_on_restore "Failed to decompress and untar columnstore localfiles\ncommand:\npigz -dc -p $PARALLEL_THREADS <(for chunk in \$chunk_list; do $awscli s3 cp \"$backup_bucket/$load_date/\$chunk\" - ; done) | tar xf - \n\n" @@ -2558,21 +2568,21 @@ run_restore() printf " Done \n" ;; *) # unknown option - handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac - + else - # Columnstore.xml and cmapi are renamed so that only specifc values are restored + # Columnstore.xml and cmapi are renamed so that only specifc values are restored s3cp $backup_bucket/$load_date/configs/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup s3cp $backup_bucket/$load_date/configs/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup - s3cp $backup_bucket/$load_date/configs/storagemanager.cnf $STORAGEMANGER_CNF + s3cp $backup_bucket/$load_date/configs/storagemanager.cnf $STORAGEMANGER_CNF s3sync $backup_bucket/$load_date/storagemanager/cache/data$pm_number $cs_cache/data$pm_number/ " - Done cache/data$pm_number/\n" s3sync $backup_bucket/$load_date/storagemanager/metadata/data$pm_number $cs_metadata/data$pm_number/ " - Done metadata/data$pm_number/\n" if ! $skip_mdb; then s3sync $backup_bucket/$load_date/configs/mysql/$pm/ $MARIADB_SERVER_CONFIGS_PATH/ " - Done $MARIADB_SERVER_CONFIGS_PATH/\n"; fi if s3ls "$backup_bucket/$today/storagemanager/journal/data$ASSIGNED_DBROOT" > /dev/null 2>&1; then s3sync $backup_bucket/$load_date/storagemanager/journal/data$pm_number $cs_journal/data$pm_number/ " - Done journal/data$pm_number/\n" - else + else echo " - Done journal/data$pm_number was empty" fi fi; @@ -2581,44 +2591,44 @@ run_restore() printf "[+] Adjusting storagemanager.cnf ... \n" target_bucket=$backup_bucket target_prefix="prefix = $load_date\/columnstoreData\/" - if [ -n "$new_bucket" ]; then - target_bucket=$new_bucket; - target_prefix="# prefix \= cs\/"; + if [ -n "$new_bucket" ]; then + target_bucket=$new_bucket; + target_prefix="# prefix \= cs\/"; fi - if [ -n "$new_region" ]; then + if [ -n "$new_region" ]; then if [ $cloud == "gcp" ]; then endpoint="storage.googleapis.com"; else endpoint="s3.$new_region.amazonaws.com"; fi; - sed -i "s|^endpoint =.*|endpoint = ${endpoint}|g" $STORAGEMANGER_CNF; - sed -i "s|^region =.*|region = ${new_region}|g" $STORAGEMANGER_CNF; + sed -i "s|^endpoint =.*|endpoint = ${endpoint}|g" $STORAGEMANGER_CNF; + sed -i "s|^region =.*|region = ${new_region}|g" $STORAGEMANGER_CNF; echo " - Adjusted endpoint & region"; fi # Removes prefix of the protocol and escapes / for sed target_bucket_name=$(echo "${target_bucket#$protocol://}" | sed "s/\//\\\\\//g") - if [ -n "$new_key" ]; then - sed -i "s|aws_access_key_id =.*|aws_access_key_id = ${new_key}|g" $STORAGEMANGER_CNF; - echo " - Adjusted aws_access_key_id"; + if [ -n "$new_key" ]; then + sed -i "s|aws_access_key_id =.*|aws_access_key_id = ${new_key}|g" $STORAGEMANGER_CNF; + echo " - Adjusted aws_access_key_id"; fi - if [ -n "$new_secret" ]; then - sed -i "s|aws_secret_access_key =.*|aws_secret_access_key = ${new_secret}|g" $STORAGEMANGER_CNF; - echo " - Adjusted aws_secret_access_key"; + if [ -n "$new_secret" ]; then + sed -i "s|aws_secret_access_key =.*|aws_secret_access_key = ${new_secret}|g" $STORAGEMANGER_CNF; + echo " - Adjusted aws_secret_access_key"; fi - bucket=$( grep -m 1 "^bucket =" $STORAGEMANGER_CNF | sed "s/\//\\\\\//g"); + bucket=$( grep -m 1 "^bucket =" $STORAGEMANGER_CNF | sed "s/\//\\\\\//g"); sed -i "s/$bucket/bucket = $target_bucket_name/g" $STORAGEMANGER_CNF; echo " - Adjusted bucket"; - + prefix=$( grep -m 1 "^prefix =" $STORAGEMANGER_CNF | sed "s/\//\\\\\//g"); - if [ ! -z "$prefix" ]; then - sed -i "s/$prefix/$target_prefix/g" $STORAGEMANGER_CNF; - echo " - Adjusted prefix"; + if [ ! -z "$prefix" ]; then + sed -i "s/$prefix/$target_prefix/g" $STORAGEMANGER_CNF; + echo " - Adjusted prefix"; fi; # Check permissions chown -R $columnstore_user:$columnstore_user /var/lib/columnstore/ chown -R root:root $MARIADB_SERVER_CONFIGS_PATH/ - + # Confirm S3 connection works if [ $mode == "direct" ]; then echo "[+] Checking S3 Connection ..."; if testS3Connection $xtra_cmd_args; then echo " - S3 Connection passes" ; else handle_early_exit_on_restore "\n[X] S3 Connection issues - retest/configure\n"; fi; fi; printf "[+] MariaDB Columnstore Done\n\n" @@ -2626,10 +2636,10 @@ run_restore() if ! $skip_mdb; then printf "Restore MariaDB Server\n" if [[ -n "$compress_format" ]]; then - # Handle compressed mariadb-backup restore + # Handle compressed mariadb-backup restore mbd_prefix="$backup_bucket/$load_date/$split_file_mdb_prefix.$compress_format" case $compress_format in - pigz) + pigz) printf "[+] Decompressing mariadb-backup bucket -> local ... " rm -rf $MARIADB_PATH/* if [ $cloud == "gcp" ]; then @@ -2640,7 +2650,7 @@ run_restore() chunk_list=$($awscli s3 ls "$mbd_prefix" | awk '{print $NF}') cd / - + if ! eval "pigz -dc -p $PARALLEL_THREADS <(for chunk in \$chunk_list; do $awscli s3 cp "$backup_bucket/$load_date/\$chunk" -; done) | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH"; then handle_early_exit_on_restore "Failed to decompress mariadb backup\ncommand:\npigz -dc -p $PARALLEL_THREADS <(for chunk in \$chunk_list; do $awscli s3 cp "$backup_bucket/$load_date/\$chunk" -; done) | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH \n" fi; @@ -2650,29 +2660,29 @@ run_restore() if eval "mariabackup --prepare --target-dir=$MARIADB_PATH $xtra_cmd_args"; then printf " Done\n"; else handle_early_exit_on_restore "Failed to --prepare\nmariabackup --prepare --target-dir=$MARIADB_PATH"; fi; ;; *) # unknown option - handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" + handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac - + else # Copy the MariaDB data, prepare and put in place printf " - Copying down MariaDB data ... " - rm -rf $backup_location$load_date/mysql/ + rm -rf $backup_location$load_date/mysql/ mkdir -p $backup_location$load_date/mysql/ s3sync $backup_bucket/$load_date/mysql/ $backup_location$load_date/mysql " Done\n" # Run prepare and copy back printf " - Running mariabackup --prepare ... " - if eval "mariabackup --prepare --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args"; then - printf " Done\n"; - else - echo "failed"; + if eval "mariabackup --prepare --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args"; then + printf " Done\n"; + else + echo "failed"; fi; rm -rf /var/lib/mysql/* > /dev/null printf " - Running mariabackup --copy-back ... " - if eval "mariabackup --copy-back --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args"; then - printf " Done\n"; - else - echo "failed"; + if eval "mariabackup --copy-back --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args"; then + printf " Done\n"; + else + echo "failed"; fi; fi; @@ -2689,19 +2699,20 @@ run_restore() fi if [[ $pm == "pm1" ]]; then restore_columnstore_values; fi; - + end=$(date +%s) runtime=$((end-start)) printf "\nRuntime: $runtime\n" printf "$final_message\n\n" - if ! $quiet; then - if [ $pm == "pm1" ]; then - echo -e " - Last you need to manually configure mariadb replication between nodes" - echo -e " - systemctl start mariadb " - echo -e " - systemctl start mariadb-columnstore-cmapi " - echo -e " - mcs cluster start \n\n" + if ! $quiet; then + if [ $pm == "pm1" ]; then + echo -e "Next: " + if [ $DBROOT_COUNT -gt 1 ]; then echo -e "[!!] Check Replicas to manually configure mariadb replication"; fi + echo -e "systemctl start mariadb " + echo -e "systemctl start mariadb-columnstore-cmapi " + echo -e "mcs cluster start \n\n" else - printf " - Last you need to manually configure mariadb replication between nodes\n" + echo -e "[!!] Check this replica to manually configure mariadb replication after primary node is ready" echo -e "Example:" echo -e "mariadb -e \"stop slave;CHANGE MASTER TO MASTER_HOST='\$pm1' , MASTER_USER='repuser' , MASTER_PASSWORD='aBcd123%123%aBc' , MASTER_USE_GTID = slave_pos;start slave;show slave status\G\"" echo -e "mariadb -e \"show slave status\G\" | grep \"Slave_\" \n" @@ -2720,10 +2731,10 @@ load_default_dbrm_variables() { skip_storage_manager=false mode="once" quiet=false - + list_dbrm_backups=false dbrm_dir="/var/lib/columnstore/data1/systemFiles/dbrm" - if [ "$storage" == "S3" ]; then + if [ "$storage" == "S3" ]; then dbrm_dir="/var/lib/columnstore/storagemanager" fi } @@ -2738,7 +2749,7 @@ load_default_dbrm_restore_variables() { skip_storage_manager=false dbrm_dir="/var/lib/columnstore/data1/systemFiles/dbrm" - if [ "$storage" == "S3" ]; then + if [ "$storage" == "S3" ]; then dbrm_dir="/var/lib/columnstore/storagemanager" fi } @@ -2747,8 +2758,8 @@ print_dbrm_backup_help_text() { echo " Columnstore DBRM Backup - -m | --mode ['loop','once']; Determines if this script runs in a forever loop sleeping -i minutes or just once - -i | --interval Number of minutes to sleep when --mode loop + -m | --mode ['loop','once']; Determines if this script runs in a forever loop sleeping -i minutes or just once + -i | --interval Number of minutes to sleep when --mode loop -r | --retention-days Retain dbrm backups created within the last X days, the rest are deleted -p | --path path of where to save the dbrm backups on disk -nb | --name-backup custom name to prefex dbrm backups with @@ -2759,7 +2770,7 @@ print_dbrm_backup_help_text() { Examples: ./$0 dbrm_backup --mode loop --interval 90 --retention-days 7 --path /mnt/dbrm_backups ./$0 dbrm_backup --mode once --retention-days 7 --path /mnt/dbrm_backups -nb my-one-off-backup - + Cron Example: */60 */3 * * * root bash /root/$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups >> /tmp/dbrm_backups/cs_backup.log 2>&1 "; @@ -2775,10 +2786,10 @@ print_dbrm_restore_help_text() { -sdbk| --skip-dbrm-backup skip backing up dbrms brefore restoring -ssm | --skip-storage-manager skip restoring storagemanager directory - Default: ./$0 dbrm_restore --path /tmp/dbrm_backups + Default: ./$0 dbrm_restore --path /tmp/dbrm_backups Examples: - ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 + ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 --no-start "; } @@ -2927,9 +2938,9 @@ validation_prechecks_for_dbrm_backup() { confirm_numerical_or_decimal_else_fail "$retention_days" "Retention" # Check backup location exists - if [ ! -d $backup_location ]; then - echo "[+] Created: $backup_location" - mkdir "$backup_location"; + if [ ! -d $backup_location ]; then + echo "Created: $backup_location" + mkdir "$backup_location"; fi; # Confirm bucket connection @@ -2949,7 +2960,7 @@ validation_prechecks_before_listing_restore_options() { printf "ls -la $backup_location\n\n" exit 1; fi - + # Check if backup directory is empty if [ -z "$(find "$backup_location" -mindepth 1 | head )" ]; then printf "[!!] Backups Directory is empty --path $backup_location \n" @@ -2959,30 +2970,30 @@ validation_prechecks_before_listing_restore_options() { } validation_prechecks_for_dbrm_restore() { - + printf "Prechecks\n" echo "--------------------------------------------------------------------------" # Confirm storage not empty if [ -z "$storage" ]; then printf "[!] Empty storage: \ncheck: grep -m 1 \"^service = \" \$STORAGEMANGER_CNF | awk '{print \$3}' \n\n"; fi; # Check backup directory exists - if [ ! -d $backup_location ]; then + if [ ! -d $backup_location ]; then printf "[!] \$backup_location: Path of backups does Not exist\n" printf "Path: $backup_location\n\n" exit 2; fi - # Check specific backup exists + # Check specific backup exists backup_folder_to_restore_dbrms=$backup_folder_to_restore if [ $storage == "S3" ]; then backup_folder_to_restore_dbrms="${backup_folder_to_restore}/dbrms" fi - if [ ! -d "${backup_location}/${backup_folder_to_restore_dbrms}" ]; then + if [ ! -d "${backup_location}/${backup_folder_to_restore_dbrms}" ]; then printf "[!] \$backup_folder_to_restore: Path of backup to restore does Not exist\n" printf "Path: ${backup_location}/${backup_folder_to_restore_dbrms}\n\n" exit 2; - else + else echo " - Backup directory exists" if [ "$(ls -A ${backup_location}/${backup_folder_to_restore_dbrms})" ]; then @@ -3001,7 +3012,7 @@ validation_prechecks_for_dbrm_restore() { "oidbitmap" "SMTxnID" ) - + # Check if all expected files exist for file in "${expected_files[@]}"; do if [ ! -f "${backup_location}/${backup_folder_to_restore_dbrms}/${file}" ]; then @@ -3013,13 +3024,13 @@ validation_prechecks_for_dbrm_restore() { # For S3 check storagemanager dir exists in backup unless skip storagemanager is passed if [ "$storage" == "S3" ] && [ $skip_storage_manager == false ]; then - if [ ! -d "${backup_location}/${backup_folder_to_restore}/metadata" ]; then + if [ ! -d "${backup_location}/${backup_folder_to_restore}/metadata" ]; then printf "\n[!!] Path Not Found: ${backup_location}/${backup_folder_to_restore}/metadata \n" printf "Retry with a different backup to restore or use flag --skip-storage-manager\n\n" exit 2; fi; fi - + printf " - Backup contains all files\n" @@ -3043,13 +3054,13 @@ validation_prechecks_for_dbrm_restore() { # Download cs_package_manager.sh if not exists if [ ! -f "cs_package_manager.sh" ]; then wget https://raw.githubusercontent.com/mariadb-corporation/mariadb-columnstore-engine/develop/cmapi/scripts/cs_package_manager.sh; chmod +x cs_package_manager.sh; - fi; - if source cs_package_manager.sh source ;then + fi; + if source cs_package_manager.sh source ;then echo " - Sourced cs_package_manager.sh" else printf "\n[!!] Failed to source cs_package_manager.sh\n\n" exit 1; - + fi # Confirm the function exists and the source of cs_package_manager.sh worked @@ -3076,6 +3087,12 @@ validation_prechecks_for_dbrm_restore() { done } +print_if_not_quiet() { + if ! $quiet; then + printf "$1"; + fi +} + process_dbrm_backup() { load_default_dbrm_variables @@ -3092,15 +3109,15 @@ process_dbrm_backup() { if ! $quiet ; then - printf "\nDBRM Backup\n"; + printf "\nDBRM Backup\n"; echo "--------------------------------------------------------------------------" if [ "$storage" == "S3" ]; then echo "Skips: Storagemanager($skip_storage_manager)"; fi; echo "--------------------------------------------------------------------------" - printf "CS Storage: $storage\n"; - printf "Source: $dbrm_dir\n"; - printf "Backups: $backup_location\n"; - if [ "$mode" == "loop" ]; then - printf "Interval: $backup_interval_minutes minutes\n"; + printf "CS Storage: $storage\n"; + printf "Source: $dbrm_dir\n"; + printf "Backups: $backup_location\n"; + if [ "$mode" == "loop" ]; then + printf "Interval: $backup_interval_minutes minutes\n"; fi; printf "Retention: $retention_days day(s)\n" printf "Mode: $mode\n" @@ -3115,17 +3132,17 @@ process_dbrm_backup() { timestamp=$(date +%Y%m%d_%H%M%S) backup_folder="$backup_location/${backup_base_name}_${timestamp}" mkdir -p "$backup_folder" - + # Copy files to the backup directory if [[ $skip_storage_manager == false || $storage == "LocalStorage" ]]; then - if ! $quiet; then printf " - copying $dbrm_dir ..."; fi; + print_if_not_quiet " - copying $dbrm_dir ..."; cp -arp "$dbrm_dir"/* "$backup_folder" - if ! $quiet; then printf " Done\n"; fi; + print_if_not_quiet " Done\n"; fi if [ "$storage" == "S3" ]; then # smcat em files to disk - if ! $quiet; then printf " - copying DBRMs from bucket ..."; fi; + print_if_not_quiet " - copying DBRMs from bucket ..."; mkdir $backup_folder/dbrms/ smls /data1/systemFiles/dbrm 2>/dev/null > $backup_folder/dbrms/dbrms.txt smcat /data1/systemFiles/dbrm/BRM_saves_current 2>/dev/null > $backup_folder/dbrms/BRM_saves_current @@ -3142,39 +3159,38 @@ process_dbrm_backup() { smcat /data1/systemFiles/dbrm/oidbitmap 2>/dev/null > $backup_folder/dbrms/oidbitmap smcat /data1/systemFiles/dbrm/SMTxnID 2>/dev/null > $backup_folder/dbrms/SMTxnID smcat /data1/systemFiles/dbrm/tablelocks 2>/dev/null > $backup_folder/dbrms/tablelocks - if ! $quiet; then printf " Done\n"; fi; + print_if_not_quiet " Done\n"; fi if [ $retention_days -gt 0 ] ; then # Clean up old backups # example: find /tmp/dbrm_backups -maxdepth 1 -type d -name "dbrm_backup_*" -mtime +1 -exec rm -r {} \; - if ! $quiet; then printf " - applying retention policy ..."; fi; + print_if_not_quiet " - applying retention policy ..."; find "$backup_location" -maxdepth 1 -type d -name "${backup_base_name}_*" -mtime +$retention_days -exec rm -r {} \; - if ! $quiet; then printf " Done\n"; fi; + print_if_not_quiet " Done\n"; fi; printf "Created: $backup_folder\n" - - - if [ "$mode" == "once" ]; then + + if [ "$mode" == "once" ]; then end=$(date +%s) runtime=$((end-start)) - if ! $quiet; then printf "Runtime: $runtime\n"; fi; - break; + if ! $quiet; then printf "\nRuntime: $runtime\n"; fi; + break; fi; printf "Sleeping ... $sleep_seconds seconds\n" sleep "$sleep_seconds" done - - if ! $quiet; then printf "Complete\n\n"; fi; + + print_if_not_quiet "Complete\n\n" } is_cmapi_installed() { cmapi_installed_command="" case $package_manager in - yum ) + yum ) cmapi_installed_command="yum list installed MariaDB-columnstore-cmapi &> /dev/null;"; ;; apt ) @@ -3201,30 +3217,30 @@ start_mariadb_cmapi_columnstore() { # For verbose debugging #grep -i rollbackAll /var/log/mariadb/columnstore/debug.log | tail -n 3 | awk '{ print $1, $2, $3, $(NF-2), $(NF-1), $NF }' - + } # Currently assumes systemd installed shutdown_columnstore_mariadb_cmapi() { - pf=35 - init_cs_down - wait_cs_down 0 + local print_formatting=35 + init_cs_down + wait_cs_down 0 - printf "%-${pf}s ... " " - Stopping MariaDB Server" + printf "%-${print_formatting}s ... " " - Stopping MariaDB Server" if ! systemctl stop mariadb; then echo "[!!] Failed to stop mariadb" exit 1; - else + else printf "Done\n" fi if is_cmapi_installed ; then - printf "%-${pf}s ... " " - Stopping CMAPI" + printf "%-${print_formatting}s ... " " - Stopping CMAPI" if ! systemctl stop mariadb-columnstore-cmapi; then echo "[!!] Failed to stop CMAPI" exit 1; - else + else printf "Done\n" fi fi @@ -3240,7 +3256,7 @@ shutdown_columnstore_mariadb_cmapi() { # em_file_full_path # storagemanager_dir_exists get_latest_em_from_directory() { - + subdir_dbrms="" latest_em_file="" em_file_size="" @@ -3259,12 +3275,12 @@ get_latest_em_from_directory() { return 1; fi - + if [ -d "${1}/metadata" ]; then latest_em_meta_file=$(find "${subdir_metadata}" -maxdepth 1 -type f -name "BRM_saves*_em.meta" -exec ls -lat {} + | awk 'NR==1 {printf "%-12s %-4s %-2s %-5s %s\n", $5, $6, $7, $8, $9}'| head -n 1 ) else # Handle missing metadata directory & guess the latest em file based on the largest size - + # Example: find /tmp/dbrm_backups/dbrm_backup_20240605_180906/dbrms -maxdepth 1 -type f -name "BRM_saves*_em" -exec ls -lhS {} + latest_em_meta_file=$(find "${subdir_dbrms}" -maxdepth 1 -type f -name "BRM_saves*_em" -exec ls -lhS {} + | awk 'NR==1 {printf "%-12s %-4s %-2s %-5s %s\n", $5, $6, $7, $8, $9}'| head -n 1) storagemanager_dir_exists=false @@ -3279,7 +3295,7 @@ get_latest_em_from_directory() { if [ ! -f $latest_em_file ]; then echo "S3 List Option: Failed to find $latest_em_file" exit; - fi + fi else subdir_dbrms="$1" latest_em_file=$(find "${subdir_dbrms}" -maxdepth 1 -type f -name "BRM_saves*_em" -exec ls -lat {} + | awk 'NR==1 {printf "%-12s %-4s %-2s %-5s %s\n", $5, $6, $7, $8, $9}'| head -n 1) @@ -3307,7 +3323,7 @@ list_restore_options_from_backups() { vss_file=$(ls -la "${subdir_dbrms}/${version_prefix}_vss" 2>/dev/null | awk 'NR==1 {print $5}' ) if [ $storagemanager_dir_exists == false ]; then vss_file+=" (No Storagemanager Dir)" - fi; + fi; printf "%-45s %-13s %-15s %-12s %-12s %-10s %-10s\n" "$(basename "$subdir")" "$em_file_created" "$em_file_name" "$em_file_size" "$journal_file" "$vbbm_file" "$vss_file" fi done @@ -3316,7 +3332,7 @@ list_restore_options_from_backups() { } process_dbrm_restore() { - + load_default_dbrm_restore_variables parse_dbrm_restore_variables "$@" @@ -3325,31 +3341,31 @@ process_dbrm_restore() { echo "--------------------------------------------------------------------------" echo "Skips: DBRM Backup($skip_dbrm_backup) Storagemanager($skip_storage_manager)" echo "--------------------------------------------------------------------------" - printf "CS Storage: $storage \n" + printf "CS Storage: $storage \n" printf "Backups Directory: $backup_location \n" printf "Backup to Restore: $backup_folder_to_restore \n\n" validation_prechecks_before_listing_restore_options - # Display restore options + # Display restore options if [ -z "$backup_folder_to_restore" ]; then printf "[!] Pick Option\n" list_restore_options_from_backups "$@" printf "\nExample: \n" printf " --directory dbrm_backup_20240103_183536 \n\n" printf "Define which backup to restore via flag --directory \n" - echo "Rerun: $0 $@ --directory xxxxxxx" + echo "Rerun: $0 $@ --directory xxxxxxx" echo "" exit 1; fi; - + validation_prechecks_for_dbrm_restore shutdown_columnstore_mariadb_cmapi # Take an automated backup if [[ $skip_dbrm_backup == false ]]; then printf " - Saving a DBRM backup before restoring ... \n" - if ! process_dbrm_backup -p $backup_location -r 9999 -nb dbrms_before_restore_backup --quiet ; then + if ! process_dbrm_backup -p $backup_location -r 9999 -nb dbrms_before_restore_backup --quiet ; then echo "[!!] Failed to take a DBRM backup before restoring" echo "exiting ..." exit 1; @@ -3358,18 +3374,18 @@ process_dbrm_restore() { # Detect newest date _em from the set, if smaller than the current one throw a warning get_latest_em_from_directory "${backup_location}/${backup_folder_to_restore}" - if [ ! -f $em_file_full_path ]; then + if [ ! -f $em_file_full_path ]; then echo "[!] Failed to parse _em file: $em_file_full_path doesnt exist" exit 1; fi; echo "em_file_full_path: $em_file_full_path" - + echo "latest_em_file: $latest_em_file" echo "em_file_size: $em_file_size" echo "em_file_created: $em_file_created" echo "storagemanager_dir_exists: $storagemanager_dir_exists" echo "subdir_dbrms: $subdir_dbrms" - + em_file_name=$(basename $em_file_full_path) prefix="${em_file_name%%_em}" echo "em_file_name: $em_file_name" @@ -3382,7 +3398,7 @@ process_dbrm_restore() { fi # split logic between S3 & LocalStorage - if [ $storage == "S3" ]; then + if [ $storage == "S3" ]; then process_s3_dbrm_restore else process_localstorage_dbrm_restore @@ -3397,26 +3413,26 @@ smput_or_error() { printf "[!] Failed to smput: $1\n" else printf "." - fi + fi } -# Depends on +# Depends on # em_file - most recent EM # em_file_full_path - full path to most recent EM # em_file_name - Just the file name of the EM # prefix - Prefix of the EM file process_s3_dbrm_restore() { - + printf_offset=45 printf "\nBefore DBRMs Restore\n" echo "--------------------------------------------------------------------------" if ! command -v smls > /dev/null; then printf "[!] smls not installed ... Exiting\n\n" exit 1 - else + else current_status=$(smls /data1/systemFiles/dbrm/ 2>/dev/null); if [ $? -ne 0 ]; then - printf "\n[!] Failed to get smls status\n\n" + printf "\n[!] Failed to get smls status\n\n" exit 1 fi echo "$current_status" | grep -E "BRM_saves_em|BRM_saves_vbbm|BRM_saves_vss|BRM_saves_journal|BRM_saves_current|oidbitmap" @@ -3459,7 +3475,7 @@ process_s3_dbrm_restore() { printf "[!] Failed to smput: BRM_saves_current\n" else printf "." - fi + fi em_files=( "BRM_saves_journal" @@ -3485,7 +3501,7 @@ process_s3_dbrm_restore() { printf "Done\n" fi printf "%-${printf_offset}s ... " " - clearShm" - clearShm + clearShm printf "Done\n" sleep 2 @@ -3503,7 +3519,7 @@ process_s3_dbrm_restore() { echo "--------------------------------------------------------------------------" current_status=$(smls /data1/systemFiles/dbrm/ 2>/dev/null); if [ $? -ne 0 ]; then - printf "\n[!] Failed to get smls status\n\n" + printf "\n[!] Failed to get smls status\n\n" exit 1 fi echo "$current_status" | grep -E "BRM_saves_em|BRM_saves_vbbm|BRM_saves_vss|BRM_saves_journal|BRM_saves_current|oidbitmap" @@ -3528,7 +3544,7 @@ process_s3_dbrm_restore() { printf "\nDBRM Restore Complete\n\n" } -# Depends on +# Depends on # em_file - most recent EM # em_file_full_path - full path to most recent EM # em_file_name - Just the file name of the EM @@ -3541,9 +3557,9 @@ process_localstorage_dbrm_restore() { printf " - Clearing active DBRMs ... " if rm -rf $dbrm_dir ; then printf "Done\n" - else + else echo "Failed to delete files in $dbrm_dir " - exit 1; + exit 1; fi printf "\nRestoring DBRMs\n" @@ -3551,7 +3567,7 @@ process_localstorage_dbrm_restore() { printf " - Desired EM: $em_file_full_path\n" printf " - Copying DBRMs: \"${backup_location}/${backup_folder_to_restore_dbrms}\" -> \"$dbrm_dir\" \n" cp -arp "${backup_location}/${backup_folder_to_restore_dbrms}" $dbrm_dir - + if [ "$prefix" != "BRM_saves" ]; then printf " - Restoring Prefix: $prefix \n" @@ -3567,7 +3583,7 @@ process_localstorage_dbrm_restore() { sleep 2 manually_run_loadbrm_and_savebrm - + printf "\nAfter DBRM Restore\n" echo "--------------------------------------------------------------------------" ls -la "${dbrm_dir}" | grep -E "BRM_saves_em|BRM_saves_vbbm|BRM_saves_vss|BRM_saves_journal|BRM_saves_current" @@ -3594,45 +3610,45 @@ process_localstorage_dbrm_restore() { manually_run_loadbrm_and_savebrm() { - pf_offset=45 - printf "%-${pf_offset}s ... " " - Running load_brm" + local print_formatting=45 + printf "%-${print_formatting}s ... " " - Running load_brm" if ! sudo -su mysql /usr/bin/load_brm /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves ; then printf "\n[!!] Failed to complete load_brm successfully\n\n" exit 1; fi; - printf "%-${pf_offset}s ... " " - Starting mcs-controllernode" + printf "%-${print_formatting}s ... " " - Starting mcs-controllernode" if ! systemctl start mcs-controllernode ; then echo "[!!] Failed to start mcs-controllernode " exit 1; else printf "Done\n" - fi; + fi; - printf "%-${pf_offset}s ... " " - Confirming extent map readable" + printf "%-${print_formatting}s ... " " - Confirming extent map readable" if ! editem -i >/dev/null ; then echo "[!!] Failed to run editem -i (read the EM)" exit 1; - else + else printf "Done\n" fi; - printf "%-${pf_offset}s ... \n" " - Running save_brm" + printf "%-${print_formatting}s ... \n" " - Running save_brm" if ! sudo -u mysql /usr/bin/save_brm ; then echo "[!!] Failed to run save_brm" exit 1; fi - printf "%-${pf_offset}s ... " " - Stopping mcs-controllernode" + printf "%-${print_formatting}s ... " " - Stopping mcs-controllernode" if ! systemctl stop mcs-controllernode; then echo "[!!] Failed to stop mcs-controllernode" exit 1; else printf "Done\n" fi - - if [ $storage == "S3" ]; then - printf "%-${pf_offset}s ... " " - Stopping mcs-storagemanager" + + if [ $storage == "S3" ]; then + printf "%-${print_formatting}s ... " " - Stopping mcs-storagemanager" if ! systemctl stop mcs-storagemanager ; then echo "[!!] Failed to stop mcs-storagemanager " exit 1; @@ -3641,8 +3657,8 @@ manually_run_loadbrm_and_savebrm() { fi fi; - printf "%-${pf_offset}s ... " " - clearShm" - clearShm + printf "%-${print_formatting}s ... " " - clearShm" + clearShm printf "Done\n" sleep 2 @@ -3650,7 +3666,7 @@ manually_run_loadbrm_and_savebrm() { process_backup() -{ +{ load_default_backup_variables; parse_backup_variables "$@"; print_backup_variables; @@ -3678,29 +3694,29 @@ print_mcs_bk_mgr_version_info() { } case "$action" in - 'help' | '--help' | '-help' | '-h') + 'help' | '--help' | '-help' | '-h') print_action_help_text ;; - 'backup') + 'backup') process_backup "$@"; ;; - 'dbrm_backup') + 'dbrm_backup') process_dbrm_backup "$@"; ;; - 'dbrm_restore') + 'dbrm_restore') process_dbrm_restore "$@"; ;; - 'restore') + 'restore') process_restore "$@"; ;; - '-v' | 'version' ) + '-v' | 'version' | '--version') print_mcs_bk_mgr_version_info ;; 'source' ) return 0; ;; - *) - printf "\nunknown action: $action\n" + *) + printf "\nunknown action: $action\n" print_action_help_text ;; esac