#!/bin/bash ######################################################################## # Copyright (c) 2020 MariaDB Corporation Ab # # Use of this software is governed by the Business Source License included # in the LICENSE.TXT file and at www.mariadb.com/bsl11. # # Change Date: 2024-07-10 # # On the date above, in accordance with the Business Source License, use # of this software will be governed by version 2 or later of the General # Public License. # ######################################################################## # Documentation: bash mcs_backup_manager.sh help # Version: 3.4 # # Backup Example # LocalStorage: sudo ./mcs_backup_manager.sh backup # S3: sudo ./mcs_backup_manager.sh backup -bb s3://my-cs-backups # ######################################################################## # # Restore Example # LocalStorage: sudo ./mcs_backup_manager.sh restore -l # S3: sudo ./mcs_backup_manager.sh restore -bb s3://my-cs-backups -l # ######################################################################## start=$(date +%s) action=$1 print_action_help_text() { echo " MariaDB Columnstore Backup Manager Actions: backup Full & Incremental columnstore backup with additional flags to augment the backup taken restore Restore a backup taken with this script dbrm_backup Quick hot backup of internal columnstore metadata only - only use under support recommendation Documentation: bash $0 help Example: bash $0 backup help " } check_operating_system() { OPERATING_SYSTEM=$(awk -F= '/^ID=/{gsub(/"/, "", $2); print $2}' /etc/os-release) # Supported OS case $OPERATING_SYSTEM in centos ) return 1; ;; rhel ) return 1 ;; debian ) return 1; ;; rocky ) return 1; ;; ubuntu ) return 1; ;; *) # unknown option printf "\ncheck_operating_system: unknown os & version: $OPERATING_SYSTEM\n" exit 2; esac } load_default_backup_variables() { check_operating_system # What directory to store the backups on this machine or the target machine. # Consider write permissions of the scp user and the user running this script. # Mariadb-backup will use this location as a tmp dir for S3 and remote backups temporarily # Example: /mnt/backups/ backup_location="/tmp/backups/" # Are the backups going to be stored on the same machine this script is running on or another server - if Remote you need to setup scp= # Options: "Local" or "Remote" backup_destination="Local" # Used only if backup_destination="Remote" # The user/credentials that will be used to scp the backup files # Example: "centos@10.14.51.62" scp="" # Only used if storage=S3 # Name of the bucket to store the columnstore backups # Example: "s3://my-cs-backups" backup_bucket="" # OS specific Defaults case $OPERATING_SYSTEM in centos | rocky | rhel ) MARIADB_SERVER_CONFIGS_PATH="/etc/my.cnf.d" ;; ubuntu | debian ) MARIADB_SERVER_CONFIGS_PATH="/etc/mysql/mariadb.conf.d/" ;; *) # unknown option handle_failed_dependencies "\nload_default_backup_variables: unknown os & version: $OPERATING_SYSTEM\n"; esac # Fixed Paths CS_CONFIGS_PATH="/etc/columnstore" DBRM_PATH="/var/lib/columnstore/data1/systemFiles/dbrm" STORAGEMANAGER_PATH="/var/lib/columnstore/storagemanager" STORAGEMANGER_CNF="$CS_CONFIGS_PATH/storagemanager.cnf" # Configurable Paths cs_metadata=$(grep ^metadata_path $STORAGEMANGER_CNF | cut -d "=" -f 2 | tr -d " ") cs_journal=$(grep ^journal_path $STORAGEMANGER_CNF | cut -d "=" -f 2 | tr -d " ") cs_cache=$(grep -A25 "\[Cache\]" $STORAGEMANGER_CNF | grep ^path | cut -d "=" -f 2 | tr -d " ") # What storage topogoly is being used by Columnstore - found in /etc/columnstore/storagemanager.cnf # Options: "LocalStorage" or "S3" storage=$(grep -m 1 "^service = " $STORAGEMANGER_CNF | awk '{print $3}') # Name of the existing bucket used in the cluster - found in /etc/columnstore/storagemanager.cnf # Example: "mcs-20211101172656965400000001" bucket=$( grep -m 1 "^bucket =" $STORAGEMANGER_CNF | awk '{print $3}') # modes ['direct','indirect'] - direct backups run on the columnstore nodes themselves. indirect run on another machine that has read-only mounts associated with columnstore/mariadb mode="direct" # Name of the Configuration file to load variables from config_file=".cs-backup-config" # Track your write speed with "dstat --top-cpu --top-io" # Monitor individual rsyncs with ` watch -n0.25 "ps aux | grep 'rsync -a' | grep -vE 'color|grep' | wc -l; ps aux | grep 'rsync -a' | grep -vE 'color|grep' " ` # Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes parrallel_rsync=false # Directory stucture: /var/lib/columnstore/data$dbroot/$dir1.dir/$dir2.dir/$dir3.dir/$dir4.dir/$partition.dir/FILE.$seqnum.cdf ; # DEPTH [1,2,3,4] determines at what level to begin (dir1/dir2/dir3/dir4) the number of PARALLEL_FOLDERS to issue DEPTH=3 # PARALLEL_FOLDERS determines the number of parent directories to have rsync x PARALLEL_THREADS running at the same time ( for DEPTH=3 i.e /var/lib/columnstore/data$dbroot/$dir1.dir/$dir2.dir/$dir3.dir) PARALLEL_FOLDERS=4 # PARALLEL_THREADS determines the number of threads to rsync subdirectories of the set DEPTH level ( for DEPTH=3 i.e /var/lib/columnstore/data$dbroot/$dir1.dir/$dir2.dir/$dir3.dir/X) # PARALLEL_THREADS is also used in for pigz compression parallelism PARALLEL_THREADS=4 # Google Cloud # PARALLEL_UPLOAD=50 RETRY_LIMIT=3 # Other Variables #today=$(date +%m-%d-%Y-%H%M%S) today=$(date +%m-%d-%Y) HA=false if [ ! -f /var/lib/columnstore/local/module ]; then pm="pm1"; else pm=$(cat /var/lib/columnstore/local/module); fi; PM_NUMBER=$(echo "$pm" | tr -dc '0-9') if [[ -z $PM_NUMBER ]]; then PM_NUMBER=1; fi; #source_ips=$(grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}" /etc/columnstore/Columnstore.xml) #source_host_names=$(grep "" /etc/columnstore/Columnstore.xml) cmapi_key="$(grep "x-api-key" $CS_CONFIGS_PATH/cmapi_server.conf | awk '{print $3}' | tr -d "'" )"; final_message="Backup Complete" skip_save_brm=false skip_locks=false skip_polls=false skip_mdb=false skip_bucket_data=false quiet=false xtra_s3_args="" xtra_cmd_args="" rsync_flags=" -av"; poll_interval=5 poll_max_wait=60; # Compression Variables compress_format="" split_file_mdb_prefix="mariadb-backup.tar" split_file_cs_prefix="cs-localfiles.tar" timeout=999999 logfile="backup.log" col_xml_backup="Columnstore.xml.source_backup" cmapi_backup="cmapi_server.conf.source_backup" # Used by on premise S3 vendors # Example: "http://127.0.0.1:8000" s3_url="" no_verify_ssl=false # Tracks if flush read lock has been run read_lock=false incremental=false columnstore_online=false confirm_xmllint_installed # Number of DBroots # Integer usually 1 or 3 DBROOT_COUNT=$(xmllint --xpath "string(//DBRootCount)" $CS_CONFIGS_PATH/Columnstore.xml) ASSIGNED_DBROOT=$(xmllint --xpath "string(//ModuleDBRootID$PM_NUMBER-1-3)" $CS_CONFIGS_PATH/Columnstore.xml) } parse_backup_variables() { # Dynamic Arguments while [[ $# -gt 0 ]]; do key="$1" case $key in backup) shift # past argument ;; -bl|--backup-location) backup_location="$2" shift # past argument shift # past value ;; -bd|--backup-destination) backup_destination="$2" shift # past argument shift # past value ;; -scp|--secure-copy-protocol) scp="$2" shift # past argument shift # past value ;; -bb|--backup-bucket) backup_bucket="$2" shift # past argument shift # past value ;; -url| --endpoint-url) s3_url="$2" shift # past argument shift # past value ;; -s|--storage) storage="$2" shift # past argument shift # past value ;; -i|--incremental) incremental=true today="$2" shift # past argument shift # past value ;; -P|--parallel) parrallel_rsync=true PARALLEL_THREADS="$2" shift # past argument shift # past value ;; -ha | --highavilability) HA=true shift # past argument ;; -f| --config-file) config_file="$2" shift # past argument shift # past value ;; -sbrm| --skip-save-brm) skip_save_brm=true shift # past argument ;; -spoll| --skip-polls) skip_polls=true shift # past argument ;; -slock| --skip-locks) skip_locks=true shift # past argument ;; -smdb | --skip-mariadb-backup) skip_mdb=true shift # past argument ;; -sb | --skip-bucket-data) skip_bucket_data=true shift # past argument ;; -nb | --name-backup) today="$2" shift # past argument shift # past value ;; -m | --mode) mode="$2" shift # past argument shift # past value ;; -c | --compress) compress_format="$2" shift # past argument shift # past value ;; -q | --quiet) quiet=true shift # past argument ;; -nv-ssl| --no-verify-ssl) no_verify_ssl=true shift # past argument ;; -pi| --poll-interval) poll_interval="$2" shift # past argument shift # past value ;; -pmw| --poll-max-wait) poll_max_wait="$2" shift # past argument shift # past value ;; -h|--help|-help|help) print_backup_help_text; exit 1; ;; *) # unknown option handle_early_exit_on_backup "\nunknown flag: $1\n" true esac done # Load config file if [ -f $config_file ]; then source $config_file fi # Adjustment for indirect mode if [ $mode == "indirect" ]; then skip_locks=true; skip_save_brm=true; fi; } print_backup_help_text() { echo " Columnstore Backup -bl | --backup-location Directory where the backup will be saved -bd | --backup-destination If the directory is 'Local' or 'Remote' to this script -scp | --secure-copy-protocol scp connection to remote server if -bd 'Remote' -bb | --backup-bucket Bucket name for where to save S3 backups -url | --endpoint-url Onprem url to s3 storage api example: http://127.0.0.1:8000 -nv-ssl| --no-verify-ssl Skips verifying ssl certs, useful for onpremise s3 storage -s | --storage The storage used by columnstore data 'LocalStorage' or 'S3' -i | --incremental Adds columnstore deltas to an existing full backup -P | --parallel Number of parallel rsync/compression threads to run -f | --config-file Path to backup configuration file to load variables from -sbrm | --skip-save-brm Skip saving brm prior to running a backup - ideal for dirty backups -slock | --skip-locks Skip issuing write locks - ideal for dirty backups -spoll | --skip-polls Skip sql checks confirming no write/cpimports running -smdb | --skip-mariadb-backup Skip running a mariadb-backup for innodb data - ideal for incremental dirty backups -sb | --skip-bucket-data Skip taking a copy of the columnstore data in the bucket -pi | --poll-interval Number of seconds between poll checks for active writes & cpimports -pmw | --poll-max-wait Max number of minutes for polling checks for writes to wait before exiting as a failed backup attempt -q | --quiet Silence verbose copy command outputs -c | --compress Compress backup in X format - Options: [ pigz ] -nb | --name-backup Define the name of the backup - default: date +%m-%d-%Y -ha | --highavilability Hint wether shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) HA S3 ( /var/lib/columnstore/storagemanager/ ) Local Storage Examples: ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage -P 8 ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage --incremental 02-18-2022 ./$0 backup -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -s LocalStorage S3 Examples: ./$0 backup -bb s3://my-cs-backups -s S3 ./$0 backup -bb s3://my-cs-backups -c pigz --quiet -sb ./$0 backup -bb gs://my-cs-backups -s S3 --incremental 02-18-2022 ./$0 backup -bb s3://my-onpremise-bucket -s S3 -url http://127.0.0.1:8000 Cron Example: */60 */24 * * * root bash /root/$0 -bb s3://my-cs-backups -s S3 >> /root/csBackup.log 2>&1 "; # Hidden flags # -m | --mode Options ['direct','indirect'] - direct backups run on the columnstore nodes themselves. indirect run on another machine that has read-only mounts associated with columnstore/mariadb } print_backup_variables() { # Log Variables s1=20 s2=20 printf "\nBackup Variables\n" echo "--------------------------------------------------------------------------" if [ -f $config_file ]; then printf "%-${s1}s %-${s2}s\n" "Configuration File:" "$config_file"; fi echo "--------------------------------------------------------------------------" printf "%-10s %-10s %-10s %-10s %-10s\n" "Skips:" "BRM($skip_save_brm)" "Locks($skip_locks)" "MariaDB($skip_mdb)" "Bucket($skip_bucket_data)"; echo "--------------------------------------------------------------------------" printf "%-${s1}s %-${s2}s\n" "Storage:" "$storage"; printf "%-${s1}s %-${s2}s\n" "Folder:" "$today"; printf "%-${s1}s %-${s2}s\n" "DB Root Count:" "$DBROOT_COUNT"; printf "%-${s1}s %-${s2}s\n" "PM:" "$pm"; printf "%-${s1}s %-${s2}s\n" "Highly Available:" "$HA"; printf "%-${s1}s %-${s2}s\n" "Incremental:" "$incremental"; printf "%-${s1}s %-${s2}s\n" "Timestamp:" "$(date +%m-%d-%Y-%H%M%S)"; if [[ -n "$compress_format" ]]; then printf "%-${s1}s %-${s2}s\n" "Compression:" "true"; printf "%-${s1}s %-${s2}s\n" "Compression Format:" "$compress_format"; printf "%-${s1}s %-${s2}s\n" "Compression Threads:" "$PARALLEL_THREADS"; else printf "%-${s1}s %-${s2}s\n" "Parallel Enabled:" "$parrallel_rsync"; if $parrallel_rsync ; then printf "%-${s1}s %-${s2}s\n" "Parallel Threads:" "$PARALLEL_THREADS"; fi; fi if [ $storage == "LocalStorage" ]; then printf "%-${s1}s %-${s2}s\n" "Backup Destination:" "$backup_destination"; if [ $backup_destination == "Remote" ]; then printf "%-${s1}s %-${s2}s\n" "scp:" "$scp"; fi; printf "%-${s1}s %-${s2}s\n" "Backup Location:" "$backup_location"; fi if [ $storage == "S3" ]; then printf "%-${s1}s %-${s2}s\n" "Active Bucket:" "$bucket"; printf "%-${s1}s %-${s2}s\n" "Backup Bucket:" "$backup_bucket"; fi echo "--------------------------------------------------------------------------" } check_package_managers() { package_manager=''; if command -v apt &> /dev/null ; then if ! command -v dpkg-query &> /dev/null ; then printf "[!!] Cant access dpkg-query\n" exit 1; fi package_manager="apt"; fi if command -v yum &> /dev/null ; then package_manager="yum"; fi if [ $package_manager == '' ]; then handle_failed_dependencies "[!!] No package manager found: yum or apt must be installed" exit 1; fi; } confirm_xmllint_installed() { if ! command -v xmllint > /dev/null; then printf "[!] xmllint not installed ... attempting auto install\n\n" check_package_managers case $package_manager in yum ) install_command="yum install libxml2 -y"; ;; apt ) install_command="apt install libxml2-utils -y --quiet"; ;; *) # unknown option handle_failed_dependencies "[!!] package manager not implemented: $package_manager\n" exit 2; esac if ! eval $install_command; then handle_failed_dependencies "[!] Failed to install xmllint\nThis is required." exit 1; fi fi } confirm_rsync_installed() { if ! command -v rsync > /dev/null; then printf "[!] rsync not installed ... attempting auto install\n\n" check_package_managers case $package_manager in yum ) install_command="yum install rsync -y"; ;; apt ) install_command="apt install rsync -y --quiet"; ;; *) # unknown option log_debug_message "[!!] package manager not implemented: $package_manager\n" exit 2; esac if ! eval $install_command; then handle_failed_dependencies "[!] Failed to install rsync\nThis is required." exit 1; fi fi } confirm_mariadb_backup_installed() { if $skip_mdb ; then return; fi; if ! command -v mariadb-backup > /dev/null; then printf "[!] mariadb-backup not installed ... attempting auto install\n\n" check_package_managers case $package_manager in yum ) install_command="yum install MariaDB-backup -y"; ;; apt ) install_command="apt install mariadb-backup -y --quiet"; ;; *) # unknown option log_debug_message "[!!] package manager not implemented: $package_manager\n" exit 2; esac if ! eval $install_command; then handle_failed_dependencies "[!] Failed to install MariaDB-backup\nThis is required." exit 1; fi fi } confirm_pigz_installed() { if ! command -v pigz > /dev/null; then printf "[!] pigz not installed ... attempting auto install\n\n" check_package_managers case $package_manager in yum ) install_command="yum install pigz -y"; ;; apt ) install_command="apt install pigz -y --quiet"; ;; *) # unknown option log_debug_message "[!!] package manager not implemented: $package_manager\n" exit 2; esac if ! eval $install_command; then handle_failed_dependencies "[!] Failed to install pigz\nThis is required for '-c pigz'" exit 1; fi fi } check_for_dependancies() { # Check pidof works if [ $mode != "indirect" ] && ! command -v pidof > /dev/null; then handle_failed_dependencies "\n\n[!] Please make sure pidof is installed and executable\n\n" fi # used for save_brm and defining columnstore_user if ! command -v stat > /dev/null; then handle_failed_dependencies "\n\n[!] Please make sure stat is installed and executable\n\n" fi confirm_rsync_installed confirm_mariadb_backup_installed if [ $1 == "backup" ] && [ $mode != "indirect" ] && ! command -v dbrmctl > /dev/null; then handle_failed_dependencies "\n\n[!] dbrmctl unreachable to issue lock \n\n" fi if [ $storage == "S3" ]; then # Default cloud cloud="aws" # Critical argument for S3 - determine which cloud if [ -z "$backup_bucket" ]; then handle_failed_dependencies "\n undefined --backup_bucket: $backup_bucket \nfor examples see: ./$0 backup --help\n"; fi if [[ $backup_bucket == gs://* ]]; then cloud="gcp"; protocol="gs"; elif [[ $backup_bucket == s3://* ]]; then cloud="aws"; protocol="s3"; else handle_failed_dependencies "\n Invalid --backup_bucket - doesnt lead with gs:// or s3:// - $backup_bucket\n"; fi if [ $cloud == "gcp" ]; then # If GCP - Check gsutil installed gsutil="" protocol="gs" if command -v mcs_gsutil > /dev/null; then gsutil=$(which mcs_gsutil 2>/dev/null) elif ! command -v gsutil > /dev/null; then which gsutil echo "Hints: curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-443.0.0-linux-x86_64.tar.gz tar -xvf google-cloud-cli-443.0.0-linux-x86_64.tar.gz ./google-cloud-sdk/install.sh -q sudo ln -s \$PWD/google-cloud-sdk/bin/gsutil /usr/bin/gsutil Then A) gsutil config -a or B) gcloud auth activate-service-account --key-file=user-file.json " handle_failed_dependencies "\n\nPlease make sure gsutil cli is installed configured and executable\n" else gsutil=$(which gsutil 2>/dev/null) fi # gsutil sytax for silent if $quiet; then xtra_s3_args+="-q"; fi else # on prem S3 will use aws cli # If AWS - Check aws-cli installed awscli="" protocol="s3" if command -v mcs_aws > /dev/null; then awscli=$(which mcs_aws 2>/dev/null) elif ! command -v aws > /dev/null; then which aws echo "Hints: curl \"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip\" -o \"awscliv2.zip\" unzip awscliv2.zip sudo ./aws/install aws configure" handle_failed_dependencies "\n\n Please make sure aws cli is installed configured and executable\nSee existing .cnf aws credentials with: grep aws_ $STORAGEMANGER_CNF \n\n" else awscli=$(which aws 2>/dev/null) fi # aws sytax for silent if $quiet; then xtra_s3_args+="--quiet"; fi fi; fi } validation_prechecks_for_backup() { echo "Prechecks ..." # Adjust rsync for incremental copy additional_rysnc_flags="" if $quiet; then xtra_cmd_args+=" 2> /dev/null"; additional_rysnc_flags+=" -a"; else additional_rysnc_flags+=" -av "; fi if $incremental ; then additional_rysnc_flags+=" --inplace --no-whole-file --delete"; fi; if [ -z "$mode" ]; then printf "\n[!!!] Required field --mode: $mode - is empty\n"; exit 1; fi if [ "$mode" != "direct" ] && [ "$mode" != "indirect" ] ; then printf "\n[!!!] Invalid field --mode: $mode\n"; exit 1; fi # Poll Variable Checks confirm_numerical_or_decimal_else_fail "$poll_interval" "poll_interval" confirm_numerical_or_decimal_else_fail "$poll_max_wait" "poll_max_wait" max_poll_attempts=$((poll_max_wait * 60 / poll_interval)) if [ "$max_poll_attempts" -lt 1 ]; then max_poll_attempts=1; fi; # Detect if columnstore online if [ $mode == "direct" ]; then if [ -z $(pidof PrimProc) ] || [ -z $(pidof WriteEngineServer) ]; then printf " - Columnstore is OFFLINE \n"; export columnstore_online=false; else printf " - Columnstore is ONLINE - safer if offline \n"; export columnstore_online=true; fi fi; # Validate compression option if [[ -n "$compress_format" ]]; then case $compress_format in pigz) confirm_pigz_installed ;; *) # unknown option handle_early_exit_on_backup "\n[!!!] Invalid field --compress: $compress_format\n\n" true esac fi; # If Remote save - Check ssh works to remote if [ $backup_destination == "Remote" ]; then if ssh -o ConnectTimeout=10 $scp echo ok 2>&1 ;then printf 'SSH Works\n' else handle_early_exit_on_backup "Failed Command: ssh $scp\n" true fi fi # Storage Based Checks if [ $storage == "LocalStorage" ]; then # Incremental Job checks if $incremental; then # Cant continue if this folder (which represents a full backup) doesnt exists if [ $backup_destination == "Local" ]; then if [ -d $backup_location$today ]; then printf "[+] Full backup directory exists\n"; else handle_early_exit_on_backup "[X] Full backup directory ($backup_location$today) DOES NOT exist \n\n" true; fi; elif [ $backup_destination == "Remote" ]; then if [[ $(ssh $scp test -d $backup_location$today && echo exists) ]]; then printf "[+] Full backup directory exists\n"; else handle_early_exit_on_backup "[X] Full backup directory ($backup_location$today) DOES NOT exist on remote $scp \n\n" true; fi fi fi elif [ $storage == "S3" ]; then # Adjust s3api flags for onpremise/custom endpoints add_s3_api_flags="" if [ -n "$s3_url" ]; then add_s3_api_flags+=" --endpoint-url $s3_url"; fi; if $no_verify_ssl; then add_s3_api_flags+=" --no-verify-ssl"; fi; # Validate addtional relevant arguments for S3 if [ -z "$backup_bucket" ]; then echo "Invalid --backup_bucket: $backup_bucket - is empty"; exit 1; fi # Check cli access to bucket if [ $cloud == "gcp" ]; then if $gsutil ls $backup_bucket > /dev/null ; then printf " - Success listing backup bucket\n" else printf "\n[X] Failed to list bucket contents... \nCheck $gsutil credentials: $gsutil config -a" handle_early_exit_on_backup "\n$gsutil ls $backup_bucket \n\n" true fi else # Check aws cli access to bucket if $( $awscli $add_s3_api_flags s3 ls $backup_bucket > /dev/null ) ; then printf " - Success listing backup bucket\n" else printf "\n[X] Failed to list bucket contents... \nCheck aws cli credentials: aws configure" handle_early_exit_on_backup "\naws $add_s3_api_flags s3 ls $backup_bucket \n" true fi fi; # Incremental Job checks if $incremental; then # Cant continue if this folder (which represents a full backup) doesnt exists if [ $cloud == "gcp" ]; then if [[ $( $gsutil ls $backup_bucket/$today | head ) ]]; then printf "[+] Full backup directory exists\n"; else handle_early_exit_on_backup "[X] Full backup directory ($backup_bucket/$today) DOES NOT exist in GCS \nCheck - $gsutil ls $backup_bucket/$today | head \n\n" true; fi else if [[ $( $awscli $add_s3_api_flags s3 ls $backup_bucket/$today | head ) ]]; then printf "[+] Full backup directory exists\n"; else handle_early_exit_on_backup "[X] Full backup directory ($backup_bucket/$today) DOES NOT exist in S3 \nCheck - aws $add_s3_api_flags s3 ls $backup_bucket/$today | head \n\n" true; fi fi; fi else handle_early_exit_on_backup "Invalid Variable storage: $storage" true fi } cs_read_only_wait_loop() { retry_limit=1800 retry_counter=0 read_only_mode_message="DBRM is currently Read Only!" current_status=$(dbrmctl status); printf " - Confirming CS Readonly... " while [ "$current_status" != "$read_only_mode_message" ]; do sleep 1 printf "." current_status=$(dbrmctl status); if [ $? -ne 0 ]; then handle_early_exit_on_backup "\n[!] Failed to get dbrmctl status\n\n" fi if [ $retry_counter -ge $retry_limit ]; then handle_early_exit_on_backup "\n[!] Set columnstore readonly wait retry limit exceeded: $retry_counter \n\n" fi ((retry_counter++)) done printf "Done\n" } # Having columnstore offline is best for non-volatile backups # If online - issue a flush tables with read lock and set DBRM to readonly issue_write_locks() { if [ $mode == "indirect" ] || $skip_locks; then printf "\n"; return; fi; if ! $skip_mdb && ! pidof mariadbd > /dev/null; then handle_early_exit_on_backup "\n[X] MariaDB is offline ... Needs to be online to issue read only lock and to run mariadb-backup \n\n" true; fi; printf "\nLocks \n"; # Pre 23.10.2 CS startreadonly doesnt exist - so poll cpimports added to protect them if ! $skip_polls; then poll_check_no_active_sql_writes poll_check_no_active_cpimports fi; ORIGINAL_READONLY_STATUS=$(mariadb -qsNe "select @@read_only;") # Set MariaDB Server ReadOnly Mode if pidof mariadbd > /dev/null; then printf " - Issuing read-only lock to MariaDB Server ... "; if mariadb -e "FLUSH TABLES WITH READ LOCK;"; then mariadb -e "set global read_only=ON;" read_lock=true printf " Done\n"; else handle_early_exit_on_backup "\n[X] Failed issuing read-only lock\n\n" fi fi if [ $pm == "pm1" ]; then # Set Columnstore ReadOnly Mode startreadonly_exists=$(dbrmctl -h 2>&1 | grep "startreadonly") printf " - Issuing read-only lock to Columnstore Engine ... "; if ! $columnstore_online; then printf "Skip since offline\n\n"; elif [ $DBROOT_COUNT == "1" ] && [[ -n "$startreadonly_exists" ]]; then if dbrmctl startreadonly ; then cs_read_only_wait_loop printf " \n"; else handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via dbrmctl startreadonly\n" fi; elif [ $DBROOT_COUNT == "1" ]; then if dbrmctl readonly ; then printf " \n"; else handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via dbrmctl readonly \n" fi else if command -v mcs &> /dev/null && command -v jq &> /dev/null ; then if ! mcs cluster set mode --mode readonly | jq -r tostring ; then handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n" fi else # Older CS versions dont have mcs cli cmapiResponse=$(curl -s -X PUT https://127.0.0.1:8640/cmapi/0.4.0/cluster/mode-set --header 'Content-Type:application/json' --header "x-api-key:$cmapi_key" --data '{"timeout":20, "mode": "readonly"}' -k); if [[ $cmapiResponse == '{"error":'* ]] ; then handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n" else printf "$cmapiResponse \n"; fi fi fi else cs_read_only_wait_loop printf " \n"; fi } poll_check_no_active_sql_writes() { printf " - Polling For Active Writes ... " query="SELECT COUNT(*) FROM information_schema.processlist where user != 'root' AND command = 'Query' AND ( info LIKE '%INSERT%' OR info LIKE '%UPDATE%' OR info LIKE '%DELETE%' OR info LIKE '%LOAD DATA%');" attempts=0 no_writes=false while [ "$attempts" -lt "$max_poll_attempts" ]; do active_writes=$( mariadb -s -N -e "$query" ) if [ "$active_writes" -le 0 ]; then printf "Done\n" no_writes=true break else # extra_info=$( mariadb -e "SELECT * FROM information_schema.processlist where user != 'root' AND command = 'Query' AND info LIKE '%INSERT%' OR info LIKE '%UPDATE%' OR info LIKE '%DELETE%' " ) # echo "Active write operations detected: $active_writes" # echo "$extra_info" printf "." if ! $quiet; then printf "\nActive write operations detected: $active_writes"; fi; sleep "$poll_interval" ((attempts++)) fi done if ! $no_writes; then handle_early_exit_on_backup "\n[X] Exceeded poll_max_wait: $poll_max_wait minutes\nActive write operations detected: $active_writes \n" true fi; } poll_check_no_active_cpimports() { printf " - Polling For Active cpimports ... " attempts=0 no_cpimports=false if ! $columnstore_online ; then printf "Skip since offline\n"; return ; fi; while [ "$attempts" -lt "$max_poll_attempts" ]; do active_cpimports=$(viewtablelock 2>/dev/null ) if [[ "$active_cpimports" == *"No tables are locked"* ]]; then printf "Done\n" no_cpimports=true break else printf "." if ! $quiet; then printf "\n$active_cpimports"; fi; sleep "$poll_interval" ((attempts++)) fi done if ! $no_cpimports; then handle_early_exit_on_backup "\n[X] Exceeded poll_max_wait: $poll_max_wait minutes\nActive cpimports\n$active_cpimports \n" true fi; } run_save_brm() { if $skip_save_brm || [ $pm != "pm1" ] || ! $columnstore_online || [ $mode == "indirect" ]; then return; fi; printf "\nBlock Resolution Manager\n" local tmpDir="/tmp/DBRMbackup-$today" if [ ! -d "$tmpDir" ]; then mkdir -p $tmpDir; fi; # Copy extent map locally just in case save_brm fails if [ $storage == "LocalStorage" ]; then cp -R $DBRM_PATH/* $tmpDir elif [ $storage == "S3" ]; then cp -R $STORAGEMANAGER_PATH/* $tmpDir # Base Set eval "smcat /data1/systemFiles/dbrm/BRM_saves_current > $tmpDir/BRM_saves_current $xtra_cmd_args" eval "smcat /data1/systemFiles/dbrm/BRM_saves_journal > $tmpDir/BRM_saves_journal $xtra_cmd_args" eval "smcat /data1/systemFiles/dbrm/BRM_saves_em > $tmpDir/BRM_saves_em $xtra_cmd_args" eval "smcat /data1/systemFiles/dbrm/BRM_saves_vbbm > $tmpDir/BRM_saves_vbbm $xtra_cmd_args" eval "smcat /data1/systemFiles/dbrm/BRM_saves_vss > $tmpDir/BRM_saves_vss $xtra_cmd_args" # A Set eval "smcat /data1/systemFiles/dbrm/BRM_savesA_em > $tmpDir/BRM_savesA_em $xtra_cmd_args" eval "smcat /data1/systemFiles/dbrm/BRM_savesA_vbbm > $tmpDir/BRM_savesA_vbbm $xtra_cmd_args" eval "smcat /data1/systemFiles/dbrm/BRM_savesA_vss > $tmpDir/BRM_savesA_vss $xtra_cmd_args" # B Set eval "smcat /data1/systemFiles/dbrm/BRM_savesB_em > $tmpDir/BRM_savesB_em $xtra_cmd_args" eval "smcat /data1/systemFiles/dbrm/BRM_savesB_vbbm > $tmpDir/BRM_savesB_vbbm $xtra_cmd_args" eval "smcat /data1/systemFiles/dbrm/BRM_savesB_vss > $tmpDir/BRM_savesB_vss $xtra_cmd_args" fi printf " - Saving BRMs... \n" brmOwner=$(stat -c "%U" $DBRM_PATH/) if sudo -u $brmOwner /usr/bin/save_brm ; then rm -rf $tmpDir else printf "\n Failed: save_brm error - see /var/log/messages - backup @ $tmpDir \n\n"; handle_early_exit_on_backup fi } # Example: s3sync # Example: s3sync s3://$bucket $backup_bucket/$today/columnstoreData "Done - Columnstore data sync complete" s3sync() { local from=$1 local to=$2 local success_message=$3 local failed_message=$4 local retries=${5:-0} local cmd="" if [ $cloud == "gcp" ]; then cmd="$gsutil $xtra_s3_args -m rsync -r -d $from $to" # gsutil throws WARNINGS if not directed to /dev/null if $quiet; then cmd+=" 2>/dev/null"; fi else # Default AWS cmd="$awscli $xtra_s3_args $add_s3_api_flags s3 sync $from $to" fi if eval $cmd; then if [ -n "$success_message" ]; then printf "$success_message"; fi; else if [ $retries -lt $RETRY_LIMIT ]; then echo "$cmd" echo "Retrying: $retries" sleep 1; s3sync "$1" "$2" "$3" "$4" "$(($retries+1))" else if [ -n "$failed_message" ]; then printf "$failed_message"; fi; echo "Was Trying: $cmd" echo "exiting ..." handle_early_exit_on_backup fi fi } # Example: s3cp s3cp() { local from=$1 local to=$2 local cmd="" if [ $cloud == "gcp" ]; then cmd="$gsutil $xtra_s3_args cp $from $to" else # Default AWS cmd="$awscli $xtra_s3_args $add_s3_api_flags s3 cp $from $to" fi; $cmd } # Example: s3rm s3rm() { local path=$1 local cmd="" if [ $cloud == "gcp" ]; then cmd="$gsutil $xtra_s3_args -m rm -r $path" else # Default AWS cmd="$awscli $xtra_s3_args $add_s3_api_flags s3 rm $path --recursive" fi; $cmd } # Example: s3ls s3ls() { local path=$1 local cmd="" if [ $cloud == "gcp" ]; then cmd="$gsutil ls $path" else # Default AWS cmd="$awscli $add_s3_api_flags s3 ls $path" fi; $cmd } clear_read_lock() { if [ $mode == "indirect" ] || $skip_locks; then return; fi; printf "\nClearing Locks\n"; # Clear MDB Lock if pidof mariadbd > /dev/null && [ $read_lock ]; then printf " - Clearing read-only lock on MariaDB Server ... "; if mariadb -e "UNLOCK TABLES;" && mariadb -qsNe "set global read_only=$ORIGINAL_READONLY_STATUS;"; then read_lock=false; printf " Done\n" else handle_early_exit_on_backup "\n[X] Failed clearing readLock\n" true fi fi if [ $pm == "pm1" ]; then # Clear CS Lock printf " - Clearing read-only lock on Columnstore Engine ... "; if ! $columnstore_online; then printf "Skip since offline\n" elif [ $DBROOT_COUNT == "1" ]; then if dbrmctl readwrite ; then printf " "; else handle_early_exit_on_backup "\n[X] Failed clearing columnstore BRM lock via dbrmctl\n" true; fi elif command -v mcs &> /dev/null && command -v jq &> /dev/null ; then if ! mcs cluster set mode --mode readwrite | jq -r tostring ;then handle_early_exit_on_backup "\n[X] Failed issuing columnstore BRM lock via cmapi\n"; fi elif curl -s -X PUT https://127.0.0.1:8640/cmapi/0.4.0/cluster/mode-set --header 'Content-Type:application/json' --header "x-api-key:$cmapi_key" --data '{"timeout":20, "mode": "readwrite"}' -k ; then printf " \n" else handle_early_exit_on_backup "\n[X] Failed clearing columnstore BRM lock\n" true; fi fi } # handle_ called when certain checks/functionality fails handle_failed_dependencies() { printf "\nDependency Checks Failed: $1" alert "$1" exit 1; } # first argument is the error message # 2nd argument true = skip clear_read_lock, false= dont skip handle_early_exit_on_backup() { skip_clear_locks=${2:-false} if ! $skip_clear_locks; then clear_read_lock; fi; printf "\nBackup Failed: $1\n" alert "$1" exit 1; } handle_early_exit_on_restore() { printf "\nRestore Failed: $1\n" alert "$1" exit 1; } handle_ctrl_c_backup() { echo "Ctrl+C captured. handle_early_exit_on_backup..." handle_early_exit_on_backup } alert() { # echo "Not implemented yet" # slack, email, webhook curl endpoint etc. return; } # initiate_rsync uses this to wait and monitor all running jobs # deepParallelRsync uses this to limit the parallelism to a max of $PARALLEL_FOLDERS times $PARALLEL_THREADS wait_on_rsync() { local visualize=$1 local pollSleepTime=$2 local concurrentThreshHold=$3 local dbrootToSync=$4 local rsyncInProgress="$(pgrep 'rsync -a' | wc -l )"; local total=0 local w=0; while [ $rsyncInProgress -gt "$concurrentThreshHold" ]; do if ! $quiet && $visualize && [ $(($w % 10)) -eq 0 ]; then if [[ $total == 0 ]]; then total=$(du -sh /var/lib/columnstore/data$dbrootToSync/) else echo -E "$rsyncInProgress rsync processes running ... seconds: $w" echo "Status: $(du -sh $backup_location$today/data$dbrootToSync/)" echo "Goal: $total" echo "Monitor rsync: ps aux | grep 'rsync -a' | grep -vE 'color|grep' " fi; fi #ps aux | grep 'rsync -a' | grep -vE 'color|grep' # DBEUG sleep $pollSleepTime rsyncInProgress="$(pgrep 'rsync -a' | wc -l )"; ((w++)) done } initiate_rsyncs() { local dbrootToSync=$1 parallel_rysnc_flags=" -a " if $incremental ; then parallel_rysnc_flags+=" --inplace --no-whole-file --delete"; fi; deepParallelRsync /var/lib/columnstore/data$dbrootToSync 1 $DEPTH data$dbrootToSync & sleep 2; #jobs wait_on_rsync true 2 0 $dbrootToSync wait } # A recursive function that increments depthCurrent+1 each directory it goes deeper and issuing rsync on each directory remaing at the target depth # Example values: # path: /var/lib/columnstore/data1 # depthCurrent: 1 # depthTarget: 3 # depthCurrent: data1 deepParallelRsync() { path=$1 depthCurrent=$2 depthTarget=$3 relativePath=$4 # echo "DEBUG: # path=$1 # depthCurrent=$2 # depthTarget=$3 # relativePath=$4" for fullFilePath in $path/*; do local fileName=$(basename $fullFilePath) # echo "DEBUG - filename: $fileName" # If a directory, keep going deeper until depthTarget reached if [ -d $fullFilePath ]; then #echo "DEBUG - Directory $file" mkdir -p $backup_location$today/$relativePath/$fileName if [ $depthCurrent -ge $depthTarget ]; then #echo "DEBUG - copy to relative: $backup_location$today/$relativePath/" if ls $fullFilePath | xargs -P $PARALLEL_THREADS -I {} rsync $parallel_rysnc_flags $fullFilePath/{} $backup_location$today/$relativePath/$fileName ; then echo " + Completed: $backup_location$today/$relativePath/$fileName" else echo "Failed: $backup_location$today/$relativePath/$fileName" exit 1; fi else # echo "DEBUG - Fork Deeper - $fullFilePath " wait_on_rsync false "0.5" $PARALLEL_FOLDERS # Since target depth not reached, recursively call for each directory deepParallelRsync $fullFilePath "$((depthCurrent+1))" $depthTarget "$relativePath/$fileName" & fi # If a file, rsync right away elif [ -f $fullFilePath ]; then rsync $additional_rysnc_flags $fullFilePath $backup_location$today/$relativePath/ # If filename is * then the directory is empty elif [ "$fileName" == "*" ]; then # echo "DEBUG - Skipping $relativePath - empty"; continue else printf "\n\n Warning - Unhandled - $fullFilePath \n\n"; fi done wait } run_backup() { if [ $storage == "LocalStorage" ]; then if [ $backup_destination == "Local" ]; then printf "\nLocal Storage Backup\n" # Check/Create Directories printf " - Creating Backup Directories... " if [[ -n "$compress_format" ]]; then mkdir -p $backup_location$today else mkdir -p $backup_location$today/mysql mkdir -p $backup_location$today/configs mkdir -p $backup_location$today/configs/mysql # Check/Create CS Data Directories i=1 while [ $i -le $DBROOT_COUNT ]; do if [[ $ASSIGNED_DBROOT == "$i" || $HA == true ]]; then mkdir -p $backup_location$today/data$i ; fi ((i++)) done fi printf " Done\n" # Backup Columnstore data i=1 while [ $i -le $DBROOT_COUNT ]; do if [[ $ASSIGNED_DBROOT == "$i" || $HA == true ]]; then if [[ -n "$compress_format" ]]; then # For compression keep track of dirs & files to include and compress/stream in the end - $compress_paths compress_paths="/var/lib/columnstore/data$i/* " elif $parrallel_rsync ; then printf " - Parallel Rsync CS Data$i... \n" initiate_rsyncs $i printf " Done\n" else printf " - Syncing Columnstore Data$i... " eval "rsync $additional_rysnc_flags /var/lib/columnstore/data$i/* $backup_location$today/data$i/ $xtra_cmd_args"; printf " Done\n" fi; fi ((i++)) done # Backup MariaDB data if [ $ASSIGNED_DBROOT == "1" ]; then # logic to increment mysql and keep count if we want to backup incremental mysql data # i=1 # latestMysqlIncrement=0 # while [ $i -le 365 ]; do # if [ -d $backup_location/mysql$i ]; then ; else latestMysqlIncrement=i; break; fi; # done # MariaDB backup wont rerun if folder exists - so clear it before running mariadb-backup if [ -d $backup_location$today/mysql ]; then rm -rf $backup_location$today/mysql fi # Run mariadb-backup if ! $skip_mdb; then if [[ -n "$compress_format" ]]; then printf " - Compressing MariaDB Data... " mbd_prefix="$backup_location$today/$split_file_mdb_prefix.$compress_format" case $compress_format in pigz) # Handle Cloud if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $mbd_prefix 2>> $logfile; then handle_early_exit_on_backup "\nFailed mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $mbd_prefix 2>> $logfile \n" fi printf " Done @ $mbd_prefix\n" ;; *) # unknown option handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" esac else printf " - Copying MariaDB Data... " if eval "mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root $xtra_cmd_args" ; then printf " Done \n"; else printf "\n Failed: mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root\n"; handle_early_exit_on_backup; fi fi else echo "[!] Skipping mariadb-backup" fi # Backup CS configurations if [[ ! -n "$compress_format" ]]; then printf " - Copying Columnstore Configs... " eval "rsync $additional_rysnc_flags $CS_CONFIGS_PATH/Columnstore.xml $backup_location$today/configs/ $xtra_cmd_args" eval "rsync $additional_rysnc_flags $STORAGEMANGER_CNF $backup_location$today/configs/ $xtra_cmd_args" if [ -f $CS_CONFIGS_PATH/cmapi_server.conf ]; then eval "rsync $additional_rysnc_flags $CS_CONFIGS_PATH/cmapi_server.conf $backup_location$today/configs/ $xtra_cmd_args"; fi printf " Done\n" else # When restoring - dont want to overwrite the new systems entire Columnstore.xml rm -rf $CS_CONFIGS_PATH/$col_xml_backup rm -rf $CS_CONFIGS_PATH/$cmapi_backup cp $CS_CONFIGS_PATH/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup cp $CS_CONFIGS_PATH/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup compress_paths+=" $CS_CONFIGS_PATH/$col_xml_backup $STORAGEMANGER_CNF $CS_CONFIGS_PATH/$cmapi_backup" fi; fi # Backup MariaDB configurations if ! $skip_mdb; then if [[ -z "$compress_format" ]]; then printf " - Copying MariaDB Configs... " mkdir -p $backup_location$today/configs/mysql/$pm/ eval "rsync $additional_rysnc_flags $MARIADB_SERVER_CONFIGS_PATH/* $backup_location$today/configs/mysql/$pm/ $xtra_cmd_args" printf " Done\n\n" else compress_paths+=" $MARIADB_SERVER_CONFIGS_PATH/*" fi fi # Handle compression for Columnstore Data & Configs if [[ -n "$compress_format" ]]; then cs_prefix="$backup_location$today/$split_file_cs_prefix.$compress_format" compressed_split_size="250M"; case $compress_format in pigz) printf " - Compressing CS Data & Configs... " if ! eval "tar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $cs_prefix 2>> $logfile"; then handle_early_exit_on_backup "[!] - Compression Failed \ntar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS -c > $cs_prefix 2>> $logfile \n" fi printf " Done @ $cs_prefix\n" ;; *) # unknown option handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" esac fi if $incremental ; then # Log each incremental run now=$(date "+%m-%d-%Y %H:%M:%S"); echo "$pm updated on $now" >> $backup_location$today/incrementallyUpdated.txt final_message="Incremental Backup Complete" else # Create restore job file extra_flags="" if [[ -n "$compress_format" ]]; then extra_flags+=" -c $compress_format"; fi; if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; echo "./$0 restore -l $today -bl $backup_location -bd $backup_destination -s $storage --dbroots $DBROOT_COUNT -m $mode $extra_flags --quiet" > $backup_location$today/restore.job fi final_message+=" @ $backup_location$today" elif [ $backup_destination == "Remote" ]; then # Check/Create Directories on remote server printf "[+] Checking Remote Directories... " i=1 makeDataDirectories="" while [ $i -le $DBROOT_COUNT ]; do makeDataDirectories+="mkdir -p $backup_location$today/data$i ;" ((i++)) done echo $makeDirectories ssh $scp " mkdir -p $backup_location$today/mysql ; $makeDataDirectories mkdir -p $backup_location$today/configs ; mkdir -p $backup_location$today/configs/mysql; mkdir -p $backup_location$today/configs/mysql/$pm " printf " Done\n" printf "[~] Rsync Remote Columnstore Data... \n" i=1 while [ $i -le $DBROOT_COUNT ]; do #if [ $pm == "pm$i" ]; then rsync -a $additional_rysnc_flags /var/lib/columnstore/data$i/* $scp:$backup_location$today/data$i/ ; fi if [ $pm == "pm$i" ]; then find /var/lib/columnstore/data$i/* -mindepth 1 -maxdepth 2 | xargs -P $PARALLEL_THREADS -I {} rsync $additional_rysnc_flags /var/lib/columnstore/data$i/* $scp:$backup_location$today/data$i/ fi ((i++)) done printf "[+] Done - rsync\n" # Backup MariaDB configurations if ! $skip_mdb; then printf "[~] Backing up MariaDB configurations... \n" rsync $additional_rysnc_flags $MARIADB_SERVER_CONFIGS_PATH/* $scp:$backup_location$today/configs/mysql/$pm/ printf "[+] Done - configurations\n" fi # Backup MariaDB data if [ $pm == "pm1" ]; then if ! $skip_mdb; then printf "[~] Backing up mysql... \n" mkdir -p $backup_location$today if mariadb-backup --backup --target-dir=$backup_location$today --user=root ; then rsync -a $backup_location$today/* $scp:$backup_location$today/mysql rm -rf $backup_location$today else printf "\n Failed: mariadb-backup --backup --target-dir=$backup_location$today --user=root\n\n"; handle_early_exit_on_backup fi else echo "[!] Skipping mariadb-backup" fi; # Backup CS configurations rsync $additional_rysnc_flags $CS_CONFIGS_PATH/Columnstore.xml $scp:$backup_location$today/configs/ rsync $additional_rysnc_flags $STORAGEMANGER_CNF $scp:$backup_location$today/configs/ if [ -f $CS_CONFIGS_PATH/cmapi_server.conf ]; then rsync $additional_rysnc_flags $CS_CONFIGS_PATH/cmapi_server.conf $scp:$backup_location$today/configs/; fi fi if $incremental ; then now=$(date "+%m-%d-%Y +%H:%M:%S") ssh $scp "echo \"$pm updated on $now\" >> $backup_location$today/incrementallyUpdated.txt" final_message="Incremental Backup Complete" else # Create restore job file ssh $scp "echo './$0 restore -l $today -bl $backup_location -bd $backup_destination -s $storage -scp $scp --dbroots $DBROOT_COUNT' > $backup_location$today/restore.job" fi final_message+=" @ $scp:$backup_location$today" fi elif [ $storage == "S3" ]; then printf "S3 Backup\n" # Conconsistency check - wait for assigned journal dir to be empty trap handle_ctrl_c_backup SIGINT i=1 j_counts=$(find $cs_journal/data$ASSIGNED_DBROOT/* -type f 2>/dev/null | wc -l) max_wait=180 printf " - Checking storagemanager/journal/data$ASSIGNED_DBROOT/* " while [[ $j_counts -gt 0 ]]; do if [ $i -gt $max_wait ]; then handle_early_exit_on_backup "\n[!] max_wait exceeded for $cs_journal/data$ASSIGNED_DBROOT/* to sync with bucket "; fi; if (( $i%10 == 0 )); then printf "\n[!] Not empty yet - found $j_counts files @ $cs_journal/data$ASSIGNED_DBROOT/*\n"; fi; sleep 1 i=$(($i+1)) j_counts=$(find $cs_journal/data$ASSIGNED_DBROOT/* -type f 2>/dev/null | wc -l) printf "." done printf " Done\n" # Backup storagemanager - columnstore s3 metadata, journal, cache if [[ -z "$compress_format" ]]; then printf " - Syncing Columnstore Metadata \n" if $HA; then s3sync $STORAGEMANAGER_PATH $backup_bucket/$today/storagemanager " - Done storagemanager/*\n" "\n\n[!!!] sync failed - storagemanager/*\n\n"; else s3sync $cs_cache/data$ASSIGNED_DBROOT $backup_bucket/$today/storagemanager/cache/data$ASSIGNED_DBROOT " + cache/data$ASSIGNED_DBROOT\n" "\n\n[!!!] sync failed - cache/data$ASSIGNED_DBROOT\n\n"; s3sync $cs_metadata/data$ASSIGNED_DBROOT $backup_bucket/$today/storagemanager/metadata/data$ASSIGNED_DBROOT " + metadata/data$ASSIGNED_DBROOT\n" "\n\n[!!!] sync failed - metadata/data$ASSIGNED_DBROOT\n\n" s3sync $cs_journal/data$ASSIGNED_DBROOT $backup_bucket/$today/storagemanager/journal/data$ASSIGNED_DBROOT " + journal/data$ASSIGNED_DBROOT\n" "\n\n[!!!] sync failed - journal/data$ASSIGNED_DBROOT\n\n" fi; else # For compression keep track of dirs & files to include and compress/stream in the end - $compress_paths compress_paths="$cs_cache/data$ASSIGNED_DBROOT $cs_metadata/data$ASSIGNED_DBROOT $cs_journal/data$ASSIGNED_DBROOT " fi # PM1 mostly backups everything else if [ $ASSIGNED_DBROOT == "1" ]; then # Backup CS configurations if [[ -z "$compress_format" ]]; then s3sync $CS_CONFIGS_PATH/ $backup_bucket/$today/configs/ " + $CS_CONFIGS_PATH/\n" "\n\n[!!!] sync failed - $CS_CONFIGS_PATH/\n\n"; else compress_paths+="$CS_CONFIGS_PATH " fi # Backup CS bucket data if ! $skip_bucket_data; then printf " - Saving Columnstore data ... " s3sync $protocol://$bucket $backup_bucket/$today/columnstoreData " Done \n" else printf " - [!] Skipping columnstore bucket data \n" fi # Backup MariaDB Server Data if ! $skip_mdb; then # Handle compressed mariadb backup if [[ -n "$compress_format" ]]; then printf " - Compressing MariaDB data local -> bucket... " mbd_prefix="$backup_bucket/$today/$split_file_mdb_prefix.$compress_format" case $compress_format in pigz) # Handle Cloud if [ $cloud == "gcp" ]; then if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter="gsutil cp - ${mbd_prefix}_\$FILE 2>$logfile" - chunk 2>$logfile; then handle_early_exit_on_backup "\nFailed mariadb-backup --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter=\"gsutil cp - ${mbd_prefix}_\$FILE 2>$logfile\" - chunk 2>$logfile \n" fi elif [ $cloud == "aws" ]; then if ! mariabackup --user=root --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter="aws s3 cp - ${mbd_prefix}_\$FILE 2>$logfile 1>&2" - chunk 2>$logfile; then handle_early_exit_on_backup "\nFailed mariadb-backup --backup --stream xbstream --parallel $PARALLEL_THREADS --ftwrl-wait-timeout=$timeout --ftwrl-wait-threshold=999999 --extra-lsndir=/tmp/checkpoint_out 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b 250M --filter=\"aws s3 cp - ${mbd_prefix}_\$FILE 2>$logfile 1>&2\" - chunk 2>$logfile\n" fi fi printf " Done @ $mbd_prefix\n" ;; *) # unknown option handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac else # Handle uncompressed mariadb backup printf " - Syncing MariaDB data ... \n" rm -rf $backup_location$today/mysql if mkdir -p $backup_location$today/mysql ; then if eval "mariabackup --user=root --backup --target-dir=$backup_location$today/mysql/ $xtra_cmd_args"; then printf " + mariadb-backup @ $backup_location$today/mysql/ \n" else handle_early_exit_on_backup "\nFailed mariadb-backup --backup --target-dir=$backup_location$today/mysql --user=root\n" true fi s3rm $backup_bucket/$today/mysql/ s3sync $backup_location$today/mysql $backup_bucket/$today/mysql/ " + mariadb-backup to bucket @ $backup_bucket/$today/mysql/ \n" rm -rf $backup_location$today/mysql else handle_early_exit_on_backup "\nFailed making directory for MariaDB backup\ncommand: mkdir -p $backup_location$today/mysql\n" fi fi else echo "[!] Skipping mariadb-backup" fi fi # Backup mariadb server configs if ! $skip_mdb; then if [[ -z "$compress_format" ]]; then printf " - Syncing MariaDB configurations \n" s3sync $MARIADB_SERVER_CONFIGS_PATH/ $backup_bucket/$today/configs/mysql/$pm/ " + $MARIADB_SERVER_CONFIGS_PATH/\n" "\n\n[!!!] sync failed - try alone and debug\n\n" else compress_paths+="$MARIADB_SERVER_CONFIGS_PATH" fi fi # Handles streaming the compressed columnstore local files if [[ -n "$compress_format" ]]; then cs_prefix="$backup_bucket/$today/$split_file_cs_prefix.$compress_format" compressed_split_size="250M"; case $compress_format in pigz) printf " - Compressing CS local files -> bucket ... " if [ $cloud == "gcp" ]; then if ! eval "tar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b $compressed_split_size --filter=\"gsutil cp - ${cs_prefix}_\\\$FILE 2>$logfile 1>&2\" - chunk 2>$logfile"; then handle_early_exit_on_backup "[!] - Compression/Split/Upload Failed \n" fi elif [ $cloud == "aws" ]; then if ! eval "tar cf - $compress_paths 2>>$logfile | pigz -p $PARALLEL_THREADS 2>> $logfile | split -d -a 5 -b $compressed_split_size --filter=\"aws s3 cp - ${cs_prefix}_\\\$FILE 2>$logfile 1>&2\" - chunk 2>$logfile"; then handle_early_exit_on_backup "[!] - Compression/Split/Upload Failed \n" fi fi printf " Done @ $cs_prefix\n" ;; *) # unknown option handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac fi # Create restore job file & update incremental log if $incremental ; then now=$(date "+%m-%d-%Y +%H:%M:%S") increment_file="incremental_history.txt" # download S3 file, append to it and reupload if [[ $(s3ls "$backup_bucket/$today/$increment_file") ]]; then s3cp $backup_bucket/$today/$increment_file $increment_file fi echo "$pm updated on $now" >> $increment_file s3cp $increment_file $backup_bucket/$today/$increment_file echo "[+] Updated $backup_bucket/$today/$increment_file" rm -rf $increment_file final_message="Incremental Backup Complete" else # Create restore job file extra_flags="" if [[ -n "$compress_format" ]]; then extra_flags+=" -c $compress_format"; fi; if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; if $skip_bucket_data; then extra_flags+=" --skip-bucket-data"; fi; if [ -n "$s3_url" ]; then extra_flags+=" -url $s3_url"; fi; echo "./$0 restore -l $today -s $storage -bb $backup_bucket -dbs $DBROOT_COUNT -m $mode -nb $protocol://$bucket $extra_flags --quiet --continue" > restoreS3.job s3cp restoreS3.job $backup_bucket/$today/restoreS3.job rm -rf restoreS3.job fi final_message+=" @ $backup_bucket/$today" fi clear_read_lock end=$(date +%s) runtime=$((end-start)) printf "\n[+] Runtime: $runtime\n" printf "[+] $final_message\n\n" } load_default_restore_variables() { # What date folder to load from the backup_location load_date='' # Where the backup to load is found # Example: /mnt/backups/ backup_location="/tmp/backups/" # Is this backup on the same or remote server compared to where this script is running # Options: "Local" or "Remote" backup_destination="Local" # Used only if backup_destination=Remote # The user/credentials that will be used to scp the backup files # Example: "centos@10.14.51.62" scp="" # What storage topogoly was used by Columnstore in the backup - found in storagemanager.cnf # Options: "LocalStorage" or "S3" storage="LocalStorage" # Flag for high available systems (meaning shared storage exists supporting the topology so that each node sees all data) HA=false # When set to true skips the enforcement that new_bucket should be empty prior to starting a restore continue=false # modes ['direct','indirect'] - direct backups run on the columnstore nodes themselves. indirect run on another machine that has read-only mounts associated with columnstore/mariadb mode="direct" # Name of the Configuration file to load variables from config_file=".cs-backup-config" # Only used if storage=S3 # Name of the bucket to copy into to store the backup S3 data # Example: "backup-mcs-bucket" backup_bucket='' # Fixed Paths MARIADB_PATH="/var/lib/mysql" CS_CONFIGS_PATH="/etc/columnstore" DBRM_PATH="/var/lib/columnstore/data1/systemFiles/dbrm" STORAGEMANAGER_PATH="/var/lib/columnstore/storagemanager" STORAGEMANGER_CNF="$CS_CONFIGS_PATH/storagemanager.cnf" # Configurable Paths cs_metadata=$(grep ^metadata_path $STORAGEMANGER_CNF | cut -d "=" -f 2 | tr -d " ") cs_journal=$(grep ^journal_path $STORAGEMANGER_CNF | cut -d "=" -f 2 | tr -d " ") cs_cache=$(grep -A25 "\[Cache\]" $STORAGEMANGER_CNF | grep ^path | cut -d "=" -f 2 | tr -d " ") if [ ! -f /var/lib/columnstore/local/module ]; then pm="pm1"; else pm=$(cat /var/lib/columnstore/local/module); fi; pm_number=$(echo "$pm" | tr -dc '0-9') # Defines the set of new bucket information to copy the backup to when restoring new_bucket='' new_region='' new_key='' new_secret='' # Used by on premise S3 vendors # Example: "http://127.0.0.1:8000" s3_url="" no_verify_ssl=false # Number of DBroots # Integer usually 1 or 3 DBROOT_COUNT=1 # Google Cloud # PARALLEL_UPLOAD=50 RETRY_LIMIT=3 # Compression Variables compress_format="" split_file_mdb_prefix="mariadb-backup.tar" split_file_cs_prefix="cs-localfiles.tar" timeout=999999 logfile="backup.log" col_xml_backup="Columnstore.xml.source_backup" cmapi_backup="cmapi_server.conf.source_backup" PARALLEL_THREADS=4 # Other final_message="Restore Complete" mariadb_user="mysql" columnstore_user="mysql" skip_mdb=false skip_bucket_data=false quiet=false xtra_s3_args="" xtra_cmd_args="" rsync_flags=" -av"; } parse_restore_variables() { # Dynamic Arguments while [[ $# -gt 0 ]]; do key="$1" case $key in restore) shift # past argument ;; -l|--load) load_date="$2" shift # past argument shift # past value ;; -bl|--backup-location) backup_location="$2" shift # past argument shift # past value ;; -bd|--backup-destination) backup_destination="$2" shift # past argument shift # past value ;; -scp|--secure-copy-protocol) scp="$2" shift # past argument shift # past value ;; -bb|--backup-bucket) backup_bucket="$2" shift # past argument shift # past value ;; -url| --endpoint-url) s3_url="$2" shift # past argument shift # past value ;; -s|--storage) storage="$2" shift # past argument shift # past value ;; -dbs|--dbroots) DBROOT_COUNT="$2" shift # past argument shift # past value ;; -pm | --nodeid) pm="$2" pm_number=$(echo "$pm" | tr -dc '0-9') shift # past argument shift # past value ;; -nb | --new-bucket) new_bucket="$2" shift # past argument shift # past value ;; -nr | --new-region) new_region="$2" shift # past argument shift # past value ;; -nk | --new-key) new_key="$2" shift # past argument shift # past value ;; -ns | --new-secret) new_secret="$2" shift # past argument shift # past value ;; -P|--parallel) PARALLEL_THREADS="$2" shift # past argument shift # past value ;; -ha | --highavilability) HA=true shift # past argument ;; -cont| --continue) continue=true shift # past argument ;; -f|--config-file) config_file="$2" shift # past argument shift # past value ;; -smdb | --skip-mariadb-backup) skip_mdb=true shift # past argument ;; -sb | --skip-bucket-data) skip_bucket_data=true shift # past argument ;; -m | --mode) mode="$2" shift # past argument shift # past value ;; -c | --compress) compress_format="$2" shift # past argument shift # past value ;; -q | --quiet) quiet=true shift # past argument ;; -nv-ssl| --no-verify-ssl) no_verify_ssl=true shift # past argument ;; -h|--help|-help|help) print_restore_help_text; exit 1; ;; *) # unknown option handle_early_exit_on_restore "\nunknown flag: $1\n" esac done # Load config file if [ -f $config_file ]; then source $config_file fi } print_restore_help_text() { echo " Columnstore Restore -l | --load What backup to load -bl | --backup-location Directory where the backup was saved -bd | --backup-destination If the directory is 'Local' or 'Remote' to this script -dbs | --dbroots Number of database roots in the backup -scp | --secure-copy-protocol scp connection to remote server if -bd 'Remote' -bb | --backup_bucket Bucket name for where to find the S3 backups -url | --endpoint-url Onprem url to s3 storage api example: http://127.0.0.1:8000 -nv-ssl| --no-verify-ssl) Skips verifying ssl certs, useful for onpremise s3 storage -s | --storage The storage used by columnstore data 'LocalStorage' or 'S3' -pm | --nodeid Forces the handling of the restore as this node as opposed to whats detected on disk -nb | --new-bucket Defines the new bucket to copy the s3 data to from the backup bucket. Use -nb if the new restored cluster should use a different bucket than the backup bucket itself -nr | --new-region Defines the region of the new bucket to copy the s3 data to from the backup bucket -nk | --new-key Defines the aws key to connect to the new_bucket -ns | --new-secret Defines the aws secret of the aws key to connect to the new_bucket -f | --config-file Path to backup configuration file to load variables from -cont| --continue This acknowledges data in your --new_bucket is ok to delete when restoring S3 -smdb| --skip-mariadb-backup Skip restoring mariadb server via mariadb-backup - ideal for only restoring columnstore -sb | --skip-bucket-data Skip restoring columnstore data in the bucket - ideal if looking to only restore mariadb server -q | --quiet Silence verbose copy command outputs -c | --compress Hint that the backup is compressed in X format - Options: [ pigz ] -ha | --highavilability Hint for if shared storage is attached @ below on all nodes to see all data HA LocalStorage ( /var/lib/columnstore/dataX/ ) HA S3 ( /var/lib/columnstore/storagemanager/ ) Local Storage Examples: ./$0 restore -s LocalStorage -bl /tmp/backups/ -bd Local -l 12-29-2021 ./$0 restore -s LocalStorage -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -l 12-29-2021 S3 Storage Examples: ./$0 restore -s S3 -bb s3://my-cs-backups -l 12-29-2021 ./$0 restore -s S3 -bb gs://on-premise-bucket -l 12-29-2021 -url http://127.0.0.1:8000 ./$0 restore -s S3 -bb s3://my-cs-backups -l 08-16-2022 -nb s3://new-data-bucket -nr us-east-1 -nk AKIAxxxxxxx3FHCADF -ns GGGuxxxxxxxxxxnqa72csk5 -ha "; # Hidden flags # -m | --mode Options ['direct','indirect'] - direct backups run on the columnstore nodes themselves. indirect run on another machine that has read-only mounts associated with columnstore/mariadb } print_restore_variables() { printf "\nRestore Variables\n" if [ -f $config_file ]; then printf "%-${s1}s %-${s2}s\n" "Configuration File:" "$config_file"; source $config_file fi if [ $mode == "indirect" ] || $skip_mdb || $skip_bucket_data; then echo "------------------------------------------------" echo "Skips: MariaDB($skip_mdb) Bucket($skip_bucket_data)"; fi; echo "------------------------------------------------" if [[ -n "$compress_format" ]]; then echo "Compression: true" echo "Compression Format: $compress_format"; else echo "Compression: false" fi if [ $storage == "LocalStorage" ]; then echo "Backup Location: $backup_location" echo "Backup Destination: $backup_destination" echo "Scp: $scp" echo "Storage: $storage" echo "Load Date: $load_date" echo "timestamp: $(date +%m-%d-%Y-%H%M%S)" echo "DB Root Count: $DBROOT_COUNT" echo "PM: $pm" echo "PM Number: $pm_number" elif [ $storage == "S3" ]; then echo "Backup Location: $backup_location" echo "Storage: $storage" echo "Load Date: $load_date" echo "timestamp: $(date +%m-%d-%Y-%H%M%S)" echo "PM: $pm" echo "PM Number: $pm_number" echo "Active bucket: $( grep -m 1 "^bucket =" $STORAGEMANGER_CNF)" echo "Active endpoint: $( grep -m 1 "^endpoint =" $STORAGEMANGER_CNF)" echo "Backup Bucket: $backup_bucket" echo "------------------------------------------------" echo "New Bucket: $new_bucket" echo "New Region: $new_region" echo "New Key: $new_key" echo "New Secret: $new_secret" echo "High Availabilty: $HA" fi echo "------------------------------------------------" } validation_prechecks_for_restore() { echo "Prechecks ..." if [ $storage != "LocalStorage" ] && [ $storage != "S3" ]; then handle_early_exit_on_restore "Invalid script variable storage: $storage\n"; fi if [ -z "$load_date" ]; then handle_early_exit_on_restore "\n[!!!] Required field --load: $load_date - is empty\n" ; fi if [ -z "$mode" ]; then handle_early_exit_on_restore "\n[!!!] Required field --mode: $mode - is empty\n" ; fi if [ "$mode" != "direct" ] && [ "$mode" != "indirect" ] ; then handle_early_exit_on_restore "\n[!!!] Invalid field --mode: $mode\n"; fi mariadb_user=$(stat -c "%U" /var/lib/mysql/) columnstore_user=$(stat -c "%U" /var/lib/columnstore/) # Adjust s3api flags for onpremise/custom endpoints add_s3_api_flags="" if [ -n "$s3_url" ]; then add_s3_api_flags+=" --endpoint-url $s3_url"; fi if $no_verify_ssl; then add_s3_api_flags+=" --no-verify-ssl"; fi; # If remote backup - Validate that scp works if [ $backup_destination == "Remote" ]; then if ssh $scp echo ok 2>&1 ;then printf 'SSH Works\n\n' else handle_early_exit_on_restore "Failed Command: ssh $scp" fi fi # Make sure the database is offline if [ "$mode" == "direct" ]; then if [ -z $(pidof PrimProc) ]; then printf " - Columnstore Status ... Offline\n"; else handle_early_exit_on_restore "\n[X] Columnstore is ONLINE - please turn off \n\n"; fi if [ -z $(pidof mariadbd) ]; then printf " - MariaDB Server Status ... Offline\n"; else handle_early_exit_on_restore "\n[X] MariaDB is ONLINE - please turn off \n\n"; fi fi # If cmapi installed - make cmapi turned off check_package_managers cmapi_installed_command="" case $package_manager in yum ) cmapi_installed_command="yum list installed MariaDB-columnstore-cmapi &> /dev/null;"; ;; apt ) cmapi_installed_command="dpkg-query -s mariadb-columnstore-cmapi &> /dev/null;"; ;; *) # unknown option echo "\npackage manager not implemented: $package_manager\n" exit 2; esac if eval $cmapi_installed_command ; then if [ -z $(pidof /usr/share/columnstore/cmapi/python/bin/python3) ]; then printf " - Columnstore Management API Status .. Offline\n"; else handle_early_exit_on_restore "\n[X] Cmapi is ONLINE - please turn off \n\n"; fi else printf " - Columnstore Management API Status .. Not Installed - Skipping\n"; fi # Validate addtional relevant arguments per storage option if [ $storage == "LocalStorage" ]; then if [ -z "$backup_location" ]; then handle_early_exit_on_restore "Invalid --backup_location: $backup_location - is empty"; fi if [ -z "$backup_destination" ]; then handle_early_exit_on_restore "Invalid --backup_destination: $backup_destination - is empty"; fi if [ $backup_destination == "Remote" ] && [ -d $backup_location$load_date ]; then echo "Switching to '-bd Local'"; backup_destination="Local"; fi if [ $backup_destination == "Local" ]; then if [ ! -d $backup_location ]; then handle_early_exit_on_restore "Invalid directory --backup_location: $backup_location - doesnt exist"; fi if [ -z "$(ls -A "$backup_location")" ]; then echo "Invalid --backup_location: $backup_location - directory is empty."; fi; if [ ! -d $backup_location$load_date ]; then handle_early_exit_on_restore "Invalid directory --load: $backup_location$load_date - doesnt exist"; else printf " - Backup directory exists\n"; fi fi; local files=$(ls $backup_location$load_date) if [[ -n "$compress_format" ]]; then case "$compress_format" in pigz) flag_cs_local=false flag_mysql=$skip_mdb while read -r line; do folder_name=$(echo "$line" | awk '{ print $NF }') case "$folder_name" in *mariadb-backup.tar.pigz*) flag_mysql=true ;; *cs-localfiles.tar.pigz*) flag_cs_local=true ;; esac done <<< "$files" if $flag_mysql && $flag_cs_local ; then echo " - Backup subdirectories exist" else echo "[X] Backup is missing subdirectories: " if ! $flag_cs_local; then echo "cs-localfiles.tar.pigz"; fi; if ! $flag_mysql; then echo "mariadb-backup.tar.pigz"; fi; handle_early_exit_on_restore "[!] Backup is missing subdirectories... \nls $backup_location$load_date\n\n" fi ;; *) # unknown option handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" esac else flag_configs=false flag_data=false flag_mysql=$skip_mdb files=$(ls $backup_location$load_date ) while read -r line; do folder_name=$(echo "$line" | awk '{ print $NF }') case "$folder_name" in *configs*) flag_configs=true ;; *data*) flag_data=true ;; *mysql*) flag_mysql=true ;; esac done <<< "$files" if $flag_configs && $flag_data && $flag_mysql; then echo " - Backup subdirectories exist" else echo "[X] Backup is missing subdirectories: " if ! $flag_configs; then echo "configs"; fi; if ! $flag_data; then echo "data"; fi; if ! $flag_mysql; then echo "mysql"; fi; handle_early_exit_on_restore " Backup is missing subdirectories... \nls $backup_location$load_date\n" fi fi elif [ $storage == "S3" ]; then # Check for concurrency settings - https://docs.aws.amazon.com/cli/latest/topic/s3-config.html if [ $cloud == "aws" ]; then cli_concurrency=$(sudo grep max_concurrent_requests ~/.aws/config 2>/dev/null ); if [ -z "$cli_concurrency" ]; then echo "[!!!] check: '~/.aws/config' - We recommend increasing s3 concurrency for better throughput to/from S3. This value should scale with avaialble CPU and networking capacity"; printf "example: aws configure set default.s3.max_concurrent_requests 200\n"; fi; fi; # Validate addtional relevant arguments if [ -z "$backup_bucket" ]; then handle_early_exit_on_restore "Invalid --backup_bucket: $backup_bucket - is empty"; fi # Prepare commands for each cloud if [ $cloud == "gcp" ]; then check1="$gsutil ls $backup_bucket"; check2="$gsutil ls -b $new_bucket"; check3="$gsutil ls $new_bucket"; check4="$gsutil ls $backup_bucket/$load_date"; check5="$gsutil ls $backup_bucket/$load_date/"; else check1="$awscli $add_s3_api_flags s3 ls $backup_bucket" check2="$awscli $add_s3_api_flags s3 ls | grep $new_bucket" check3="$awscli $add_s3_api_flags s3 ls $new_bucket" check4="$awscli $add_s3_api_flags s3 ls $backup_bucket/$load_date" check5="$awscli $add_s3_api_flags s3 ls $backup_bucket/$load_date/" fi; # Check aws cli access to bucket if $check1 > /dev/null ; then echo -e " - Success listing backup bucket" else handle_early_exit_on_restore "[X] Failed to list bucket contents...\n$check1\n" fi # New bucket exists and empty check if [ "$new_bucket" ] && [ $pm == "pm1" ]; then # Removing as storage.buckets.get permission likely not needed # new_bucket_exists=$($check2); if [ -z "$new_bucket_exists" ]; then handle_early_exit_on_restore "[!!!] Didnt find new bucket - Check: $check2\n"; fi; echo "[+] New Bucket exists"; # Throw warning if new bucket is NOT empty if ! $continue; then nb_contents=$($check3 | wc -l); if [ $nb_contents -lt 1 ]; then echo " - New Bucket is empty"; else echo "[!!!] New bucket is NOT empty... $nb_contents files exist... exiting"; echo "add "--continue" to skip this exit"; echo -e "\nExample empty bucket command:\n aws s3 rm $new_bucket --recursive\n gsutil -m rm -r $new_bucket/* \n"; handle_early_exit_on_restore "Please empty bucket or add --continue \n"; fi; else echo " - [!] Skipping empty new_bucket check"; fi; fi # Check if s3 bucket load date exists if $check4 > /dev/null ; then echo -e " - Backup directory exists" else handle_early_exit_on_restore "\n[X] Backup directory load date ($backup_bucket/$load_date) DOES NOT exist in S3 \nCheck - $check4 \n\n"; fi # Check if s3 bucket load date subdirectories exist if folders=$($check5) > /dev/null ; then if [[ -n "$compress_format" ]]; then # when compressed, storagemanager/configs wont exist but rather pigz_chunkXXX does. and mysql could be split up case "$compress_format" in pigz) flag_cs_local=false flag_mysql=$skip_mdb flag_columnstoreData=$skip_bucket_data while read -r line; do folder_name=$(echo "$line" | awk '{ print $NF }') case "$folder_name" in *columnstoreData*) flag_columnstoreData=true ;; *mariadb-backup.tar.pigz*) flag_mysql=true ;; *cs-localfiles.tar.pigz*) flag_cs_local=true ;; esac done <<< "$folders" if $flag_pigz_chunk && $flag_mysql && $flag_columnstoreData ; then echo " - Backup subdirectories exist" else echo "[X] Backup is missing subdirectories: " if ! $flag_cs_local; then echo "cs-localfiles.tar.pigzXXX"; fi; if ! $flag_mysql; then echo "mariadb-backup.tar.pigzXXXX"; fi; if ! $flag_columnstoreData; then echo "columnstoreData"; fi; handle_early_exit_on_restore " Backup is missing subdirectories... \n$check5\n" fi ;; *) # unknown option handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac else flag_configs=false flag_storagemanager=false flag_mysql=$skip_mdb flag_columnstoreData=$skip_bucket_data while read -r line; do folder_name=$(echo "$line" | awk '{ print $NF }') #echo "DEBUG: $folder_name" case "$folder_name" in *storagemanager*) flag_storagemanager=true ;; *mysql*) flag_mysql=true ;; *columnstoreData*) flag_columnstoreData=true ;; *configs*) flag_configs=true ;; #*) echo "unknown: $folder_name" ;; esac done <<< "$folders" if $flag_storagemanager && $flag_mysql && $flag_columnstoreData && $flag_configs; then echo " - Backup subdirectories exist" else echo "[X] Backup is missing subdirectories: " if ! $flag_storagemanager; then echo "storagemanager"; fi; if ! $flag_mysql; then echo "mysql"; fi; if ! $flag_columnstoreData; then echo "columnstoreData"; fi; if ! $flag_configs; then echo "configs"; fi; handle_early_exit_on_restore " Backup is missing subdirectories... \n$check5\n" fi fi; else handle_early_exit_on_restore " Failed to list backup contents...\n$check5\n" fi; fi; if $quiet; then xtra_cmd_args+=" 2> /dev/null"; rsync_flags=" -a"; fi } # Restore Columnstore.xml by updating critical parameters # Depends on $CS_CONFIGS_PATH/$col_xml_backup restore_columnstore_values() { printf "\nRestore Columnstore.xml Values" if ! $quiet; then printf "\n"; else printf "... "; fi; columnstore_config_pairs_to_transfer=( "HashJoin TotalUmMemory" "DBBC NumBlocksPct" "CrossEngineSupport User" "CrossEngineSupport Password" "HashJoin AllowDiskBasedJoin" "RowAggregation AllowDiskBasedAggregation" "SystemConfig MemoryCheckPercent" "JobList RequestSize" "JobList ProcessorThreadsPerScan" "DBBC NumThreads" "PrimitiveServers ConnectionsPerPrimProc" ) # mcsGetConfig -c Columnstore.xml.source_backup HashJoin TotalUmMemory for pair in "${columnstore_config_pairs_to_transfer[@]}"; do level_one=$(echo "$pair" | cut -d ' ' -f 1) level_two=$(echo "$pair" | cut -d ' ' -f 2) # Get the source value using mcsGetConfig -c source_value=$(mcsGetConfig -c "$CS_CONFIGS_PATH/$col_xml_backup" "$level_one" "$level_two") if [ -n "$source_value" ]; then # Set the value in the active Columnstore.xml using mcsSetConfig if mcsSetConfig "$level_one" "$level_two" "$source_value"; then # echo instead of printf to avoid escaping % from HashJoin TotalUmMemory if ! $quiet; then echo " - Set $level_one $level_two $source_value"; fi; else printf "\n[!] Failed to Set $level_one $level_two $source_value \n"; fi else if ! $quiet; then printf " - N/A: $level_one $level_two \n"; fi; fi done if $quiet; then printf " Done\n"; fi; } run_restore() { # Branch logic based on topology if [ $storage == "LocalStorage" ]; then # Branch logic based on where the backup resides if [ $backup_destination == "Local" ]; then # MariaDB Columnstore Restore printf "\nRestore MariaDB Columnstore LocalStorage\n" # Handle Compressed Backup Restore if [[ -n "$compress_format" ]]; then # Clear appropriate dbroot i=1; while [ $i -le $DBROOT_COUNT ]; do if [[ $pm == "pm$i" || $HA == true ]]; then printf " - Deleting Columnstore Data$i ... "; rm -rf /var/lib/columnstore/data$i/*; printf " Done \n"; fi ((i++)) done tar_flags="xf" if ! $quiet; then tar_flags+="v"; fi; cs_prefix="$backup_location$load_date/$split_file_cs_prefix.$compress_format" case $compress_format in pigz) printf " - Decompressing CS Files -> local ... " cd / if ! eval "pigz -dc -p $PARALLEL_THREADS $cs_prefix | tar $tar_flags - "; then handle_early_exit_on_restore "Failed to decompress and untar columnstore localfiles\ncommand:\npigz -dc -p $PARALLEL_THREADS $cs_prefix | tar xf - \n\n" fi printf " Done \n" ;; *) # unknown option handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac # Set permissions after restoring files i=1; while [ $i -le $DBROOT_COUNT ]; do if [[ $pm == "pm$i" || $HA == true ]]; then printf " - Chowning Columnstore Data$i ... "; chown $columnstore_user:$columnstore_user -R /var/lib/columnstore/data$i/; printf " Done \n"; fi ((i++)) done else # Handle Non-Compressed Backup Restore # Loop through per dbroot i=1; while [ $i -le $DBROOT_COUNT ]; do if [[ $pm == "pm$i" || $HA == true ]]; then printf " - Restoring Columnstore Data$i ... "; rm -rf /var/lib/columnstore/data$i/*; rsync $rsync_flags $backup_location$load_date/data$i/ /var/lib/columnstore/data$i/; chown $columnstore_user:$columnstore_user -R /var/lib/columnstore/data$i/; printf " Done Data$i \n"; fi ((i++)) done # Put configs in place printf " - Columnstore Configs ... " rsync $rsync_flags $backup_location$load_date/configs/storagemanager.cnf $STORAGEMANGER_CNF rsync $rsync_flags $backup_location$load_date/configs/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup rsync $rsync_flags $backup_location$load_date/configs/mysql/$pm/ $MARIADB_SERVER_CONFIGS_PATH/ if [ -f "$backup_location$load_date/configs/cmapi_server.conf" ]; then rsync $rsync_flags $backup_location$load_date/configs/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup ; fi; printf " Done\n" fi elif [ $backup_destination == "Remote" ]; then printf "[~] Copy MySQL Data..." tmp="localscpcopy-$load_date" rm -rf $backup_location$tmp/mysql/ mkdir -p $backup_location$tmp/mysql/ rsync $rsync_flags $scp:$backup_location/$load_date/mysql/ $backup_location$tmp/mysql/ printf "[+] Copy MySQL Data... Done\n" # Loop through per dbroot printf "[~] Columnstore Data..."; i=1; while [ $i -le $DBROOT_COUNT ]; do if [[ $pm == "pm$i" || $HA == true ]]; then rm -rf /var/lib/columnstore/data$i/; rsync -av $scp:$backup_location$load_date/data$i/ /var/lib/columnstore/data$i/ ; chown $columnstore_user:$columnstore_user -R /var/lib/columnstore/data$i/; fi ((i++)) done printf "[+] Columnstore Data... Done\n" # Put configs in place printf " - Columnstore Configs ... " rsync $rsync_flags $backup_location$load_date/configs/storagemanager.cnf $STORAGEMANGER_CNF rsync $rsync_flags $backup_location$load_date/configs/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup rsync $rsync_flags $backup_location$load_date/configs/mysql/$pm/ $MARIADB_SERVER_CONFIGS_PATH/ if [ -f "$backup_location$load_date/configs/cmapi_server.conf" ]; then rsync $rsync_flags $backup_location$load_date/configs/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup ; fi; printf " Done\n" load_date=$tmp else handle_early_exit_on_restore "Invalid Script Variable --backup_destination: $backup_destination" fi # MariaDB Server Restore printf "\nRestore MariaDB Server\n" if [[ -n "$compress_format" ]]; then # Handle compressed mariadb-backup restore mbd_prefix="$backup_location$load_date/$split_file_mdb_prefix.$compress_format" case $compress_format in pigz) printf " - Decompressing MariaDB Files -> $MARIADB_PATH ... " rm -rf $MARIADB_PATH/* cd / if ! eval "pigz -dc -p $PARALLEL_THREADS $mbd_prefix | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH"; then handle_early_exit_on_restore "Failed to decompress mariadb backup\ncommand:\npigz -dc -p $PARALLEL_THREADS $mbd_prefix | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH \n" fi; printf " Done \n" printf " - Running mariabackup --prepare ... " if eval "mariabackup --prepare --target-dir=$MARIADB_PATH $xtra_cmd_args"; then printf " Done\n"; else handle_early_exit_on_restore "Failed to --prepare\nmariabackup --prepare --target-dir=$MARIADB_PATH"; fi; ;; *) # unknown option handle_early_exit_on_backup "\nUnknown compression flag: $compress_format\n" esac else eval "mariabackup --prepare --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args" rm -rf /var/lib/mysql/* eval "mariabackup --copy-back --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args" fi printf " - Chowning MariaDB Files ... "; chown -R $mariadb_user:$mariadb_user /var/lib/mysql/ chown -R $columnstore_user:$mariadb_user /var/log/mariadb/ printf " Done \n" elif [ $storage == "S3" ]; then printf "\nRestore MariaDB Columnstore S3\n" # Sync the columnstore data from the backup bucket to the new bucket if ! $skip_bucket_data && [ "$new_bucket" ] && [ $pm == "pm1" ]; then printf "[+] Starting Bucket Sync:\n - $backup_bucket/$load_date/columnstoreData to $new_bucket\n" s3sync $backup_bucket/$load_date/columnstoreData/ $new_bucket/ " - Done with S3 Bucket sync\n" else printf "[!] Skipping Columnstore Bucket\n" fi; printf "[+] Starting Columnstore Configurations & Metadata: \n" if [[ -n "$compress_format" ]]; then cs_prefix="$backup_bucket/$load_date/$split_file_cs_prefix.$compress_format" case $compress_format in pigz) printf " - Decompressing CS Files bucket -> local ... " if [ $cloud == "gcp" ]; then # gsutil supports simple "cat prefix*" to standard out unlike awscli cd / if ! eval "$gsutil cat $cs_prefix* | gzip -dc | tar xf - "; then handle_early_exit_on_restore "Failed to decompress and untar columnstore localfiles\ncommand:\n$gsutil cat $cs_prefix* | gzip -dc | tar xf - \n\n" fi elif [ $cloud == "aws" ]; then # List all the pigz compressed chunks in the S3 prefix - should I clear prior local files? chunk_list=$($awscli s3 ls "$cs_prefix" | awk '{print $NF}') cd / # Use process substitution to concatenate the compressed chunks and pipe to pigz and extract using tar if ! eval "pigz -dc -p $PARALLEL_THREADS <(for chunk in \$chunk_list; do $awscli s3 cp \"$backup_bucket/$load_date/\$chunk\" - ; done) | tar xf - "; then handle_early_exit_on_restore "Failed to decompress and untar columnstore localfiles\ncommand:\npigz -dc -p $PARALLEL_THREADS <(for chunk in \$chunk_list; do $awscli s3 cp \"$backup_bucket/$load_date/\$chunk\" - ; done) | tar xf - \n\n" fi fi printf " Done \n" ;; *) # unknown option handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac else # Columnstore.xml and cmapi are renamed so that only specifc values are restored s3cp $backup_bucket/$load_date/configs/Columnstore.xml $CS_CONFIGS_PATH/$col_xml_backup s3cp $backup_bucket/$load_date/configs/cmapi_server.conf $CS_CONFIGS_PATH/$cmapi_backup s3cp $backup_bucket/$load_date/configs/storagemanager.cnf $STORAGEMANGER_CNF s3sync $backup_bucket/$load_date/storagemanager/cache/data$pm_number $cs_cache/data$pm_number/ " - Done cache/data$pm_number/\n" s3sync $backup_bucket/$load_date/storagemanager/metadata/data$pm_number $cs_metadata/data$pm_number/ " - Done metadata/data$pm_number/\n" if ! $skip_mdb; then s3sync $backup_bucket/$load_date/configs/mysql/$pm/ $MARIADB_SERVER_CONFIGS_PATH/ " - Done $MARIADB_SERVER_CONFIGS_PATH/\n"; fi if s3ls "$backup_bucket/$today/storagemanager/journal/data$ASSIGNED_DBROOT" > /dev/null 2>&1; then s3sync $backup_bucket/$load_date/storagemanager/journal/data$pm_number $cs_journal/data$pm_number/ " - Done journal/data$pm_number/\n" else echo " - Done journal/data$pm_number was empty" fi fi; # Set appropraite bucket, prefix, region, key , secret printf "[+] Adjusting storagemanager.cnf ... \n" target_bucket=$backup_bucket target_prefix="prefix = $load_date\/columnstoreData\/" if [ -n "$new_bucket" ]; then target_bucket=$new_bucket; target_prefix="# prefix \= cs\/"; fi if [ -n "$new_region" ]; then if [ $cloud == "gcp" ]; then endpoint="storage.googleapis.com"; else endpoint="s3.$new_region.amazonaws.com"; fi; sed -i "s|^endpoint =.*|endpoint = ${endpoint}|g" $STORAGEMANGER_CNF; sed -i "s|^region =.*|region = ${new_region}|g" $STORAGEMANGER_CNF; echo " - Adjusted endpoint & region"; fi # Removes prefix of the protocol and escapes / for sed target_bucket_name=$(echo "${target_bucket#$protocol://}" | sed "s/\//\\\\\//g") if [ -n "$new_key" ]; then sed -i "s|aws_access_key_id =.*|aws_access_key_id = ${new_key}|g" $STORAGEMANGER_CNF; echo " - Adjusted aws_access_key_id"; fi if [ -n "$new_secret" ]; then sed -i "s|aws_secret_access_key =.*|aws_secret_access_key = ${new_secret}|g" $STORAGEMANGER_CNF; echo " - Adjusted aws_secret_access_key"; fi bucket=$( grep -m 1 "^bucket =" $STORAGEMANGER_CNF | sed "s/\//\\\\\//g"); sed -i "s/$bucket/bucket = $target_bucket_name/g" $STORAGEMANGER_CNF; echo " - Adjusted bucket"; prefix=$( grep -m 1 "^prefix =" $STORAGEMANGER_CNF | sed "s/\//\\\\\//g"); if [ ! -z "$prefix" ]; then sed -i "s/$prefix/$target_prefix/g" $STORAGEMANGER_CNF; echo " - Adjusted prefix"; fi; # Check permissions chown -R $columnstore_user:$columnstore_user /var/lib/columnstore/ chown -R root:root $MARIADB_SERVER_CONFIGS_PATH/ # Confirm S3 connection works if [ $mode == "direct" ]; then echo "[+] Checking S3 Connection ..."; if testS3Connection $xtra_cmd_args; then echo " - S3 Connection passes" ; else handle_early_exit_on_restore "\n[X] S3 Connection issues - retest/configure\n"; fi; fi; printf "[+] MariaDB Columnstore Done\n\n" if ! $skip_mdb; then printf "Restore MariaDB Server\n" if [[ -n "$compress_format" ]]; then # Handle compressed mariadb-backup restore mbd_prefix="$backup_bucket/$load_date/$split_file_mdb_prefix.$compress_format" case $compress_format in pigz) printf "[+] Decompressing mariadb-backup bucket -> local ... " rm -rf $MARIADB_PATH/* if [ $cloud == "gcp" ]; then if ! eval "$gsutil cat $mbd_prefix* | pigz -dc -p $PARALLEL_THREADS | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH"; then handle_early_exit_on_restore "Failed to decompress mariadb backup\ncommand:\n$gsutil cat $mbd_prefix* | pigz -dc -p $PARALLEL_THREADS | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH\n" fi elif [ $cloud == "aws" ]; then chunk_list=$($awscli s3 ls "$mbd_prefix" | awk '{print $NF}') cd / if ! eval "pigz -dc -p $PARALLEL_THREADS <(for chunk in \$chunk_list; do $awscli s3 cp "$backup_bucket/$load_date/\$chunk" -; done) | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH"; then handle_early_exit_on_restore "Failed to decompress mariadb backup\ncommand:\npigz -dc -p $PARALLEL_THREADS <(for chunk in \$chunk_list; do $awscli s3 cp "$backup_bucket/$load_date/\$chunk" -; done) | mbstream -p $PARALLEL_THREADS -x -C $MARIADB_PATH \n" fi; fi; printf " Done \n" printf " - Running mariabackup --prepare ... " if eval "mariabackup --prepare --target-dir=$MARIADB_PATH $xtra_cmd_args"; then printf " Done\n"; else handle_early_exit_on_restore "Failed to --prepare\nmariabackup --prepare --target-dir=$MARIADB_PATH"; fi; ;; *) # unknown option handle_early_exit_on_backup "\nunknown compression flag: $compress_format\n" esac else # Copy the MariaDB data, prepare and put in place printf " - Copying down MariaDB data ... " rm -rf $backup_location$load_date/mysql/ mkdir -p $backup_location$load_date/mysql/ s3sync $backup_bucket/$load_date/mysql/ $backup_location$load_date/mysql " Done\n" # Run prepare and copy back printf " - Running mariabackup --prepare ... " if eval "mariabackup --prepare --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args"; then printf " Done\n"; else echo "failed"; fi; rm -rf /var/lib/mysql/* > /dev/null printf " - Running mariabackup --copy-back ... " if eval "mariabackup --copy-back --target-dir=$backup_location$load_date/mysql/ $xtra_cmd_args"; then printf " Done\n"; else echo "failed"; fi; fi; # Check permissions printf " - Chowning mysql and log dirs ... " chown -R $mariadb_user:$mariadb_user /var/lib/mysql/ chown -R $columnstore_user:$mariadb_user /var/log/mariadb/ printf " Done \n" printf "[+] MariaDB Server Done\n" else echo "[!] Skipping mariadb server restore" fi fi if [[ $pm == "pm1" ]]; then restore_columnstore_values; fi; end=$(date +%s) runtime=$((end-start)) printf "\n[+] Runtime: $runtime\n" printf "[+] $final_message\n\n" if ! $quiet; then if [ $pm == "pm1" ]; then echo -e " - Last you need to manually configure mariadb replication between nodes" echo -e " - systemctl start mariadb " echo -e " - systemctl start mariadb-columnstore-cmapi " echo -e " - mcs cluster start \n\n" else printf " - Last you need to manually configure mariadb replication between nodes\n" echo -e "Example:" echo -e "mariadb -e \"stop slave;CHANGE MASTER TO MASTER_HOST='\$pm1' , MASTER_USER='repuser' , MASTER_PASSWORD='aBcd123%123%aBc' , MASTER_USE_GTID = slave_pos;start slave;show slave status\G\"" echo -e "mariadb -e \"show slave status\G\" | grep \"Slave_\" \n" fi fi } load_default_dbrm_variables() { # Default variables backup_base_name="dbrm_backup" backup_interval_minutes=90 retention_days=7 backup_location=/tmp/dbrm_backups STORAGEMANGER_CNF="/etc/columnstore/storagemanager.cnf" storage=$(grep -m 1 "^service = " $STORAGEMANGER_CNF | awk '{print $3}') mode="once" quiet=false dbrm_dir="/var/lib/columnstore/data1/systemFiles/dbrm" if [ "$storage" == "S3" ]; then dbrm_dir="/var/lib/columnstore/storagemanager" fi } # for next release load_default_dbrm_restore_variables() { backup_location=/tmp/dbrm_backups STORAGEMANGER_CNF="/etc/columnstore/storagemanager.cnf" storage=$(grep -m 1 "^service = " $STORAGEMANGER_CNF | awk '{print $3}') backup_folder_to_restore="" dbrm_dir="/var/lib/columnstore/data1/systemFiles/dbrm" if [ "$storage" == "S3" ]; then dbrm_dir="/var/lib/columnstore/storagemanager" fi } print_dbrm_backup_help_text() { echo " Columnstore DBRM Backup -m | --mode 'loop' or 'once' ; Determines if this script runs in a forever loop sleeping -i minutes or just once -i | --interval Number of minutes to sleep when --mode loop -r | --retention-days Number of days of dbrm backups to retain - script will delete based on last update file time -p | --path path of where to save the dbrm backups on disk -nb | --name-backup custom name to prefex dbrm backups with Default: ./$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups Examples: ./$0 dbrm_backup --mode loop --interval 90 --retention-days 7 --path /mnt/dbrm_backups ./$0 dbrm_backup --mode once --retention-days 7 --path /mnt/dbrm_backups -nb my-one-off-backup Cron Example: */60 */3 * * * root bash /root/$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups >> /tmp/dbrm_backups/cs_backup.log 2>&1 "; } # for next release print_dbrm_restore_help_text() { echo " Columnstore DBRM Restore -p | --path path of where to save the dbrm backups on disk -d | --directory date or directory chose to restore from Default: ./$0 dbrm_restore --path /tmp/dbrm_backups Examples: ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup12252023 "; } parse_dbrms_variables() { # Dynamic Arguments while [[ $# -gt 0 ]]; do key="$1" case $key in dbrm_backup) shift # past argument ;; -i|--interval) backup_interval_minutes="$2" shift # past argument shift # past value ;; -r|--retention-days) retention_days="$2" shift # past argument shift # past value ;; -p|--path) backup_location="$2" shift # past argument shift # past value ;; -m|--mode) mode="$2" shift # past argument shift # past value ;; -nb|--name-backup) backup_base_name="$2" shift # past argument shift # past value ;; -q | --quiet) quiet=true shift # past argument ;; -h|--help|-help|help) print_dbrm_backup_help_text; exit 1; ;; *) # unknown option printf "\nunknown flag: $1\n" print_dbrm_backup_help_text exit 1; esac done } # for next release parse_dbrm_restore_variables() { # Dynamic Arguments while [[ $# -gt 0 ]]; do key="$1" case $key in dbrm_restore) shift # past argument ;; -p|--path) backup_location="$2" shift # past argument shift # past value ;; -d|--directory) backup_folder_to_restore="$2" shift # past argument shift # past value ;; -h|--help|-help|help) print_dbrm_restore_help_text; exit 1; ;; *) # unknown option printf "\nunknown flag: $1\n" print_dbrm_restore_help_text exit 1; esac done } confirm_numerical_or_decimal_else_fail() { local input="$1" local variable_name="$2" # Regular expression to match numerical or decimal values if [[ $input =~ ^[0-9]+(\.[0-9]+)?$ ]]; then return 0 else printf "[!] $variable_name = '$input' is not numerical or decimal.\n\n" alert "[!] $variable_name = '$input' is not numerical or decimal.\n\n" exit 2; fi } validation_prechecks_for_dbrm_backup() { # Confirm storage not empty if [ -z "$storage" ]; then printf "[!] Empty storage: \ncheck: grep -m 1 \"^service = \" \$STORAGEMANGER_CNF | awk '{print \$3}' \n\n"; fi; # Check mode type errors="" case $mode in once) errors+="" ;; loop) errors+="" ;; *) # unknown option printf "\nunknown mode: $mode\n" printf "Only 'once' & 'loop' allowed\n\n" print_dbrm_backup_help_text exit 2; esac # Check numbers confirm_numerical_or_decimal_else_fail "$backup_interval_minutes" "Interval" confirm_numerical_or_decimal_else_fail "$retention_days" "Retention" # Check backup location exists if [ ! -d $backup_location ]; then echo "[+] Created: $backup_location" mkdir "$backup_location"; fi; # Confirm bucket connection if [ "$storage" == "S3" ]; then if ! testS3Connection 1>/dev/null 2>/dev/null; then printf "\n[!] Failed testS3Connection\n\n" exit 1; fi fi; } # for next release validation_prechecks_for_dbrm_restore() { # Confirm storage not empty if [ -z "$storage" ]; then printf "[!] Empty storage: \ncheck: grep -m 1 \"^service = \" \$STORAGEMANGER_CNF | awk '{print \$3}' \n\n"; fi; # Check backup location exists if [ ! -d $backup_location ]; then echo "[!] \$backup_location: Expected directory of dbrm_backups" exit 2; fi # Confirm bucket connection if [ "$storage" == "S3" ]; then if ! testS3Connection 1>/dev/null 2>/dev/null; then printf "\n[!] Failed testS3Connection\n\n" exit 1; fi fi; } process_dbrm_backup() { load_default_dbrm_variables parse_dbrms_variables "$@"; if ! $quiet ; then printf "\n[+] Inputs\n"; printf " CS Storage: $storage\n"; printf " Source: $dbrm_dir\n"; printf " Backups: $backup_location\n"; if [ "$mode" == "loop" ]; then printf " Interval: $backup_interval_minutes minutes\n"; fi; printf " Retention: $retention_days day(s)\n" printf " Mode: $mode\n\n" fi; validation_prechecks_for_dbrm_backup sleep_seconds=$((backup_interval_minutes * 60)); while true; do # Create a new backup directory timestamp=$(date +%Y%m%d_%H%M%S) backup_folder="$backup_location/${backup_base_name}_${timestamp}" mkdir -p "$backup_folder" # Copy files to the backup directory cp -arp "$dbrm_dir"/* "$backup_folder" if [ "$storage" == "S3" ]; then # smcat em files to disk mkdir $backup_folder/dbrms/ smls /data1/systemFiles/dbrm 2>/dev/null > $backup_folder/dbrms/dbrms.txt smcat /data1/systemFiles/dbrm/BRM_saves_current 2>/dev/null > $backup_folder/dbrms/BRM_saves_current smcat /data1/systemFiles/dbrm/BRM_saves_journal 2>/dev/null > $backup_folder/dbrms/BRM_saves_journal smcat /data1/systemFiles/dbrm/BRM_saves_em 2>/dev/null > $backup_folder/dbrms/BRM_saves_em smcat /data1/systemFiles/dbrm/BRM_saves_vbbm 2>/dev/null > $backup_folder/dbrms/BRM_saves_vbbm smcat /data1/systemFiles/dbrm/BRM_saves_vss 2>/dev/null > $backup_folder/dbrms/BRM_saves_vss smcat /data1/systemFiles/dbrm/BRM_savesA_em 2>/dev/null > $backup_folder/dbrms/BRM_savesA_em smcat /data1/systemFiles/dbrm/BRM_savesA_vbbm 2>/dev/null > $backup_folder/dbrms/BRM_savesA_vbbm smcat /data1/systemFiles/dbrm/BRM_savesA_vss 2>/dev/null > $backup_folder/dbrms/BRM_savesA_vss smcat /data1/systemFiles/dbrm/BRM_savesB_em 2>/dev/null > $backup_folder/dbrms/BRM_savesB_em smcat /data1/systemFiles/dbrm/BRM_savesB_vbbm 2>/dev/null > $backup_folder/dbrms/BRM_savesB_vbbm smcat /data1/systemFiles/dbrm/BRM_savesB_vss 2>/dev/null > $backup_folder/dbrms/BRM_savesB_vss fi # Clean up old backups # example: find /tmp/dbrmBackups/ -maxdepth 1 -type d -name "dbrm_backup_*" -mtime +1 -exec rm -r {} \; find "$backup_location" -maxdepth 1 -type d -name "${backup_base_name}_*" -mtime +$retention_days -exec rm -r {} \; printf "[+] Created: $backup_folder\n" if [ "$mode" == "once" ]; then break; fi; printf "[+] Sleeping ... $sleep_seconds seconds\n" sleep "$sleep_seconds" done if ! $quiet; then printf "[+] Complete\n\n"; fi; } # for next release process_dbrm_restore() { load_default_dbrm_restore_variables parse_dbrm_restore_variables "$@" # print current job variables printf "\n[+] Inputs\n"; printf " Backups Directory: $backup_location\n"; printf " Backup to Restore: $backup_folder_to_restore\n"; printf " CS Storage: $storage\n"; printf " Restore Target: $dbrm_dir\n"; # Display restore options if [ -z "$backup_folder_to_restore" ]; then printf "\n[!!] --directory is empty\n" printf "See Options Below: \n" printf "%-30s %-12s %-13s %-63s %-12s\n" "Option" "Size" "Last updated" "File Location" "Journal Size"; # Iterate over subdirectories for subdir in "${backup_location}"/*; do # Find the most recently modified file in the current subdirectory latest_em_file=$(find "${subdir}" -maxdepth 1 -type f -name "BRM_saves*_em" -exec ls -lt --time=ctime {} + | awk 'NR==1 {printf "%-12s %-4s %-2s %-5s %s\n", $5, $6, $7, $8, $9}'| head -n 1) journal_file=$(ls -la "${subdir}/BRM_saves_journal" | awk 'NR==1 {print $5}' ) printf "%-30s %-90s %-12s\n" "$(basename "$subdir")" "$latest_em_file" "$journal_file"; done printf "\n[!!] define which backup to restore eg. --directory dbrm_backup_20240103_183536\n" printf "exiting ...\n\n" exit 1; fi; validation_prechecks_for_dbrm_restore if [ ! -f "cs_package_manager.sh" ]; then wget https://raw.githubusercontent.com/mariadb-corporation/mariadb-columnstore-engine/develop/extra/cs_package_manager.sh; chmod +x cs_package_manager.sh; fi; source cs_package_manager.sh source # Confirm the function exists and the source of cs_package_manager.sh worked if command -v check_package_managers &> /dev/null; then # The function exists, call it check_package_managers else echo "Error: 'check_package_managers' function not found via cs_package_manager.sh"; exit 1; fi init_cs_down wait_cs_down 0 if ! process_dbrm_backup -nb dbrms_before_restore_backup --quiet ; then echo "[!!] Failed to take a DBRM backup before restoring" echo "exiting ..." exit 1; fi; # split logic between S3 & LocalStorage if [ $storage == "S3" ]; then process_s3_dbrm_restore else process_localstorage_dbrm_restore fi; } # for next release process_s3_dbrm_restore() { # shuffle DBRMs - detect newest date _em from the set, if smaller than the current one throw a warning # manually run load_brm # manually run save_brm # start columnstore # run a basic health check return 1; } # for next release process_localstorage_dbrm_restore() { # shuffle DBRMs - detect newest date _em from the set, if smaller than the current one throw a warning em_file=$(find "${backup_location}/${backup_folder_to_restore}" -maxdepth 1 -type f -name "BRM_saves*_em" -exec ls -lt --time=ctime {} + | head -n 1 ) #em_file=$(find /tmp/dbrm_backups/dbrm_backup_20240103_195653 -maxdepth 1 -type f -name "BRM_saves*_em" -exec ls -lt --time=ctime {} + | head -n 1) em_file_full_path=$(echo $em_file | awk '{print $9}' ) if [ ! -f $em_file_full_path ]; then echo "[!] Failed to parse _em file: $em_file_full_path doesnt exist" exit 1; fi; em_file_name=$(basename $em_file_full_path) prefix="${em_file_name%%_em}" echo "[+] Clearing Active DBRMs" rm -rf $dbrm_dir echo "[+] Copying backup DBRMs to active DBRMs dir" cp -arp "${backup_location}/${backup_folder_to_restore}" $dbrm_dir echo "[+] Restoring Prefix: $prefix" vbbm_name="${prefix}_vbbm" vss_name="${prefix}_vss" cp -arp "${dbrm_dir}/$em_file_name" "${dbrm_dir}/BRM_saves_em" cp -arp "${dbrm_dir}/$vbbm_name" "${dbrm_dir}/BRM_saves_vbbm" cp -arp "${dbrm_dir}/$vss_name" "${dbrm_dir}/BRM_saves_vss" echo "Primary Extent Map Files Now:" ls -la "${dbrm_dir}/BRM_saves_em" ls -la "${dbrm_dir}/BRM_saves_vbbm" ls -la "${dbrm_dir}/BRM_saves_vss" ls -la "${dbrm_dir}/BRM_saves_journal" sleep 2 clearShm # manually run load_brm printf "\n\n[+] Running load_brm...\n" if ! sudo -su mysql /usr/bin/load_brm /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves ; then echo "[!!] Failed to complete load_brm successfully" exit 1; fi; if ! systemctl start mcs-controllernode ; then echo "[!!] Failed to start mcs-controllernode " exit 1; fi; echo "[+] Confirming extent map readable..." if ! editem -i >/dev/null ; then echo "[!!] Failed to run editem -i (read the EM)" exit 1; fi; echo "[+] Running save_brm..." if ! sudo -u mysql /usr/bin/save_brm ; then echo "[!!] Failed to run save_brm" exit 1; fi if ! systemctl stop mcs-controllernode; then echo "[!!] Failed to stop mcs-controllernode" exit 1; fi clearShm printf "\n[+] Turning on columnstore ...\n" start_cs_cmapi_via_curl # run a basic health check backup_folder_to_restore="derp" mariadb -e "create database if not exists $backup_folder_to_restore" # confirm $backup_folder_to_restore doesnt exist mariadb $backup_folder_to_restore -e "create table t1 (a int) engine=columnstore" mariadb $backup_folder_to_restore -e "insert into t1 values (1)" mariadb $backup_folder_to_restore -e "update t1 set a=1" mariadb $backup_folder_to_restore -e "delete from t1 where a=1" mariadb -e "drop database $backup_folder_to_restore" } process_backup() { load_default_backup_variables; parse_backup_variables "$@"; print_backup_variables; check_for_dependancies "backup"; validation_prechecks_for_backup; issue_write_locks; run_save_brm; run_backup; } process_restore() { load_default_restore_variables; parse_restore_variables "$@"; print_restore_variables; check_for_dependancies "restore"; validation_prechecks_for_restore; run_restore; } case "$action" in 'help' | '--help' | '-help' | '-h') print_action_help_text ;; 'backup') process_backup "$@"; ;; 'dbrm_backup') process_dbrm_backup "$@"; ;; 'dbrm_restore') # for next release process_dbrm_restore "$@"; ;; 'restore') process_restore "$@"; ;; *) printf "\nunknown action: $action\n" print_action_help_text ;; esac exit 0;