database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

fixed on premise s3 backup & restore --endpoint-url flag

Browse Source
This commit is contained in:
Allen Herrera
2023-09-27 13:44:37 -04:00
committed by Leonid Fedorov
parent d2b05afdc1
commit 221dea1788
1 changed files with 16 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
extra/columnstore_backup.sh
View File

@ -45,8 +45,8 @@ load_default_backup_variables()
# Configurable Paths # Configurable Paths
cs_metadata=$(grep ^metadata_path $STORAGEMANGER_CNF | cut -d "=" -f 2 | tr -d " ") cs_metadata=$(grep ^metadata_path $STORAGEMANGER_CNF | cut -d "=" -f 2 | tr -d " ")
cs_journal=$(grep ^journal_path $STORAGEMANGER_CNF | cut -d "=" -f 2 | tr -d " ") cs_journal=$(grep ^journal_path $STORAGEMANGER_CNF | cut -d "=" -f 2 | tr -d " ")
cs_cache=$(grep -A25 "\[Cache\]" $STORAGEMANGER_CNF | grep ^path | cut -d "=" -f 2 | tr -d " ") cs_cache=$(grep -A25 "\[Cache\]" $STORAGEMANGER_CNF | grep ^path | cut -d "=" -f 2 | tr -d " ")
# What storage topogoly is being used by Columnstore - found in /etc/columnstore/storagemanager.cnf # What storage topogoly is being used by Columnstore - found in /etc/columnstore/storagemanager.cnf
# Options: "LocalStorage" or "S3" # Options: "LocalStorage" or "S3"
@ -101,7 +101,7 @@ load_default_backup_variables()
xtra_cmd_args="" xtra_cmd_args=""
# Used by on premise S3 vendors # Used by on premise S3 vendors
# Example: "127.0.0.1:9001" # Example: "http://127.0.0.1:8000"
s3_url="" s3_url=""
# Tracks if flush read lock has been run # Tracks if flush read lock has been run
@ -141,7 +141,7 @@ parse_backup_variables()
shift # past argument shift # past argument
shift # past value shift # past value
;; ;;
-url| --s3-endpoint) -url| --endpoint-url)
s3_url="$2" s3_url="$2"
shift # past argument shift # past argument
shift # past value shift # past value
@ -234,7 +234,7 @@ print_backup_help_text()
-bd | --backup_destination if the directory is 'Local' or 'Remote' to this script -bd | --backup_destination if the directory is 'Local' or 'Remote' to this script
-scp scp connection to remote server if -bd 'Remote' -scp scp connection to remote server if -bd 'Remote'
-bb | --backup_bucket bucket name for where to save S3 backups -bb | --backup_bucket bucket name for where to save S3 backups
-url | --s3-endpoint onprem url to s3 storage api example: 127.0.0.1:9001 -url | --endpoint-url onprem url to s3 storage api example: http://127.0.0.1:8000
-s | --storage the storage used by columnstore data 'LocalStorage' or 'S3' -s | --storage the storage used by columnstore data 'LocalStorage' or 'S3'
-i | --incremental adds columnstore deltas to an existing full backup -i | --incremental adds columnstore deltas to an existing full backup
-P | --parallel number of parallel rsync threads to run -P | --parallel number of parallel rsync threads to run
@ -258,7 +258,7 @@ print_backup_help_text()
S3 Examples: S3 Examples:
./columnstore_backup.sh -bb s3://my-cs-backups -s S3 ./columnstore_backup.sh -bb s3://my-cs-backups -s S3
./columnstore_backup.sh -bb gs://my-cs-backups -s S3 --incremental 02-18-2022 ./columnstore_backup.sh -bb gs://my-cs-backups -s S3 --incremental 02-18-2022
./columnstore_backup.sh -bb s3://my-premise-bucket -s S3 -url 127.0.0.1:9001 ./columnstore_backup.sh -bb s3://my-premise-bucket -s S3 -url http://127.0.0.1:8000
Cron Example: Cron Example:
*/30 * * * * root bash /root/columnstore_backup.sh -bb s3://my-cs-backups -s S3 >> /root/csBackup.log 2>&1 */30 * * * * root bash /root/columnstore_backup.sh -bb s3://my-cs-backups -s S3 >> /root/csBackup.log 2>&1
@ -422,7 +422,7 @@ validation_prechecks_for_backup()
# Adjust s3api flags for onpremise/custom endpoints # Adjust s3api flags for onpremise/custom endpoints
add_s3_api_flags="" add_s3_api_flags=""
if [ ! -z "$s3_url" ]; then add_s3_api_flags=" --s3-endpoint $s3_url"; fi; if [ ! -z "$s3_url" ]; then add_s3_api_flags=" --endpoint-url $s3_url"; fi;
# Validate addtional relevant arguments for S3 # Validate addtional relevant arguments for S3
@ -684,14 +684,14 @@ handle_early_exit_on_backup()
{ {
skip_read_unlock=${2:-false} skip_read_unlock=${2:-false}
if ! $skip_read_unlock; then clearReadLock; fi; if ! $skip_read_unlock; then clearReadLock; fi;
printf "\nBackup Failed: $1" printf "\nBackup Failed: $1\n"
alert $1 alert $1
exit 1; exit 1;
} }
handle_early_exit_on_restore() handle_early_exit_on_restore()
{ {
printf "\nRestore Failed: $1" printf "\nRestore Failed: $1\n"
alert $1 alert $1
exit 1; exit 1;
} }
@ -1045,6 +1045,7 @@ run_backup()
extra_flags="" extra_flags=""
if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi; if $skip_mdb; then extra_flags+=" --skip-mariadb-backup"; fi;
if $skip_bucket_data; then extra_flags+=" --skip-bucket-data"; fi; if $skip_bucket_data; then extra_flags+=" --skip-bucket-data"; fi;
if [ ! -z "$s3_url" ]; then extra_flags+=" -url $s3_url"; fi;
echo "./columnstore_backup.sh restore -l $today -s $storage -bb $backup_bucket -dbs $DBROOT_COUNT -m $mode -nb $protocol://$bucket $extra_flags --quiet --continue" > restoreS3.job echo "./columnstore_backup.sh restore -l $today -s $storage -bb $backup_bucket -dbs $DBROOT_COUNT -m $mode -nb $protocol://$bucket $extra_flags --quiet --continue" > restoreS3.job
s3cp restoreS3.job $backup_bucket/$today/restoreS3.job s3cp restoreS3.job $backup_bucket/$today/restoreS3.job
rm -rf restoreS3.job rm -rf restoreS3.job
@ -1107,7 +1108,7 @@ load_default_restore_variables()
new_secret='' new_secret=''
# Used by on premise S3 vendors # Used by on premise S3 vendors
# Example: "127.0.0.1:9001" # Example: "http://127.0.0.1:8000"
s3_url="" s3_url=""
# Number of DBroots # Number of DBroots
@ -1172,7 +1173,7 @@ parse_restore_variables()
shift # past argument shift # past argument
shift # past value shift # past value
;; ;;
-url| --s3-endpoint) -url| --endpoint-url)
s3_url="$2" s3_url="$2"
shift # past argument shift # past argument
shift # past value shift # past value
@ -1269,7 +1270,7 @@ print_restore_help_text()
-dbs | --dbroots Number of database roots in the backup -dbs | --dbroots Number of database roots in the backup
-scp scp connection to remote server if -bd 'Remote' -scp scp connection to remote server if -bd 'Remote'
-bb | --backup_bucket bucket name for where to find the S3 backups -bb | --backup_bucket bucket name for where to find the S3 backups
-url | --s3-endpoint Onprem url to s3 storage api example: 127.0.0.1:9001 -url | --endpoint-url Onprem url to s3 storage api example: http://127.0.0.1:8000
-s | --storage The storage used by columnstore data 'LocalStorage' or 'S3' -s | --storage The storage used by columnstore data 'LocalStorage' or 'S3'
-pm | --nodeid Forces the handling of the restore as this node as opposed to whats detected on disk -pm | --nodeid Forces the handling of the restore as this node as opposed to whats detected on disk
-nb | --new_bucket Defines the new bucket to copy the s3 data to from the backup bucket. -nb | --new_bucket Defines the new bucket to copy the s3 data to from the backup bucket.
@ -1289,7 +1290,7 @@ print_restore_help_text()
S3 Storage Examples: S3 Storage Examples:
./columnstore_backup.sh restore -s S3 -bb s3://my-cs-backups -l 12-29-2021 ./columnstore_backup.sh restore -s S3 -bb s3://my-cs-backups -l 12-29-2021
./columnstore_backup.sh restore -s S3 -bb gs://on-premise-bucket -l 12-29-2021 -url 127.0.0.1:9001 ./columnstore_backup.sh restore -s S3 -bb gs://on-premise-bucket -l 12-29-2021 -url http://127.0.0.1:8000
./columnstore_backup.sh restore -s S3 -bb s3://my-cs-backups -l 08-16-2022 -nb s3://new-data-bucket -nr us-east-1 -nk AKIAxxxxxxx3FHCADF -ns GGGuxxxxxxxxxxnqa72csk5 -ha ./columnstore_backup.sh restore -s S3 -bb s3://my-cs-backups -l 08-16-2022 -nb s3://new-data-bucket -nr us-east-1 -nk AKIAxxxxxxx3FHCADF -ns GGGuxxxxxxxxxxnqa72csk5 -ha
"; ";
} }
@ -1351,7 +1352,7 @@ validation_prechecks_for_restore() {
# Adjust s3api flags for onpremise/custom endpoints # Adjust s3api flags for onpremise/custom endpoints
add_s3_api_flags="" add_s3_api_flags=""
if [ ! -z "$s3_url" ]; then add_s3_api_flags=" --s3-endpoint $s3_url"; fi if [ ! -z "$s3_url" ]; then add_s3_api_flags=" --endpoint-url $s3_url"; fi
# If remote backup - Validate that scp works # If remote backup - Validate that scp works
if [ $backup_destination == "Remote" ]; then if [ $backup_destination == "Remote" ]; then
@ -1622,4 +1623,4 @@ case "$1" in
;; ;;
esac esac
exit 0; exit 0;