You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-30 19:23:07 +03:00
MCOL-5618-related: Duplicate to stable 23.10 (#3223)
* feat(package_manager): improved cs_package_manager functionality for better UX - add apt support for community and enterprise install - add enterprise_staging install - filter check results down to os/arch - add --token flag for enterprise (#3002) * Update cs package manager dev ubuntu (#3060) * cs_package_manager.sh - fix ubuntu dev installs * cs_package_manager.sh - remove some useless comments * debian/ubuntu dev install to use drone repo + remove wget dependancy * + exits if repo files missing * auto install aws cli for dev builds * removed commented section from cs_package_manager * feat(tooling): add rhel support in cs_package_manager.sh (#3105) * added dbrm_restore for LocalStorage & S3 * small comment cleanup * resolved PR comments for cs_package_manager.sh & mcs_backup_manager.sh * added dbrm_restore for LocalStorage & S3 * Added backup retention flag -r and incremental auto_most_recent option to mcs_backup_manager.sh * resolved PR comments for cs_package_manager.sh & mcs_backup_manager.sh * add comment to auto_select_most_recent_backup_for_incremental in mcs_backup_manager.sh * feat(cmapi,cli): MCOL-5618: Add backup, restore, backup-dbrm, restore-dbrm commands for mcs cli tool. (#3130) * MCOL-5618: Add backup, restore, backup-dbrm, restore-dbrm commands for mcs cli tool. [add] mcs_cluster_tool/helpers.py file with cook_sh_arg function inside [add] MCS_BACKUP_MANAGER_SH to mcs_cluster_tool/constants.py [add] backup and restore commands to "mcs" Typer APP * MCOL-5618: Move mcs_backup_manager.sh to cmapi/scripts. * MCOL-5618: Install mcs_backup_manager.sh with CMAPI package. * MCOL-5618: Testing fix + script minor fixes. - [fix] wrong cooking of .sh invoke string - [fix] restore_dbrm -> dbrm_restore, backup_dbrm -> dbrm_backup to use same naming in both python wrapper and mcs_backup_manager.sh - [fix] mathod of checking cmapi online or not in mcs_backup_manager.sh * MCOL-5618: Minor improvements to sync with PR 3144. [add] -ns option for dbrm_restore. * MCOL-5618: Minor fixes to sync after #3147 [add] -r option for backup command * MCOL-5618: Minor fixes to sync after PR #1398. [add] -ssm argument for dbrm backup [add] -sdbk -ssm arguments for dbrm restore [add] silenece annoying pylint warning unused argument for bachup_commands.py and restore_commnads.py * update to the latest mcs_backup_manager + cs_package_manager * Intermediate commit to move changes from old filepath to the new one. * remove old file * MCOL-5618: Fix packaging. [mv] extra/cs_package_manager.sh -> cmapi/scripts/cs_package_manager.sh to pack and deliver the script with cmapi package [add] cs_package_manager.sh to cmapi package [fix] download paths for mcs_backup_manager.sh and cs_package_manager.sh * MCOL-5618: Fix after testing [add] -P to restore help in mcs_backup_restore.sh [fix] -f argument default value in both backup and restore wrapper commands [fix] -i argument behaviour and type + default value [fix] -P argument position and help for both backup and restore wrapper commands [fix] disable some pylint warning for both backup and restore wrapper files --------- Co-authored-by: Allen Herrera <82840027+mariadb-AllenHerrera@users.noreply.github.com> Co-authored-by: Allen Herrera <allen.herrera@mariadb.com> Co-authored-by: Allen Herrera <allen@Allens-Black-MacBook-Pro.local> Co-authored-by: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com>
This commit is contained in:
Alan Mologorsky
GitHub
324
cmapi/mcs_cluster_tool/restore_commands.py
Normal file
324
cmapi/mcs_cluster_tool/restore_commands.py
Normal file
@ -0,0 +1,324 @@
|
||||
"""Typer application for restore Columnstore data."""
|
||||
import logging
|
||||
import sys
|
||||
from typing_extensions import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from cmapi_server.process_dispatchers.base import BaseDispatcher
|
||||
from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH
|
||||
from mcs_cluster_tool.decorators import handle_output
|
||||
from mcs_cluster_tool.helpers import cook_sh_arg
|
||||
|
||||
|
||||
logger = logging.getLogger('mcs_cli')
|
||||
# pylint: disable=unused-argument, too-many-arguments, too-many-locals
|
||||
# pylint: disable=invalid-name, line-too-long
|
||||
|
||||
|
||||
@handle_output
|
||||
def restore(
|
||||
l: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-l', '--load',
|
||||
help='What date folder to load from the backup_location.'
|
||||
)
|
||||
] = '',
|
||||
bl: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bl', '--backup-location',
|
||||
help=(
|
||||
'Where the backup to load is found.\n'
|
||||
'Example: /mnt/backups/'
|
||||
)
|
||||
)
|
||||
] = '/tmp/backups/',
|
||||
bd: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bd', '--backup_destination',
|
||||
help=(
|
||||
'Is this backup on the same or remote server compared to '
|
||||
'where this script is running.\n'
|
||||
'Options: "Local" or "Remote"'
|
||||
)
|
||||
)
|
||||
] = 'Local',
|
||||
scp: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-scp', '--secure-copy-protocol',
|
||||
help=(
|
||||
'Used only if --backup-destination=Remote'
|
||||
'The user/credentials that will be used to scp the backup files.'
|
||||
'Example: "centos@10.14.51.62"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
bb: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bb', '--backup-bucket',
|
||||
help=(
|
||||
'Only used if --storage=S3\n'
|
||||
'Name of the bucket to store the columnstore backups.\n'
|
||||
'Example: "s3://my-cs-backups"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
url: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-url', '--endpoint-url',
|
||||
help=(
|
||||
'Used by on premise S3 vendors.\n'
|
||||
'Example: "http://127.0.0.1:8000"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
s: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-s', '--storage',
|
||||
help=(
|
||||
'What storage topogoly is being used by Columnstore - found '
|
||||
'in /etc/columnstore/storagemanager.cnf.\n'
|
||||
'Options: "LocalStorage" or "S3"'
|
||||
)
|
||||
)
|
||||
] = 'LocalStorage',
|
||||
dbs: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-dbs', '--dbroots',
|
||||
help='Number of database roots in the backup.'
|
||||
)
|
||||
] = 1,
|
||||
pm: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-pm', '--nodeid',
|
||||
help=(
|
||||
'Forces the handling of the restore as this node as opposed '
|
||||
'to whats detected on disk.'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
nb: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-nb', '--new-bucket',
|
||||
help=(
|
||||
'Defines the new bucket to copy the s3 data to from the '
|
||||
'backup bucket. Use -nb if the new restored cluster should '
|
||||
'use a different bucket than the backup bucket itself.'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
nr: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-nr', '--new-region',
|
||||
help=(
|
||||
'Defines the region of the new bucket to copy the s3 data to '
|
||||
'from the backup bucket.'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
nk: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-nk', '--new-key',
|
||||
help='Defines the aws key to connect to the new_bucket.'
|
||||
)
|
||||
] = '',
|
||||
ns: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-ns', '--new-secret',
|
||||
help=(
|
||||
'Defines the aws secret of the aws key to connect to the '
|
||||
'new_bucket.'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
ha: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-ha/-no-ha', '--highavilability/--no-highavilability',
|
||||
help=(
|
||||
'Flag for high available systems (meaning shared storage '
|
||||
'exists supporting the topology so that each node sees '
|
||||
'all data)'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
cont: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-cont/-no-cont', '--continue/--no-continue',
|
||||
help=(
|
||||
'This acknowledges data in your --new_bucket is ok to delete '
|
||||
'when restoring S3. When set to true skips the enforcement '
|
||||
'that new_bucket should be empty prior to starting a restore.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
f: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-f', '--config-file',
|
||||
help='Path to backup configuration file to load variables from.',
|
||||
show_default=False
|
||||
)
|
||||
] = '',
|
||||
smdb: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-smdb/-no-smdb', '--skip-mariadb-backup/--no-skip-mariadb-backup',
|
||||
help=(
|
||||
'Skip restoring mariadb server via mariadb-backup - ideal for '
|
||||
'only restoring columnstore.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
sb: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-sb/-no-sb', '--skip-bucket-data/--no-skip-bucket-data',
|
||||
help=(
|
||||
'Skip restoring columnstore data in the bucket - ideal if '
|
||||
'looking to only restore mariadb server.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
m: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-m', '--mode',
|
||||
help=(
|
||||
'Modes ["direct","indirect"] - direct backups run on the '
|
||||
'columnstore nodes themselves. indirect run on another '
|
||||
'machine that has read-only mounts associated with '
|
||||
'columnstore/mariadb\n'
|
||||
),
|
||||
hidden=True
|
||||
)
|
||||
] = 'direct',
|
||||
c: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-c', '--compress',
|
||||
help=(
|
||||
'Hint that the backup is compressed in X format. '
|
||||
'Options: [ pigz ].'
|
||||
),
|
||||
show_default=False
|
||||
)
|
||||
] = '',
|
||||
P: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-P', '--parallel',
|
||||
help=(
|
||||
'Determines number of decompression and mdbstream threads. '
|
||||
'Ignored if "-c/--compress" argument not set.'
|
||||
)
|
||||
)
|
||||
] = 4,
|
||||
q: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-q/-no-q', '--quiet/--no-quiet',
|
||||
help='Silence verbose copy command outputs.'
|
||||
)
|
||||
] = False,
|
||||
nv_ssl: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl',
|
||||
help='Skips verifying ssl certs, useful for onpremise s3 storage.'
|
||||
)
|
||||
] = False,
|
||||
):
|
||||
"""Restore Columnstore (and/or MariaDB) data."""
|
||||
|
||||
# Local Storage Examples:
|
||||
# ./$0 restore -s LocalStorage -bl /tmp/backups/ -bd Local -l 12-29-2021
|
||||
# ./$0 restore -s LocalStorage -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -l 12-29-2021
|
||||
|
||||
# S3 Storage Examples:
|
||||
# ./$0 restore -s S3 -bb s3://my-cs-backups -l 12-29-2021
|
||||
# ./$0 restore -s S3 -bb gs://on-premise-bucket -l 12-29-2021 -url http://127.0.0.1:8000
|
||||
# ./$0 restore -s S3 -bb s3://my-cs-backups -l 08-16-2022 -nb s3://new-data-bucket -nr us-east-1 -nk AKIAxxxxxxx3FHCADF -ns GGGuxxxxxxxxxxnqa72csk5 -ha
|
||||
arguments = []
|
||||
for arg_name, value in locals().items():
|
||||
sh_arg = cook_sh_arg(arg_name, value)
|
||||
if sh_arg is None:
|
||||
continue
|
||||
arguments.append(sh_arg)
|
||||
cmd = f'{MCS_BACKUP_MANAGER_SH} restore {" ".join(arguments)}'
|
||||
success, _ = BaseDispatcher.exec_command(cmd, stdout=sys.stdout)
|
||||
return {'success': success}
|
||||
|
||||
|
||||
@handle_output
|
||||
def dbrm_restore(
|
||||
p: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-p', '--path',
|
||||
help='Path of where dbrm backups stored on disk.'
|
||||
)
|
||||
] = '/tmp/dbrm_backups',
|
||||
d: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-d', '--directory',
|
||||
help='Date or directory chose to restore from.'
|
||||
)
|
||||
] = '',
|
||||
ns: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-ns', '--no-start',
|
||||
help=(
|
||||
'Do not attempt columnstore startup post dbrm_restore.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
sdbk: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-sdbk/-no-sdbk', '--skip-dbrm-backup/--no-skip-dbrm-backup',
|
||||
help=(
|
||||
'Skip backing up dbrms before restoring.'
|
||||
)
|
||||
)
|
||||
] = True,
|
||||
ssm: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager',
|
||||
help='Skip backing up storagemanager directory.'
|
||||
)
|
||||
] = True,
|
||||
):
|
||||
"""Restore Columnstore DBRM data."""
|
||||
|
||||
# Default: ./$0 dbrm_restore --path /tmp/dbrm_backups
|
||||
|
||||
# Examples:
|
||||
# ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842
|
||||
# ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 --no-start
|
||||
arguments = []
|
||||
for arg_name, value in locals().items():
|
||||
sh_arg = cook_sh_arg(arg_name, value)
|
||||
if sh_arg is None:
|
||||
continue
|
||||
arguments.append(sh_arg)
|
||||
cmd = f'{MCS_BACKUP_MANAGER_SH} dbrm_restore {" ".join(arguments)}'
|
||||
success, _ = BaseDispatcher.exec_command(cmd, stdout=sys.stdout)
|
||||
return {'success': success}
|
||||
Reference in New Issue
Block a user