database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

feat(cmapi, mcs): MCOL-5941 Improve mcs cli backup\restore wrapper.

Browse Source 
* fix(mcs, wrapper): default is None for --backup-location, --backup-destination, --storage, --parallel, --highavilability, --skip-save-brm, --skip-polls, --skip-locks, --skip-mariadb-backup, --skip-bucket-data, --name-backup, --quiet, --no-verify-ssl, --poll-interval, --poll-max-wait, --retention-days, -scp, -bb, -url, -f, -m, -aro, -li and most of arguments in backup_commands.py and restore_commands.py
* fix(mcs, helpers): cook_sh_arg parser function now detects None as a value
* fix(mcs, wrapper): list -> li in typer command argument name for both backup_commands.py and restore_commands.py
* docs(mcs, wrapper): --parralel arg help message was edited to simple and readable one
* fix(mcs, wrapper): removing -no- prefixed flag variants for all bool arguments in backup_commands.py and restore_commands.py
This commit is contained in:
mariadb-AlanMologorsky
2025-04-15 05:59:22 +03:00
committed by drrtuy
parent d5c8b98162
commit 927eb4b2bd
3 changed files with 186 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
cmapi/mcs_cluster_tool/backup_commands.py
View File

@ -1,7 +1,7 @@
"""Typer application for backup Columnstore data.""" """Typer application for backup Columnstore data."""
import logging import logging
import sys import sys
from datetime import datetime from typing import Optional
import typer import typer
from typing_extensions import Annotated from typing_extensions import Annotated
@ -20,7 +20,7 @@ logger = logging.getLogger('mcs_cli')
@handle_output @handle_output
def backup( def backup(
bl: Annotated[ bl: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-bl', '--backup-location', '-bl', '--backup-location',
help=( help=(
@ -28,11 +28,12 @@ def backup(
'Consider write permissions of the scp user and the user running this script.\n' 'Consider write permissions of the scp user and the user running this script.\n'
'Mariadb-backup will use this location as a tmp dir for S3 and remote backups temporarily.\n' 'Mariadb-backup will use this location as a tmp dir for S3 and remote backups temporarily.\n'
'Example: /mnt/backups/' 'Example: /mnt/backups/'
) ),
show_default=False
) )
] = '/tmp/backups/', ] = None,
bd: Annotated[ bd: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-bd', '--backup-destination', '-bd', '--backup-destination',
help=( help=(
@ -40,11 +41,12 @@ def backup(
'script is running on or another server - if Remote you need ' 'script is running on or another server - if Remote you need '
'to setup scp=' 'to setup scp='
'Options: "Local" or "Remote"' 'Options: "Local" or "Remote"'
) ),
show_default=False
) )
] = 'Local', ] = None,
scp: Annotated[ scp: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-scp', '-scp',
help=( help=(
@ -54,20 +56,20 @@ def backup(
'Example: "centos@10.14.51.62"' 'Example: "centos@10.14.51.62"'
) )
) )
] = '', ] = None,
bb: Annotated[ bb: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-bb', '--backup-bucket', '-bb', '--backup-bucket',
help=( help=(
'Only used if --storage=S3\n' 'Only used if --storage=S3\n'
'Name of the bucket to store the columnstore backups.\n' 'Name of the bucket to store the columnstore backups.\n'
'Example: "s3://my-cs-backups"' 'Example: "s3://my-cs-backups"'
) ),
) )
] = '', ] = None,
url: Annotated[ url: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-url', '--endpoint-url', '-url', '--endpoint-url',
help=( help=(
@ -75,18 +77,19 @@ def backup(
'Example: "http://127.0.0.1:8000"' 'Example: "http://127.0.0.1:8000"'
) )
) )
] = '', ] = None,
s: Annotated[ s: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-s', '--storage', '-s', '--storage',
help=( help=(
'What storage topogoly is being used by Columnstore - found ' 'What storage topogoly is being used by Columnstore - found '
'in /etc/columnstore/storagemanager.cnf.\n' 'in /etc/columnstore/storagemanager.cnf.\n'
'Options: "LocalStorage" or "S3"' 'Options: "LocalStorage" or "S3"'
) ),
show_default=False
) )
] = 'LocalStorage', ] = None,
i: Annotated[ i: Annotated[
str, str,
typer.Option( typer.Option(
@ -101,31 +104,32 @@ def backup(
) )
] = '', ] = '',
P: Annotated[ P: Annotated[
int, Optional[int],
typer.Option( typer.Option(
'-P', '--parallel', '-P', '--parallel',
help=( help=(
'Determines if columnstore data directories will have ' 'Enables parallel rsync for faster backups, setting the '
'multiple rsync running at the same time for different ' 'number of simultaneous rsync processes. With -c/--compress, '
'subfolders to parallelize writes. ' 'sets the number of compression threads.'
'Ignored if "-c/--compress" argument not set.' ),
) show_default=False
) )
] = 4, ] = None,
ha: Annotated[ ha: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-ha/-no-ha', '--highavilability/--no-highavilability', '-ha', '--highavilability',
help=( help=(
'Hint wether shared storage is attached @ below on all nodes ' 'Hint wether shared storage is attached @ below on all nodes '
'to see all data\n' 'to see all data\n'
' HA LocalStorage ( /var/lib/columnstore/dataX/ )\n' ' HA LocalStorage ( /var/lib/columnstore/dataX/ )\n'
' HA S3 ( /var/lib/columnstore/storagemanager/ )' ' HA S3 ( /var/lib/columnstore/storagemanager/ )'
) ),
show_default=False
) )
] = False, ] = None,
f: Annotated[ f: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-f', '--config-file', '-f', '--config-file',
help=( help=(
@ -134,57 +138,63 @@ def backup(
), ),
show_default=False show_default=False
) )
] = '', ] = None,
sbrm: Annotated[ sbrm: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-sbrm/-no-sbrm', '--skip-save-brm/--no-skip-save-brm', '-sbrm', '--skip-save-brm',
help=( help=(
'Skip saving brm prior to running a backup - ' 'Skip saving brm prior to running a backup - '
'ideal for dirty backups.' 'ideal for dirty backups.'
) ),
show_default=False
) )
] = False, ] = None,
spoll: Annotated[ spoll: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-spoll/-no-spoll', '--skip-polls/--no-skip-polls', '-spoll', '--skip-polls',
help='Skip sql checks confirming no write/cpimports running.' help='Skip sql checks confirming no write/cpimports running.',
show_default=False
) )
] = False, ] = None,
slock: Annotated[ slock: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-slock/-no-slock', '--skip-locks/--no-skip-locks', '-slock', '--skip-locks',
help='Skip issuing write locks - ideal for dirty backups.' help='Skip issuing write locks - ideal for dirty backups.',
show_default=False
) )
] = False, ] = None,
smdb: Annotated[ smdb: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-smdb/-no-smdb', '--skip-mariadb-backup/--no-skip-mariadb-backup', '-smdb', '--skip-mariadb-backup',
help=( help=(
'Skip running a mariadb-backup for innodb data - ideal for ' 'Skip running a mariadb-backup for innodb data - ideal for '
'incremental dirty backups.' 'incremental dirty backups.'
) ),
show_default=False
) )
] = False, ] = None,
sb: Annotated[ sb: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-sb/-no-sb', '--skip-bucket-data/--no-skip-bucket-data', '-sb', '--skip-bucket-data',
help='Skip taking a copy of the columnstore data in the bucket.' help='Skip taking a copy of the columnstore data in the bucket.',
show_default=False
) )
] = False, ] = None,
nb: Annotated[ nb: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-nb', '--name-backup', '-nb', '--name-backup',
help='Define the name of the backup - default: $(date +%m-%d-%Y)' help='Define the name of the backup - default: $(date +%m-%d-%Y)',
show_default=False
) )
] = datetime.now().strftime('%m-%d-%Y'), ] = None,
m: Annotated[ m: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-m', '--mode', '-m', '--mode',
help=( help=(
@ -193,78 +203,86 @@ def backup(
'machine that has read-only mounts associated with ' 'machine that has read-only mounts associated with '
'columnstore/mariadb\n' 'columnstore/mariadb\n'
), ),
hidden=True hidden=True,
show_default='direct'
) )
] = 'direct', ] = None,
c: Annotated[ c: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-c', '--compress', '-c', '--compress',
help='Compress backup in X format - Options: [ pigz ].', help='Compress backup in X format - Options: [ pigz ].',
show_default=False show_default=False
) )
] = '', ] = None,
q: Annotated[ q: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-q/-no-q', '--quiet/--no-quiet', '-q', '--quiet',
help='Silence verbose copy command outputs.' help='Silence verbose copy command outputs.',
show_default=False
) )
] = False, ] = None,
nv_ssl: Annotated[ nv_ssl: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl', '-nv-ssl','--no-verify-ssl',
help='Skips verifying ssl certs, useful for onpremise s3 storage.' help='Skips verifying ssl certs, useful for onpremise s3 storage.',
show_default=False
) )
] = False, ] = None,
pi: Annotated[ pi: Annotated[
int, Optional[int],
typer.Option( typer.Option(
'-pi', '--poll-interval', '-pi', '--poll-interval',
help=( help=(
'Number of seconds between poll checks for active writes & ' 'Number of seconds between poll checks for active writes & '
'cpimports.' 'cpimports.'
) ),
show_default=False
) )
] = 5, ] = None,
pmw: Annotated[ pmw: Annotated[
int, Optional[int],
typer.Option( typer.Option(
'-pmw', '--poll-max-wait', '-pmw', '--poll-max-wait',
help=( help=(
'Max number of minutes for polling checks for writes to wait ' 'Max number of minutes for polling checks for writes to wait '
'before exiting as a failed backup attempt.' 'before exiting as a failed backup attempt.'
) ),
show_default=False
) )
] = 60, ] = None,
r: Annotated[ r: Annotated[
int, Optional[int],
typer.Option( typer.Option(
'-r', '--retention-days', '-r', '--retention-days',
help=( help=(
'Retain backups created within the last X days, ' 'Retain backups created within the last X days, '
'default 0 == keep all backups.' 'default 0 == keep all backups.'
) ),
show_default=False
) )
] = 0, ] = None,
aro: Annotated[ aro: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-aro', '--apply-retention-only', '-aro', '--apply-retention-only',
help=( help=(
'Only apply retention policy to existing backups, ' 'Only apply retention policy to existing backups, '
'does not run a backup.' 'does not run a backup.'
) ),
show_default=False
) )
] = False, ] = None,
list: Annotated[ li: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-li', '--list', '-li', '--list',
help='List backups.' help='List backups.',
show_default=False
) )
] = False, ] = None,
): ):
"""Backup Columnstore and/or MariDB data.""" """Backup Columnstore and/or MariDB data."""
@ -341,26 +359,29 @@ def dbrm_backup(
) )
] = 'dbrm_backup', ] = 'dbrm_backup',
ssm: Annotated[ ssm: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager', '-ssm', '--skip-storage-manager',
help='Skip backing up storagemanager directory.' help='Skip backing up storagemanager directory.',
show_default=False
) )
] = False, ] = None,
q: Annotated[ q: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-q/-no-q', '--quiet/--no-quiet', '-q', '--quiet',
help='Silence verbose copy command outputs.' help='Silence verbose copy command outputs.',
show_default=False
) )
] = False, ] = None,
list: Annotated[ li: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-li', '--list', '-li', '--list',
help='List backups.' help='List backups.',
show_default=False
) )
] = False, ] = None,
): ):
"""Columnstore DBRM Backup.""" """Columnstore DBRM Backup."""

17
cmapi/mcs_cluster_tool/helpers.py
View File

@ -1,22 +1,27 @@
"""Module with helper functions for mcs cli tool.""" """Module with helper functions for mcs cli tool."""
from typing import Union from typing import Optional, Union
def cook_sh_arg(arg_name: str, value:Union[str, int, bool]) -> str: def cook_sh_arg(arg_name: str, value: Union[str, int, bool]) -> Optional[str]:
"""Convert argument and and value from function locals to bash argument. """Convert argument and and value from function locals to bash argument.
:param arg_name: function argument name :param arg_name: function argument name
:type arg_name: str :type arg_name: str
:param value: function argument value :param value: function argument value
:type value: Union[str, int, bool] :type value: Union[str, int, bool]
:return: bash argument string :return: bash argument string or None
:rtype: str :rtype: Optional[str]
""" """
# skip "arguments" list and Typer ctx variables from local scope # skip "arguments" list and Typer ctx variables from local scope
if arg_name in ('arguments', 'ctx'): if arg_name in ('arguments', 'ctx'):
return None return None
# skip args that have empty string as value # skip args that have empty string or None as value
if value == '': # Condition below could be "not value", but I prefer to be explicit
# and check for empty string and None to show that it's different cases:
# empty string means that user passed empty string as value
# and None means that user didn't pass anything and our internal None
# applies
if value == '' or value is None:
return None return None
if '_' in arg_name: if '_' in arg_name:
arg_name = arg_name.replace('_', '-') arg_name = arg_name.replace('_', '-')

108
cmapi/mcs_cluster_tool/restore_commands.py
View File

@ -1,6 +1,7 @@
"""Typer application for restore Columnstore data.""" """Typer application for restore Columnstore data."""
import logging import logging
import sys import sys
from typing import Optional
import typer import typer
from typing_extensions import Annotated from typing_extensions import Annotated
@ -155,29 +156,31 @@ def restore(
) )
] = 4, ] = 4,
ha: Annotated[ ha: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-ha/-no-ha', '--highavilability/--no-highavilability', '-ha', '--highavilability',
help=( help=(
'Flag for high available systems (meaning shared storage ' 'Flag for high available systems (meaning shared storage '
'exists supporting the topology so that each node sees ' 'exists supporting the topology so that each node sees '
'all data)' 'all data)'
) ),
show_default=False
) )
] = False, ] = None,
cont: Annotated[ cont: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-cont/-no-cont', '--continue/--no-continue', '-cont', '--continue',
help=( help=(
'This acknowledges data in your --new_bucket is ok to delete ' 'This acknowledges data in your --new_bucket is ok to delete '
'when restoring S3. When set to true skips the enforcement ' 'when restoring S3. When set to true skips the enforcement '
'that new_bucket should be empty prior to starting a restore.' 'that new_bucket should be empty prior to starting a restore.'
) ),
show_default=False
) )
] = False, ] = None,
f: Annotated[ f: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-f', '--config-file', '-f', '--config-file',
help=( help=(
@ -186,29 +189,30 @@ def restore(
), ),
show_default=False show_default=False
) )
] = '', ] = None,
smdb: Annotated[ smdb: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-smdb/-no-smdb', '--skip-mariadb-backup/--no-skip-mariadb-backup', '-smdb', '--skip-mariadb-backup',
help=( help=(
'Skip restoring mariadb server via mariadb-backup - ideal for ' 'Skip restoring mariadb server via mariadb-backup - ideal for '
'only restoring columnstore.' 'only restoring columnstore.'
) ),
show_default=False
) )
] = False, ] = None,
sb: Annotated[ sb: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-sb/-no-sb', '--skip-bucket-data/--no-skip-bucket-data', '-sb', '--skip-bucket-data',
help=( help=(
'Skip restoring columnstore data in the bucket - ideal if ' 'Skip restoring columnstore data in the bucket - ideal if '
'looking to only restore mariadb server.' 'looking to only restore mariadb server.'
) )
) )
] = False, ] = None,
m: Annotated[ m: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-m', '--mode', '-m', '--mode',
help=( help=(
@ -217,11 +221,12 @@ def restore(
'machine that has read-only mounts associated with ' 'machine that has read-only mounts associated with '
'columnstore/mariadb\n' 'columnstore/mariadb\n'
), ),
hidden=True hidden=True,
show_default='direct'
) )
] = 'direct', ] = None,
c: Annotated[ c: Annotated[
str, Optional[str],
typer.Option( typer.Option(
'-c', '--compress', '-c', '--compress',
help=( help=(
@ -230,28 +235,31 @@ def restore(
), ),
show_default=False show_default=False
) )
] = '', ] = None,
q: Annotated[ q: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-q/-no-q', '--quiet/--no-quiet', '-q', '--quiet',
help='Silence verbose copy command outputs.' help='Silence verbose copy command outputs.',
show_default=False
) )
] = False, ] = None,
nv_ssl: Annotated[ nv_ssl: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl', '-nv-ssl','--no-verify-ssl',
help='Skips verifying ssl certs, useful for onpremise s3 storage.' help='Skips verifying ssl certs, useful for onpremise s3 storage.',
show_default=False,
) )
] = False, ] = None,
list: Annotated[ li: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-li', '--list', '-li', '--list',
help='List backups.' help='List backups.',
show_default=False
) )
] = False ] = None
): ):
"""Restore Columnstore (and/or MariaDB) data.""" """Restore Columnstore (and/or MariaDB) data."""
@ -291,37 +299,41 @@ def dbrm_restore(
) )
] = '', ] = '',
ns: Annotated[ ns: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-ns', '--no-start', '-ns', '--no-start',
help=( help=(
'Do not attempt columnstore startup post dbrm_restore.' 'Do not attempt columnstore startup post dbrm_restore.'
) ),
show_default=False
) )
] = False, ] = None,
sdbk: Annotated[ sdbk: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-sdbk/-no-sdbk', '--skip-dbrm-backup/--no-skip-dbrm-backup', '-sdbk', '--skip-dbrm-backup',
help=( help=(
'Skip backing up dbrms before restoring.' 'Skip backing up dbrms before restoring.'
) ),
show_default=False
) )
] = True, ] = None,
ssm: Annotated[ ssm: Annotated[
bool, Optional[bool],
typer.Option( typer.Option(
'-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager', '-ssm', '--skip-storage-manager',
help='Skip backing up storagemanager directory.' help='Skip backing up storagemanager directory.',
show_default=False
) )
] = True, ] = None,
list: Annotated[ li: Annotated[
bool, bool,
typer.Option( typer.Option(
'-li', '--list', '-li', '--list',
help='List backups.' help='List backups.',
show_default=False
) )
] = False ] = None
): ):
"""Restore Columnstore DBRM data.""" """Restore Columnstore DBRM data."""