diff --git a/cmapi/mcs_cluster_tool/backup_commands.py b/cmapi/mcs_cluster_tool/backup_commands.py index 5073786f2..1b74e6bab 100644 --- a/cmapi/mcs_cluster_tool/backup_commands.py +++ b/cmapi/mcs_cluster_tool/backup_commands.py @@ -1,7 +1,7 @@ """Typer application for backup Columnstore data.""" import logging import sys -from datetime import datetime +from typing import Optional import typer from typing_extensions import Annotated @@ -20,7 +20,7 @@ logger = logging.getLogger('mcs_cli') @handle_output def backup( bl: Annotated[ - str, + Optional[str], typer.Option( '-bl', '--backup-location', help=( @@ -28,11 +28,12 @@ def backup( 'Consider write permissions of the scp user and the user running this script.\n' 'Mariadb-backup will use this location as a tmp dir for S3 and remote backups temporarily.\n' 'Example: /mnt/backups/' - ) + ), + show_default=False ) - ] = '/tmp/backups/', + ] = None, bd: Annotated[ - str, + Optional[str], typer.Option( '-bd', '--backup-destination', help=( @@ -40,11 +41,12 @@ def backup( 'script is running on or another server - if Remote you need ' 'to setup scp=' 'Options: "Local" or "Remote"' - ) + ), + show_default=False ) - ] = 'Local', + ] = None, scp: Annotated[ - str, + Optional[str], typer.Option( '-scp', help=( @@ -54,20 +56,20 @@ def backup( 'Example: "centos@10.14.51.62"' ) ) - ] = '', + ] = None, bb: Annotated[ - str, + Optional[str], typer.Option( '-bb', '--backup-bucket', help=( 'Only used if --storage=S3\n' 'Name of the bucket to store the columnstore backups.\n' 'Example: "s3://my-cs-backups"' - ) + ), ) - ] = '', + ] = None, url: Annotated[ - str, + Optional[str], typer.Option( '-url', '--endpoint-url', help=( @@ -75,18 +77,19 @@ def backup( 'Example: "http://127.0.0.1:8000"' ) ) - ] = '', + ] = None, s: Annotated[ - str, + Optional[str], typer.Option( '-s', '--storage', help=( 'What storage topogoly is being used by Columnstore - found ' 'in /etc/columnstore/storagemanager.cnf.\n' 'Options: "LocalStorage" or "S3"' - ) + ), + show_default=False ) - ] = 'LocalStorage', + ] = None, i: Annotated[ str, typer.Option( @@ -101,31 +104,32 @@ def backup( ) ] = '', P: Annotated[ - int, + Optional[int], typer.Option( '-P', '--parallel', help=( - 'Determines if columnstore data directories will have ' - 'multiple rsync running at the same time for different ' - 'subfolders to parallelize writes. ' - 'Ignored if "-c/--compress" argument not set.' - ) + 'Enables parallel rsync for faster backups, setting the ' + 'number of simultaneous rsync processes. With -c/--compress, ' + 'sets the number of compression threads.' + ), + show_default=False ) - ] = 4, + ] = None, ha: Annotated[ - bool, + Optional[bool], typer.Option( - '-ha/-no-ha', '--highavilability/--no-highavilability', + '-ha', '--highavilability', help=( 'Hint wether shared storage is attached @ below on all nodes ' 'to see all data\n' ' HA LocalStorage ( /var/lib/columnstore/dataX/ )\n' ' HA S3 ( /var/lib/columnstore/storagemanager/ )' - ) + ), + show_default=False ) - ] = False, + ] = None, f: Annotated[ - str, + Optional[str], typer.Option( '-f', '--config-file', help=( @@ -134,57 +138,63 @@ def backup( ), show_default=False ) - ] = '', + ] = None, sbrm: Annotated[ - bool, + Optional[bool], typer.Option( - '-sbrm/-no-sbrm', '--skip-save-brm/--no-skip-save-brm', + '-sbrm', '--skip-save-brm', help=( 'Skip saving brm prior to running a backup - ' 'ideal for dirty backups.' - ) + ), + show_default=False ) - ] = False, + ] = None, spoll: Annotated[ - bool, + Optional[bool], typer.Option( - '-spoll/-no-spoll', '--skip-polls/--no-skip-polls', - help='Skip sql checks confirming no write/cpimports running.' + '-spoll', '--skip-polls', + help='Skip sql checks confirming no write/cpimports running.', + show_default=False ) - ] = False, + ] = None, slock: Annotated[ - bool, + Optional[bool], typer.Option( - '-slock/-no-slock', '--skip-locks/--no-skip-locks', - help='Skip issuing write locks - ideal for dirty backups.' + '-slock', '--skip-locks', + help='Skip issuing write locks - ideal for dirty backups.', + show_default=False ) - ] = False, + ] = None, smdb: Annotated[ - bool, + Optional[bool], typer.Option( - '-smdb/-no-smdb', '--skip-mariadb-backup/--no-skip-mariadb-backup', + '-smdb', '--skip-mariadb-backup', help=( 'Skip running a mariadb-backup for innodb data - ideal for ' 'incremental dirty backups.' - ) + ), + show_default=False ) - ] = False, + ] = None, sb: Annotated[ - bool, + Optional[bool], typer.Option( - '-sb/-no-sb', '--skip-bucket-data/--no-skip-bucket-data', - help='Skip taking a copy of the columnstore data in the bucket.' + '-sb', '--skip-bucket-data', + help='Skip taking a copy of the columnstore data in the bucket.', + show_default=False ) - ] = False, + ] = None, nb: Annotated[ - str, + Optional[str], typer.Option( '-nb', '--name-backup', - help='Define the name of the backup - default: $(date +%m-%d-%Y)' + help='Define the name of the backup - default: $(date +%m-%d-%Y)', + show_default=False ) - ] = datetime.now().strftime('%m-%d-%Y'), + ] = None, m: Annotated[ - str, + Optional[str], typer.Option( '-m', '--mode', help=( @@ -193,78 +203,86 @@ def backup( 'machine that has read-only mounts associated with ' 'columnstore/mariadb\n' ), - hidden=True + hidden=True, + show_default='direct' ) - ] = 'direct', + ] = None, c: Annotated[ - str, + Optional[str], typer.Option( '-c', '--compress', help='Compress backup in X format - Options: [ pigz ].', show_default=False ) - ] = '', + ] = None, q: Annotated[ - bool, + Optional[bool], typer.Option( - '-q/-no-q', '--quiet/--no-quiet', - help='Silence verbose copy command outputs.' + '-q', '--quiet', + help='Silence verbose copy command outputs.', + show_default=False ) - ] = False, + ] = None, nv_ssl: Annotated[ - bool, + Optional[bool], typer.Option( - '-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl', - help='Skips verifying ssl certs, useful for onpremise s3 storage.' + '-nv-ssl','--no-verify-ssl', + help='Skips verifying ssl certs, useful for onpremise s3 storage.', + show_default=False ) - ] = False, + ] = None, pi: Annotated[ - int, + Optional[int], typer.Option( '-pi', '--poll-interval', help=( 'Number of seconds between poll checks for active writes & ' 'cpimports.' - ) + ), + show_default=False ) - ] = 5, + ] = None, pmw: Annotated[ - int, + Optional[int], typer.Option( '-pmw', '--poll-max-wait', help=( 'Max number of minutes for polling checks for writes to wait ' 'before exiting as a failed backup attempt.' - ) + ), + show_default=False ) - ] = 60, + ] = None, r: Annotated[ - int, + Optional[int], typer.Option( '-r', '--retention-days', help=( 'Retain backups created within the last X days, ' 'default 0 == keep all backups.' - ) + ), + show_default=False ) - ] = 0, + ] = None, aro: Annotated[ - bool, + Optional[bool], typer.Option( '-aro', '--apply-retention-only', help=( 'Only apply retention policy to existing backups, ' 'does not run a backup.' - ) + ), + show_default=False ) - ] = False, - list: Annotated[ - bool, + ] = None, + li: Annotated[ + Optional[bool], typer.Option( '-li', '--list', - help='List backups.' + help='List backups.', + show_default=False ) - ] = False, + ] = None, ): """Backup Columnstore and/or MariDB data.""" @@ -341,26 +359,29 @@ def dbrm_backup( ) ] = 'dbrm_backup', ssm: Annotated[ - bool, + Optional[bool], typer.Option( - '-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager', - help='Skip backing up storagemanager directory.' + '-ssm', '--skip-storage-manager', + help='Skip backing up storagemanager directory.', + show_default=False ) - ] = False, + ] = None, q: Annotated[ - bool, + Optional[bool], typer.Option( - '-q/-no-q', '--quiet/--no-quiet', - help='Silence verbose copy command outputs.' + '-q', '--quiet', + help='Silence verbose copy command outputs.', + show_default=False ) - ] = False, - list: Annotated[ - bool, + ] = None, + li: Annotated[ + Optional[bool], typer.Option( '-li', '--list', - help='List backups.' + help='List backups.', + show_default=False ) - ] = False, + ] = None, ): """Columnstore DBRM Backup.""" diff --git a/cmapi/mcs_cluster_tool/helpers.py b/cmapi/mcs_cluster_tool/helpers.py index 5aceba756..442cdb19c 100644 --- a/cmapi/mcs_cluster_tool/helpers.py +++ b/cmapi/mcs_cluster_tool/helpers.py @@ -1,22 +1,27 @@ """Module with helper functions for mcs cli tool.""" -from typing import Union +from typing import Optional, Union -def cook_sh_arg(arg_name: str, value:Union[str, int, bool]) -> str: +def cook_sh_arg(arg_name: str, value: Union[str, int, bool]) -> Optional[str]: """Convert argument and and value from function locals to bash argument. :param arg_name: function argument name :type arg_name: str :param value: function argument value :type value: Union[str, int, bool] - :return: bash argument string - :rtype: str + :return: bash argument string or None + :rtype: Optional[str] """ # skip "arguments" list and Typer ctx variables from local scope if arg_name in ('arguments', 'ctx'): return None - # skip args that have empty string as value - if value == '': + # skip args that have empty string or None as value + # Condition below could be "not value", but I prefer to be explicit + # and check for empty string and None to show that it's different cases: + # empty string means that user passed empty string as value + # and None means that user didn't pass anything and our internal None + # applies + if value == '' or value is None: return None if '_' in arg_name: arg_name = arg_name.replace('_', '-') diff --git a/cmapi/mcs_cluster_tool/restore_commands.py b/cmapi/mcs_cluster_tool/restore_commands.py index 84bbdd195..dcd897b50 100644 --- a/cmapi/mcs_cluster_tool/restore_commands.py +++ b/cmapi/mcs_cluster_tool/restore_commands.py @@ -1,6 +1,7 @@ """Typer application for restore Columnstore data.""" import logging import sys +from typing import Optional import typer from typing_extensions import Annotated @@ -155,29 +156,31 @@ def restore( ) ] = 4, ha: Annotated[ - bool, + Optional[bool], typer.Option( - '-ha/-no-ha', '--highavilability/--no-highavilability', + '-ha', '--highavilability', help=( 'Flag for high available systems (meaning shared storage ' 'exists supporting the topology so that each node sees ' 'all data)' - ) + ), + show_default=False ) - ] = False, + ] = None, cont: Annotated[ - bool, + Optional[bool], typer.Option( - '-cont/-no-cont', '--continue/--no-continue', + '-cont', '--continue', help=( 'This acknowledges data in your --new_bucket is ok to delete ' 'when restoring S3. When set to true skips the enforcement ' 'that new_bucket should be empty prior to starting a restore.' - ) + ), + show_default=False ) - ] = False, + ] = None, f: Annotated[ - str, + Optional[str], typer.Option( '-f', '--config-file', help=( @@ -186,29 +189,30 @@ def restore( ), show_default=False ) - ] = '', + ] = None, smdb: Annotated[ - bool, + Optional[bool], typer.Option( - '-smdb/-no-smdb', '--skip-mariadb-backup/--no-skip-mariadb-backup', + '-smdb', '--skip-mariadb-backup', help=( 'Skip restoring mariadb server via mariadb-backup - ideal for ' 'only restoring columnstore.' - ) + ), + show_default=False ) - ] = False, + ] = None, sb: Annotated[ - bool, + Optional[bool], typer.Option( - '-sb/-no-sb', '--skip-bucket-data/--no-skip-bucket-data', + '-sb', '--skip-bucket-data', help=( 'Skip restoring columnstore data in the bucket - ideal if ' 'looking to only restore mariadb server.' ) ) - ] = False, + ] = None, m: Annotated[ - str, + Optional[str], typer.Option( '-m', '--mode', help=( @@ -217,11 +221,12 @@ def restore( 'machine that has read-only mounts associated with ' 'columnstore/mariadb\n' ), - hidden=True + hidden=True, + show_default='direct' ) - ] = 'direct', + ] = None, c: Annotated[ - str, + Optional[str], typer.Option( '-c', '--compress', help=( @@ -230,28 +235,31 @@ def restore( ), show_default=False ) - ] = '', + ] = None, q: Annotated[ - bool, + Optional[bool], typer.Option( - '-q/-no-q', '--quiet/--no-quiet', - help='Silence verbose copy command outputs.' + '-q', '--quiet', + help='Silence verbose copy command outputs.', + show_default=False ) - ] = False, + ] = None, nv_ssl: Annotated[ - bool, + Optional[bool], typer.Option( - '-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl', - help='Skips verifying ssl certs, useful for onpremise s3 storage.' + '-nv-ssl','--no-verify-ssl', + help='Skips verifying ssl certs, useful for onpremise s3 storage.', + show_default=False, ) - ] = False, - list: Annotated[ - bool, + ] = None, + li: Annotated[ + Optional[bool], typer.Option( '-li', '--list', - help='List backups.' + help='List backups.', + show_default=False ) - ] = False + ] = None ): """Restore Columnstore (and/or MariaDB) data.""" @@ -291,37 +299,41 @@ def dbrm_restore( ) ] = '', ns: Annotated[ - bool, + Optional[bool], typer.Option( '-ns', '--no-start', help=( 'Do not attempt columnstore startup post dbrm_restore.' - ) + ), + show_default=False ) - ] = False, + ] = None, sdbk: Annotated[ - bool, + Optional[bool], typer.Option( - '-sdbk/-no-sdbk', '--skip-dbrm-backup/--no-skip-dbrm-backup', + '-sdbk', '--skip-dbrm-backup', help=( 'Skip backing up dbrms before restoring.' - ) + ), + show_default=False ) - ] = True, + ] = None, ssm: Annotated[ - bool, + Optional[bool], typer.Option( - '-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager', - help='Skip backing up storagemanager directory.' + '-ssm', '--skip-storage-manager', + help='Skip backing up storagemanager directory.', + show_default=False ) - ] = True, - list: Annotated[ + ] = None, + li: Annotated[ bool, typer.Option( '-li', '--list', - help='List backups.' + help='List backups.', + show_default=False ) - ] = False + ] = None ): """Restore Columnstore DBRM data."""