diff --git a/cmapi/cmapi_server/handlers/cej.py b/cmapi/cmapi_server/handlers/cej.py index 0a66cc8ab..afa2248f6 100644 --- a/cmapi/cmapi_server/handlers/cej.py +++ b/cmapi/cmapi_server/handlers/cej.py @@ -10,9 +10,7 @@ from shutil import copyfile from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives import padding -from cmapi_server.constants import ( - MCS_DATA_PATH, MCS_SECRETS_FILENAME, MCS_SECRETS_FILE_PATH -) +from cmapi_server.constants import MCS_DATA_PATH, MCS_SECRETS_FILENAME from cmapi_server.exceptions import CEJError @@ -26,16 +24,19 @@ class CEJPasswordHandler(): """Handler for CrossEngineSupport password decryption.""" @classmethod - def secretsfile_exists(cls) -> bool: - """Check the .secrets file in MCS_SECRETS_FILE_PATH. + def secretsfile_exists(cls, directory: str = MCS_DATA_PATH) -> bool: + """Check the .secrets file in directory. Default MCS_SECRETS_FILE_PATH. + :param directory: path to the directory with .secrets file + :type directory: str, optional :return: True if file exists and not empty. :rtype: bool """ + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) try: if ( - os.path.isfile(MCS_SECRETS_FILE_PATH) and - os.path.getsize(MCS_SECRETS_FILE_PATH) > 0 + os.path.isfile(secrets_file_path) and + os.path.getsize(secrets_file_path) > 0 ): return True except Exception: @@ -49,40 +50,48 @@ class CEJPasswordHandler(): return False @classmethod - def get_secrets_json(cls) -> dict: + def get_secrets_json(cls, directory: str = MCS_DATA_PATH) -> dict: """Get json from .secrets file. - :raises CEJError: on empty\corrupted\wrong format .secrets file + :param directory: path to the directory with .secrets file + :type directory: str, optional :return: json from .secrets file :rtype: dict + :raises CEJError: on empty\corrupted\wrong format .secrets file """ - if not cls.secretsfile_exists(): - raise CEJError(f'{MCS_SECRETS_FILE_PATH} file does not exist.') - with open(MCS_SECRETS_FILE_PATH) as secrets_file: + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) + if not cls.secretsfile_exists(directory=directory): + raise CEJError(f'{secrets_file_path} file does not exist.') + with open(secrets_file_path) as secrets_file: try: secrets_json = json.load(secrets_file) except Exception: logging.error( 'Something went wrong while loading json from ' - f'{MCS_SECRETS_FILE_PATH}', + f'{secrets_file_path}', exc_info=True ) raise CEJError( - f'Looks like file {MCS_SECRETS_FILE_PATH} is corrupted or' + f'Looks like file {secrets_file_path} is corrupted or' 'has wrong format.' ) from None return secrets_json @classmethod - def decrypt_password(cls, enc_data: str) -> str: + def decrypt_password( + cls, enc_data: str, directory: str = MCS_DATA_PATH + ) -> str: """Decrypt CEJ password if needed. + :param directory: path to the directory with .secrets file + :type directory: str, optional :param enc_data: encrypted initialization vector + password in hex str :type enc_data: str :return: decrypted CEJ password :rtype: str """ - if not cls.secretsfile_exists(): + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) + if not cls.secretsfile_exists(directory=directory): logging.warning('Unencrypted CrossEngineSupport password used.') return enc_data @@ -96,18 +105,18 @@ class CEJPasswordHandler(): 'Non-hexadecimal number found in encrypted CEJ password.' ) from value_error - secrets_json = cls.get_secrets_json() + secrets_json = cls.get_secrets_json(directory=directory) encryption_key_hex = secrets_json.get('encryption_key', None) if not encryption_key_hex: raise CEJError( - f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}' + f'Empty "encryption key" found in {secrets_file_path}' ) try: encryption_key = bytes.fromhex(encryption_key_hex) except ValueError as value_error: raise CEJError( 'Non-hexadecimal number found in encryption key from ' - f'{MCS_SECRETS_FILE_PATH} file.' + f'{secrets_file_path} file.' ) from value_error cipher = Cipher( algorithms.AES(encryption_key), @@ -125,21 +134,34 @@ class CEJPasswordHandler(): return passwd_bytes.decode() @classmethod - def encrypt_password(cls, passwd: str) -> str: - iv = os.urandom(size=AES_IV_BIN_SIZE) + def encrypt_password( + cls, passwd: str, directory: str = MCS_DATA_PATH + ) -> str: + """Encrypt CEJ password. - secrets_json = cls.get_secrets_json() + :param directory: path to the directory with .secrets file + :type directory: str, optional + :param passwd: CEJ password + :type passwd: str + :return: encrypted CEJ password in uppercase hex format + :rtype: str + """ + + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) + iv = os.urandom(AES_IV_BIN_SIZE) + + secrets_json = cls.get_secrets_json(directory=directory) encryption_key_hex = secrets_json.get('encryption_key') if not encryption_key_hex: raise CEJError( - f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}' + f'Empty "encryption key" found in {secrets_file_path}' ) try: encryption_key = bytes.fromhex(encryption_key_hex) except ValueError as value_error: raise CEJError( 'Non-hexadecimal number found in encryption key from ' - f'{MCS_SECRETS_FILE_PATH} file.' + f'{secrets_file_path} file.' ) from value_error cipher = Cipher( algorithms.AES(encryption_key), @@ -151,8 +173,8 @@ class CEJPasswordHandler(): padded_data = padder.update(passwd.encode()) + padder.finalize() encrypted_data = encryptor.update(padded_data) + encryptor.finalize() - - return iv + encrypted_data + encrypted_passwd_bytes = iv + encrypted_data + return encrypted_passwd_bytes.hex().upper() @classmethod def generate_secrets_data(cls) -> dict: @@ -161,8 +183,8 @@ class CEJPasswordHandler(): :return: secrets data :rtype: dict """ - key_length = algorithms.AES256.key_size // 8 - encryption_key = os.urandom(size=key_length) + key_length = 32 # AES256 key_size + encryption_key = os.urandom(key_length) encryption_key_hex = binascii.hexlify(encryption_key).decode() secrets_dict = { 'description': 'Columnstore CrossEngineSupport password encryption/decryption key', @@ -174,11 +196,13 @@ class CEJPasswordHandler(): @classmethod def save_secrets( - cls, secrets: dict, filepath: str = MCS_SECRETS_FILE_PATH, + cls, secrets: dict, directory: str = MCS_DATA_PATH, owner: str = 'mysql' ) -> None: """Write secrets to .secrets file. + :param directory: path to the directory with .secrets file + :type directory: str, optional :param secrets: secrets dict :type secrets: dict :param filepath: path to the .secrets file @@ -186,29 +210,43 @@ class CEJPasswordHandler(): :param owner: owner of the file :type owner: str, optional """ - if cls.secretsfile_exists(): - copyfile( - filepath, - os.path.join( - os.path.dirname(filepath), - f'{os.path.basename(filepath)}.cmapi.save' + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) + if cls.secretsfile_exists(directory=directory): + if cls.get_secrets_json(directory=directory) != secrets: + copyfile( + secrets_file_path, + os.path.join( + directory, + f'{MCS_SECRETS_FILENAME}.cmapi.save' + ) ) - ) + logging.warning( + f'Backup of {secrets_file_path} file created.' + ) + else: + logging.debug( + f'No changes in {secrets_file_path} file detected.' + ) + return try: with open( - MCS_SECRETS_FILE_PATH, 'w', encoding='utf-8' + secrets_file_path, 'w', encoding='utf-8' ) as secrets_file: json.dump(secrets, secrets_file) except Exception as exc: raise CEJError(f'Write to .secrets file failed.') from exc try: - os.chmod(MCS_SECRETS_FILE_PATH, stat.S_IRUSR) + os.chmod(secrets_file_path, stat.S_IRUSR) userinfo = pwd.getpwnam(owner) - os.chown(MCS_SECRETS_FILE_PATH, userinfo.pw_uid, userinfo.pw_gid) - logging.debug(f'Permissions of .secrets file set to {owner}:read.') - logging.debug(f'Ownership of .secrets file given to {owner}.') + os.chown(secrets_file_path, userinfo.pw_uid, userinfo.pw_gid) + logging.debug( + f'Permissions of {secrets_file_path} file set to {owner}:read.' + ) + logging.debug( + f'Ownership of {secrets_file_path} file given to {owner}.' + ) except Exception as exc: raise CEJError( f'Failed to set permissions or ownership for .secrets file.' diff --git a/cmapi/cmapi_server/helpers.py b/cmapi/cmapi_server/helpers.py index ca6860e53..04845d8bb 100644 --- a/cmapi/cmapi_server/helpers.py +++ b/cmapi/cmapi_server/helpers.py @@ -344,6 +344,10 @@ def broadcast_new_config( if distribute_secrets: # TODO: do not restart cluster when put xml config only with + # distribute secrets + if not CEJPasswordHandler.secretsfile_exists(): + secrets_dict = CEJPasswordHandler.generate_secrets_data() + CEJPasswordHandler.save_secrets(secrets=secrets_dict) secrets = CEJPasswordHandler.get_secrets_json() body['secrets'] = secrets @@ -798,7 +802,7 @@ def get_cej_info(config_root): 'Columnstore.xml has an empty CrossEngineSupport.Password tag' ) - if CEJPasswordHandler.secretsfile_exists(): + if CEJPasswordHandler.secretsfile_exists() and cej_password: cej_password = CEJPasswordHandler.decrypt_password(cej_password) return cej_host, cej_port, cej_username, cej_password diff --git a/cmapi/mcs_cluster_tool/README.md b/cmapi/mcs_cluster_tool/README.md index 57088117f..73446aa9d 100644 --- a/cmapi/mcs_cluster_tool/README.md +++ b/cmapi/mcs_cluster_tool/README.md @@ -18,6 +18,8 @@ $ mcs [OPTIONS] COMMAND [ARGS]... * `dbrm_backup`: Columnstore DBRM Backup. * `restore`: Restore Columnstore (and/or MariaDB) data. * `dbrm_restore`: Restore Columnstore DBRM data. +* `cskeys`: Generates a random AES encryption key and init vector and writes them to disk. +* `cspasswd`: Encrypt a Columnstore plaintext password using the encryption key in the key file. * `help-all`: Show help for all commands in man page style. * `status`: Get status information. * `stop`: Stop the Columnstore cluster. @@ -162,6 +164,50 @@ $ mcs dbrm_restore [OPTIONS] * `-li, --list`: List backups. * `--help`: Show this message and exit. +## `mcs cskeys` + +This utility generates a random AES encryption key and init vector +and writes them to disk. The data is written to the file '.secrets', +in the specified directory. The key and init vector are used by +the utility 'cspasswd' to encrypt passwords used in Columnstore +configuration files, as well as by Columnstore itself to decrypt the +passwords. + +WARNING: Re-creating the file invalidates all existing encrypted +passwords in the configuration files. + +**Usage**: + +```console +$ mcs cskeys [OPTIONS] [DIRECTORY] +``` + +**Arguments**: + +* `[DIRECTORY]`: The directory where to store the file in. [default: /var/lib/columnstore] + +**Options**: + +* `-u, --user TEXT`: Designate the owner of the generated file. [default: mysql] +* `--help`: Show this message and exit. + +## `mcs cspasswd` + +Encrypt a Columnstore plaintext password using the encryption key in +the key file. + +**Usage**: + +```console +$ mcs cspasswd [OPTIONS] +``` + +**Options**: + +* `--password TEXT`: Password to encrypt/decrypt [required] +* `--decrypt`: Decrypt an encrypted password instead. +* `--help`: Show this message and exit. + ## `mcs help-all` Show help for all commands in man page style. diff --git a/cmapi/mcs_cluster_tool/README_DEV.md b/cmapi/mcs_cluster_tool/README_DEV.md index fbf0ebebf..d69393dea 100644 --- a/cmapi/mcs_cluster_tool/README_DEV.md +++ b/cmapi/mcs_cluster_tool/README_DEV.md @@ -7,6 +7,14 @@ ```bash typer mcs_cluster_tool/__main__.py utils docs --name mcs --output README.md ``` + Optionally could be generated from installed package. + ```bash + PYTHONPATH="/usr/share/columnstore/cmapi:/usr/share/columnstore/cmapi/deps" /usr/share/columnstore/cmapi/python/bin/python3 -m typer /usr/share/columnstore/cmapi/mcs_cluster_tool/__main__.py utils docs --name mcs --output ~/README.md + ``` +- dependencies for gem build (RHEL example) + ```bash + sudo dnf install make gcc redhat-rpm-config -y + ``` - install `md2man` (for now it's the only one tool that make convertation without any issues) ```bash sudo yum install -y ruby ruby-devel @@ -14,6 +22,6 @@ ``` - convert to perfect `.roff` file (`man` page) ```bash - md2man README.md > mcs.1 + md2man-roff README.md > mcs.1 ``` - enjoy =) \ No newline at end of file diff --git a/cmapi/mcs_cluster_tool/__main__.py b/cmapi/mcs_cluster_tool/__main__.py index 95072183d..b7bfdbadf 100644 --- a/cmapi/mcs_cluster_tool/__main__.py +++ b/cmapi/mcs_cluster_tool/__main__.py @@ -36,9 +36,16 @@ app.command( app.command( 'dbrm_restore', rich_help_panel='Tools commands' )(restore_commands.dbrm_restore) -app.command('cskeys', rich_help_panel='Tools commands')(tools_commands.cskeys) app.command( - 'cspasswd', rich_help_panel='Tools commands' + 'cskeys', rich_help_panel='Tools commands', + short_help=( + 'Generates a random AES encryption key and init vector and writes ' + 'them to disk.' + ) +)(tools_commands.cskeys) +app.command( + 'cspasswd', rich_help_panel='Tools commands', + short_help='Encrypt a Columnstore plaintext password.' )(tools_commands.cspasswd) diff --git a/cmapi/mcs_cluster_tool/decorators.py b/cmapi/mcs_cluster_tool/decorators.py index c8ac9499a..e26c6fac8 100644 --- a/cmapi/mcs_cluster_tool/decorators.py +++ b/cmapi/mcs_cluster_tool/decorators.py @@ -25,11 +25,16 @@ def handle_output(func): except typer.BadParameter as err: logger.error('Bad command line parameter.') raise err + except typer.Exit as err: # if some command used typer.Exit + #TODO: think about universal protocol to return json data and + # plain text results. + return_code = err.exit_code except Exception: logger.error( 'Undefined error during command execution', exc_info=True ) typer.echo('Unknown error, check the log file.', err=True) + raise typer.Exit(return_code) return wrapper diff --git a/cmapi/mcs_cluster_tool/mcs.1 b/cmapi/mcs_cluster_tool/mcs.1 index c465f4a71..724161f7c 100644 --- a/cmapi/mcs_cluster_tool/mcs.1 +++ b/cmapi/mcs_cluster_tool/mcs.1 @@ -27,6 +27,10 @@ $ mcs [OPTIONS] COMMAND [ARGS]... .IP \(bu 2 \fB\fCdbrm_restore\fR: Restore Columnstore DBRM data. .IP \(bu 2 +\fB\fCcskeys\fR: Generates a random AES encryption key and init vector and writes them to disk. +.IP \(bu 2 +\fB\fCcspasswd\fR: Encrypt a Columnstore plaintext password using the encryption key in the key file. +.IP \(bu 2 \fB\fChelp\-all\fR: Show help for all commands in man page style. .IP \(bu 2 \fB\fCstatus\fR: Get status information. @@ -252,6 +256,61 @@ $ mcs dbrm_restore [OPTIONS] .IP \(bu 2 \fB\fC\-\-help\fR: Show this message and exit. .RE +.SH \fB\fCmcs cskeys\fR +.PP +This utility generates a random AES encryption key and init vector +and writes them to disk. The data is written to the file \[aq]\&.secrets\[aq], +in the specified directory. The key and init vector are used by +the utility \[aq]cspasswd\[aq] to encrypt passwords used in Columnstore +configuration files, as well as by Columnstore itself to decrypt the +passwords. +.PP +WARNING: Re\-creating the file invalidates all existing encrypted +passwords in the configuration files. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cskeys [OPTIONS] [DIRECTORY] +.fi +.RE +.PP +\fBArguments\fP: +.RS +.IP \(bu 2 +\fB\fC[DIRECTORY]\fR: The directory where to store the file in. [default: /var/lib/columnstore] +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-u, \-\-user TEXT\fR: Designate the owner of the generated file. [default: mysql] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs cspasswd\fR +.PP +Encrypt a Columnstore plaintext password using the encryption key in +the key file. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cspasswd [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-password TEXT\fR: Password to encrypt/decrypt [required] +.IP \(bu 2 +\fB\fC\-\-decrypt\fR: Decrypt an encrypted password instead. +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE .SH \fB\fCmcs help\-all\fR .PP Show help for all commands in man page style. diff --git a/cmapi/mcs_cluster_tool/tools_commands.py b/cmapi/mcs_cluster_tool/tools_commands.py index 4b309b6ec..0e4377688 100644 --- a/cmapi/mcs_cluster_tool/tools_commands.py +++ b/cmapi/mcs_cluster_tool/tools_commands.py @@ -5,7 +5,9 @@ import typer from typing_extensions import Annotated -from cmapi_server.constants import MCS_SECRETS_FILE_PATH +from cmapi_server.constants import ( + MCS_DATA_PATH, MCS_SECRETS_FILENAME +) from cmapi_server.exceptions import CEJError from cmapi_server.handlers.cej import CEJPasswordHandler from mcs_cluster_tool.decorators import handle_output @@ -18,22 +20,33 @@ logger = logging.getLogger('mcs_cli') @handle_output def cskeys( - filepath: Annotated[ + user: Annotated[ str, typer.Option( - '-f', '--filepath', - help='Path to the output file', - ) - ] = MCS_SECRETS_FILE_PATH, - username: Annotated[ - str, - typer.Option( - '-u', '--username', - help='Username for the key', + '-u', '--user', + help='Designate the owner of the generated file.', ) ] = 'mysql', + directory: Annotated[ + str, + typer.Argument( + help='The directory where to store the file in.', + ) + ] = MCS_DATA_PATH ): - if CEJPasswordHandler().secretsfile_exists(): + """ + This utility generates a random AES encryption key and init vector + and writes them to disk. The data is written to the file '.secrets', + in the specified directory. The key and init vector are used by + the utility 'cspasswd' to encrypt passwords used in Columnstore + configuration files, as well as by Columnstore itself to decrypt the + passwords. + + WARNING: Re-creating the file invalidates all existing encrypted + passwords in the configuration files. + """ + filepath = os.path.join(directory, MCS_SECRETS_FILENAME) + if CEJPasswordHandler().secretsfile_exists(directory=directory): typer.echo( ( f'Secrets file "{filepath}" already exists. ' @@ -44,14 +57,18 @@ def cskeys( raise typer.Exit(code=1) elif not os.path.exists(os.path.dirname(filepath)): typer.echo( - f'Directory "{os.path.dirname(filepath)}" does not exist.', + f'Directory "{directory}" does not exist.', color='red' ) raise typer.Exit(code=1) new_secrets_data = CEJPasswordHandler().generate_secrets_data() try: - CEJPasswordHandler().save_secrets(new_secrets_data, owner=username) + CEJPasswordHandler().save_secrets( + new_secrets_data, owner=user, directory=directory + ) + typer.echo(f'Permissions of "{filepath}" set to owner:read.') + typer.echo(f'Ownership of "{filepath}" given to {user}.') except CEJError as cej_error: typer.echo(cej_error.message, color='red') raise typer.Exit(code=2) @@ -71,10 +88,14 @@ def cspasswd( bool, typer.Option( '--decrypt', - help='Decrypt the provided password', + help='Decrypt an encrypted password instead.', ) ] = False ): + """ + Encrypt a Columnstore plaintext password using the encryption key in + the key file. + """ if decrypt: try: decrypted_password = CEJPasswordHandler().decrypt_password(