feat(cmapi): MCOL-5019: review fixes.

[fix] CEJPasswordHandler class methods to use directory for cskeys file
[fix] CEJPasswordHandler.encrypt_password to return password in hex format
[fix] CEJPasswordHandler key_length
[fix] CEJPasswordHandler os.urandom call typo
[upd] mcs cli README.md and man page
[upd] mcs cli README_DEV.md
[fix] mcs_cluster_tool/decorators.py to handle typer.Exit exception
[add] various docstrings

cmapi/cmapi_server/handlers/cej.py

@@ -10,9 +10,7 @@ from shutil import copyfile
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.primitives import padding
 
-from cmapi_server.constants import (
+from cmapi_server.constants import MCS_DATA_PATH, MCS_SECRETS_FILENAME
-    MCS_DATA_PATH, MCS_SECRETS_FILENAME, MCS_SECRETS_FILE_PATH
-)
 from cmapi_server.exceptions import CEJError
 
 
@@ -26,16 +24,19 @@ class CEJPasswordHandler():
     """Handler for CrossEngineSupport password decryption."""
 
     @classmethod
-    def secretsfile_exists(cls) -> bool:
+    def secretsfile_exists(cls, directory: str = MCS_DATA_PATH) -> bool:
-        """Check the .secrets file in MCS_SECRETS_FILE_PATH.
+        """Check the .secrets file in directory. Default MCS_SECRETS_FILE_PATH.
 
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
         :return: True if file exists and not empty.
         :rtype: bool
         """
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
         try:
             if (
-                os.path.isfile(MCS_SECRETS_FILE_PATH) and
+                os.path.isfile(secrets_file_path) and
-                os.path.getsize(MCS_SECRETS_FILE_PATH) > 0
+                os.path.getsize(secrets_file_path) > 0
             ):
                 return True
         except Exception:
@@ -49,40 +50,48 @@ class CEJPasswordHandler():
         return False
 
     @classmethod
-    def get_secrets_json(cls) -> dict:
+    def get_secrets_json(cls, directory: str = MCS_DATA_PATH) -> dict:
         """Get json from .secrets file.
 
-        :raises CEJError: on empty\corrupted\wrong format .secrets file
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
         :return: json from .secrets file
         :rtype: dict
+        :raises CEJError: on empty\corrupted\wrong format .secrets file
         """
-        if not cls.secretsfile_exists():
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
-            raise CEJError(f'{MCS_SECRETS_FILE_PATH} file does not exist.')
+        if not cls.secretsfile_exists(directory=directory):
-        with open(MCS_SECRETS_FILE_PATH) as secrets_file:
+            raise CEJError(f'{secrets_file_path} file does not exist.')
+        with open(secrets_file_path) as secrets_file:
             try:
                 secrets_json = json.load(secrets_file)
             except Exception:
                 logging.error(
                     'Something went wrong while loading json from '
-                    f'{MCS_SECRETS_FILE_PATH}',
+                    f'{secrets_file_path}',
                     exc_info=True
                 )
                 raise CEJError(
-                    f'Looks like file {MCS_SECRETS_FILE_PATH} is corrupted or'
+                    f'Looks like file {secrets_file_path} is corrupted or'
                     'has wrong format.'
                 ) from None
             return secrets_json
 
     @classmethod
-    def decrypt_password(cls, enc_data: str) -> str:
+    def decrypt_password(
+        cls, enc_data: str, directory: str = MCS_DATA_PATH
+    ) -> str:
         """Decrypt CEJ password if needed.
 
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
         :param enc_data: encrypted initialization vector + password in hex str
         :type enc_data: str
         :return: decrypted CEJ password
         :rtype: str
         """
-        if not cls.secretsfile_exists():
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
+        if not cls.secretsfile_exists(directory=directory):
             logging.warning('Unencrypted CrossEngineSupport password used.')
             return enc_data
 
@@ -96,18 +105,18 @@ class CEJPasswordHandler():
                 'Non-hexadecimal number found in encrypted CEJ password.'
             ) from value_error
 
-        secrets_json = cls.get_secrets_json()
+        secrets_json = cls.get_secrets_json(directory=directory)
         encryption_key_hex = secrets_json.get('encryption_key', None)
         if not encryption_key_hex:
             raise CEJError(
-                f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}'
+                f'Empty "encryption key" found in {secrets_file_path}'
             )
         try:
             encryption_key = bytes.fromhex(encryption_key_hex)
         except ValueError as value_error:
             raise CEJError(
                 'Non-hexadecimal number found in encryption key from '
-                f'{MCS_SECRETS_FILE_PATH} file.'
+                f'{secrets_file_path} file.'
             ) from value_error
         cipher = Cipher(
             algorithms.AES(encryption_key),
@@ -125,21 +134,34 @@ class CEJPasswordHandler():
         return passwd_bytes.decode()
 
     @classmethod
-    def encrypt_password(cls, passwd: str) -> str:
+    def encrypt_password(
-        iv = os.urandom(size=AES_IV_BIN_SIZE)
+        cls, passwd: str, directory: str = MCS_DATA_PATH
+    ) -> str:
+        """Encrypt CEJ password.
 
-        secrets_json = cls.get_secrets_json()
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
+        :param passwd: CEJ password
+        :type passwd: str
+        :return: encrypted CEJ password in uppercase hex format
+        :rtype: str
+        """
+
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
+        iv = os.urandom(AES_IV_BIN_SIZE)
+
+        secrets_json = cls.get_secrets_json(directory=directory)
         encryption_key_hex = secrets_json.get('encryption_key')
         if not encryption_key_hex:
             raise CEJError(
-                f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}'
+                f'Empty "encryption key" found in {secrets_file_path}'
             )
         try:
             encryption_key = bytes.fromhex(encryption_key_hex)
         except ValueError as value_error:
             raise CEJError(
                 'Non-hexadecimal number found in encryption key from '
-                f'{MCS_SECRETS_FILE_PATH} file.'
+                f'{secrets_file_path} file.'
             ) from value_error
         cipher = Cipher(
             algorithms.AES(encryption_key),
@@ -151,8 +173,8 @@ class CEJPasswordHandler():
         padded_data = padder.update(passwd.encode()) + padder.finalize()
 
         encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
-
+        encrypted_passwd_bytes = iv + encrypted_data
-        return iv + encrypted_data
+        return encrypted_passwd_bytes.hex().upper()
 
     @classmethod
     def generate_secrets_data(cls) -> dict:
@@ -161,8 +183,8 @@ class CEJPasswordHandler():
         :return: secrets data
         :rtype: dict
         """
-        key_length = algorithms.AES256.key_size // 8
+        key_length = 32  # AES256 key_size
-        encryption_key = os.urandom(size=key_length)
+        encryption_key = os.urandom(key_length)
         encryption_key_hex = binascii.hexlify(encryption_key).decode()
         secrets_dict = {
             'description': 'Columnstore CrossEngineSupport password encryption/decryption key',
@@ -174,11 +196,13 @@ class CEJPasswordHandler():
 
     @classmethod
     def save_secrets(
-        cls, secrets: dict, filepath: str = MCS_SECRETS_FILE_PATH,
+        cls, secrets: dict, directory: str = MCS_DATA_PATH,
         owner: str = 'mysql'
     ) -> None:
         """Write secrets to .secrets file.
 
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
         :param secrets: secrets dict
         :type secrets: dict
         :param filepath: path to the .secrets file
@@ -186,29 +210,43 @@ class CEJPasswordHandler():
         :param owner: owner of the file
         :type owner: str, optional
         """
-        if cls.secretsfile_exists():
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
-            copyfile(
+        if cls.secretsfile_exists(directory=directory):
-                filepath,
+            if cls.get_secrets_json(directory=directory) != secrets:
-                os.path.join(
+                copyfile(
-                    os.path.dirname(filepath),
+                    secrets_file_path,
-                    f'{os.path.basename(filepath)}.cmapi.save'
+                    os.path.join(
+                        directory,
+                        f'{MCS_SECRETS_FILENAME}.cmapi.save'
+                    )
                 )
-            )
+                logging.warning(
+                    f'Backup of {secrets_file_path} file created.'
+                )
+            else:
+                logging.debug(
+                    f'No changes in {secrets_file_path} file detected.'
+                )
+                return
 
         try:
             with open(
-                MCS_SECRETS_FILE_PATH, 'w', encoding='utf-8'
+                secrets_file_path, 'w', encoding='utf-8'
             ) as secrets_file:
                 json.dump(secrets, secrets_file)
         except Exception as exc:
             raise CEJError(f'Write to .secrets file failed.') from exc
 
         try:
-            os.chmod(MCS_SECRETS_FILE_PATH, stat.S_IRUSR)
+            os.chmod(secrets_file_path, stat.S_IRUSR)
             userinfo = pwd.getpwnam(owner)
-            os.chown(MCS_SECRETS_FILE_PATH, userinfo.pw_uid, userinfo.pw_gid)
+            os.chown(secrets_file_path, userinfo.pw_uid, userinfo.pw_gid)
-            logging.debug(f'Permissions of .secrets file set to {owner}:read.')
+            logging.debug(
-            logging.debug(f'Ownership of .secrets file given to {owner}.')
+                f'Permissions of {secrets_file_path} file set to {owner}:read.'
+            )
+            logging.debug(
+                f'Ownership of {secrets_file_path} file given to {owner}.'
+            )
         except Exception as exc:
             raise CEJError(
                 f'Failed to set permissions or ownership for .secrets file.'

cmapi/cmapi_server/helpers.py

@@ -344,6 +344,10 @@ def broadcast_new_config(
 
     if distribute_secrets:
         # TODO: do not restart cluster when put xml config only with
+        #       distribute secrets
+        if not CEJPasswordHandler.secretsfile_exists():
+            secrets_dict = CEJPasswordHandler.generate_secrets_data()
+            CEJPasswordHandler.save_secrets(secrets=secrets_dict)
         secrets = CEJPasswordHandler.get_secrets_json()
         body['secrets'] = secrets
 
@@ -798,7 +802,7 @@ def get_cej_info(config_root):
             'Columnstore.xml has an empty CrossEngineSupport.Password tag'
         )
 
-    if CEJPasswordHandler.secretsfile_exists():
+    if CEJPasswordHandler.secretsfile_exists() and cej_password:
         cej_password = CEJPasswordHandler.decrypt_password(cej_password)
 
     return cej_host, cej_port, cej_username, cej_password

cmapi/mcs_cluster_tool/README.md

@@ -18,6 +18,8 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 * `dbrm_backup`: Columnstore DBRM Backup.
 * `restore`: Restore Columnstore (and/or MariaDB) data.
 * `dbrm_restore`: Restore Columnstore DBRM data.
+* `cskeys`: Generates a random AES encryption key and init vector and writes them to disk.
+* `cspasswd`: Encrypt a Columnstore plaintext password using the encryption key in the key file.
 * `help-all`: Show help for all commands in man page style.
 * `status`: Get status information.
 * `stop`: Stop the Columnstore cluster.
@@ -162,6 +164,50 @@ $ mcs dbrm_restore [OPTIONS]
 * `-li, --list`: List backups.
 * `--help`: Show this message and exit.
 
+## `mcs cskeys`
+
+This utility generates a random AES encryption key and init vector
+and writes them to disk. The data is written to the file '.secrets',
+in the specified directory. The key and init vector are used by
+the utility 'cspasswd' to encrypt passwords used in Columnstore
+configuration files, as well as by Columnstore itself to decrypt the
+passwords.
+
+WARNING: Re-creating the file invalidates all existing encrypted
+passwords in the configuration files.
+
+**Usage**:
+
+```console
+$ mcs cskeys [OPTIONS] [DIRECTORY]
+```
+
+**Arguments**:
+
+* `[DIRECTORY]`: The directory where to store the file in.  [default: /var/lib/columnstore]
+
+**Options**:
+
+* `-u, --user TEXT`: Designate the owner of the generated file.  [default: mysql]
+* `--help`: Show this message and exit.
+
+## `mcs cspasswd`
+
+Encrypt a Columnstore plaintext password using the encryption key in
+the key file.
+
+**Usage**:
+
+```console
+$ mcs cspasswd [OPTIONS]
+```
+
+**Options**:
+
+* `--password TEXT`: Password to encrypt/decrypt  [required]
+* `--decrypt`: Decrypt an encrypted password instead.
+* `--help`: Show this message and exit.
+
 ## `mcs help-all`
 
 Show help for all commands in man page style.

cmapi/mcs_cluster_tool/README_DEV.md

@@ -7,6 +7,14 @@
     ```bash
     typer mcs_cluster_tool/__main__.py utils docs --name mcs --output README.md
     ```
+    Optionally could be generated from installed package.
+    ```bash
+    PYTHONPATH="/usr/share/columnstore/cmapi:/usr/share/columnstore/cmapi/deps" /usr/share/columnstore/cmapi/python/bin/python3 -m  typer /usr/share/columnstore/cmapi/mcs_cluster_tool/__main__.py utils docs --name mcs --output ~/README.md
+    ```
+- dependencies for gem build (RHEL example)
+    ```bash
+    sudo dnf install make gcc redhat-rpm-config -y
+    ```
 - install `md2man` (for now it's the only one tool that make convertation without any issues)
     ```bash
     sudo yum install -y ruby ruby-devel
@@ -14,6 +22,6 @@
     ```
 - convert to perfect `.roff` file (`man` page)
     ```bash
-    md2man README.md > mcs.1
+    md2man-roff README.md > mcs.1
     ```
 - enjoy =)

cmapi/mcs_cluster_tool/__main__.py

@@ -36,9 +36,16 @@ app.command(
 app.command(
     'dbrm_restore', rich_help_panel='Tools commands'
 )(restore_commands.dbrm_restore)
-app.command('cskeys', rich_help_panel='Tools commands')(tools_commands.cskeys)
 app.command(
-    'cspasswd', rich_help_panel='Tools commands'
+    'cskeys', rich_help_panel='Tools commands',
+    short_help=(
+        'Generates a random AES encryption key and init vector and writes '
+        'them to disk.'
+    )
+)(tools_commands.cskeys)
+app.command(
+    'cspasswd', rich_help_panel='Tools commands',
+    short_help='Encrypt a Columnstore plaintext password.'
 )(tools_commands.cspasswd)
 
 

cmapi/mcs_cluster_tool/decorators.py

@@ -25,11 +25,16 @@ def handle_output(func):
         except typer.BadParameter as err:
             logger.error('Bad command line parameter.')
             raise err
+        except typer.Exit as err:  # if some command used typer.Exit
+            #TODO: think about universal protocol to return json data and
+            #      plain text results.
+            return_code = err.exit_code
         except Exception:
             logger.error(
                 'Undefined error during command execution',
                 exc_info=True
             )
             typer.echo('Unknown error, check the log file.', err=True)
+
         raise typer.Exit(return_code)
     return wrapper

cmapi/mcs_cluster_tool/mcs.1

@@ -27,6 +27,10 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 .IP \(bu 2
 \fB\fCdbrm_restore\fR: Restore Columnstore DBRM data.
 .IP \(bu 2
+\fB\fCcskeys\fR: Generates a random AES encryption key and init vector and writes them to disk.
+.IP \(bu 2
+\fB\fCcspasswd\fR: Encrypt a Columnstore plaintext password using the encryption key in the key file.
+.IP \(bu 2
 \fB\fChelp\-all\fR: Show help for all commands in man page style.
 .IP \(bu 2
 \fB\fCstatus\fR: Get status information.
@@ -252,6 +256,61 @@ $ mcs dbrm_restore [OPTIONS]
 .IP \(bu 2
 \fB\fC\-\-help\fR: Show this message and exit.
 .RE
+.SH \fB\fCmcs cskeys\fR
+.PP
+This utility generates a random AES encryption key and init vector
+and writes them to disk. The data is written to the file \[aq]\&.secrets\[aq],
+in the specified directory. The key and init vector are used by
+the utility \[aq]cspasswd\[aq] to encrypt passwords used in Columnstore
+configuration files, as well as by Columnstore itself to decrypt the
+passwords.
+.PP
+WARNING: Re\-creating the file invalidates all existing encrypted
+passwords in the configuration files.
+.PP
+\fBUsage\fP:
+.PP
+.RS
+.nf
+$ mcs cskeys [OPTIONS] [DIRECTORY]
+.fi
+.RE
+.PP
+\fBArguments\fP:
+.RS
+.IP \(bu 2
+\fB\fC[DIRECTORY]\fR: The directory where to store the file in.  [default: /var/lib/columnstore]
+.RE
+.PP
+\fBOptions\fP:
+.RS
+.IP \(bu 2
+\fB\fC\-u, \-\-user TEXT\fR: Designate the owner of the generated file.  [default: mysql]
+.IP \(bu 2
+\fB\fC\-\-help\fR: Show this message and exit.
+.RE
+.SH \fB\fCmcs cspasswd\fR
+.PP
+Encrypt a Columnstore plaintext password using the encryption key in
+the key file.
+.PP
+\fBUsage\fP:
+.PP
+.RS
+.nf
+$ mcs cspasswd [OPTIONS]
+.fi
+.RE
+.PP
+\fBOptions\fP:
+.RS
+.IP \(bu 2
+\fB\fC\-\-password TEXT\fR: Password to encrypt/decrypt  [required]
+.IP \(bu 2
+\fB\fC\-\-decrypt\fR: Decrypt an encrypted password instead.
+.IP \(bu 2
+\fB\fC\-\-help\fR: Show this message and exit.
+.RE
 .SH \fB\fCmcs help\-all\fR
 .PP
 Show help for all commands in man page style.

cmapi/mcs_cluster_tool/tools_commands.py

@@ -5,7 +5,9 @@ import typer
 from typing_extensions import Annotated
 
 
-from cmapi_server.constants import MCS_SECRETS_FILE_PATH
+from cmapi_server.constants import (
+    MCS_DATA_PATH, MCS_SECRETS_FILENAME
+)
 from cmapi_server.exceptions import CEJError
 from cmapi_server.handlers.cej import CEJPasswordHandler
 from mcs_cluster_tool.decorators import handle_output
@@ -18,22 +20,33 @@ logger = logging.getLogger('mcs_cli')
 
 @handle_output
 def cskeys(
-    filepath: Annotated[
+    user: Annotated[
         str,
         typer.Option(
-            '-f', '--filepath',
+            '-u', '--user',
-            help='Path to the output file',
+            help='Designate the owner of the generated file.',
-        )
-    ] = MCS_SECRETS_FILE_PATH,
-    username: Annotated[
-        str,
-        typer.Option(
-            '-u', '--username',
-            help='Username for the key',
         )
     ] = 'mysql',
+    directory: Annotated[
+        str,
+        typer.Argument(
+            help='The directory where to store the file in.',
+        )
+    ] = MCS_DATA_PATH
 ):
-    if CEJPasswordHandler().secretsfile_exists():
+    """
+    This utility generates a random AES encryption key and init vector
+    and writes them to disk. The data is written to the file '.secrets',
+    in the specified directory. The key and init vector are used by
+    the utility 'cspasswd' to encrypt passwords used in Columnstore
+    configuration files, as well as by Columnstore itself to decrypt the
+    passwords.
+
+    WARNING: Re-creating the file invalidates all existing encrypted
+    passwords in the configuration files.
+    """
+    filepath = os.path.join(directory, MCS_SECRETS_FILENAME)
+    if CEJPasswordHandler().secretsfile_exists(directory=directory):
         typer.echo(
             (
                 f'Secrets file "{filepath}" already exists. '
@@ -44,14 +57,18 @@ def cskeys(
         raise typer.Exit(code=1)
     elif not os.path.exists(os.path.dirname(filepath)):
         typer.echo(
-            f'Directory "{os.path.dirname(filepath)}" does not exist.',
+            f'Directory "{directory}" does not exist.',
             color='red'
         )
         raise typer.Exit(code=1)
 
     new_secrets_data = CEJPasswordHandler().generate_secrets_data()
     try:
-        CEJPasswordHandler().save_secrets(new_secrets_data, owner=username)
+        CEJPasswordHandler().save_secrets(
+            new_secrets_data, owner=user, directory=directory
+        )
+        typer.echo(f'Permissions of "{filepath}" set to owner:read.')
+        typer.echo(f'Ownership of "{filepath}" given to {user}.')
     except CEJError as cej_error:
         typer.echo(cej_error.message, color='red')
         raise typer.Exit(code=2)
@@ -71,10 +88,14 @@ def cspasswd(
         bool,
         typer.Option(
             '--decrypt',
-            help='Decrypt the provided password',
+            help='Decrypt an encrypted password instead.',
         )
     ] = False
 ):
+    """
+    Encrypt a Columnstore plaintext password using the encryption key in
+    the key file.
+    """
     if decrypt:
         try:
             decrypted_password = CEJPasswordHandler().decrypt_password(