feat(cmapi): MCOL-5019: distributing cskeys secrets file, move cskeys and cspasswd functions to mcs cli.

[add] distribute .secrets file to all nodes while adding a new node
[add] encrypt_password, generate_secrets_data, save_secrets to CEJPasswordHandler
[add] tools section to mcs cli tool
[add] mcs_cluster_tool/tools_commands.py file with cskeys and cspasswd commands
[add] cskeys and cspasswd commands to tools section of mcs cli
[mv] backup/restore commands to tools section mcs cli
[fix] minor imports ordering
[fix] constants

cmapi/cmapi_server/constants.py

@@ -20,9 +20,11 @@ EM_PATH_SUFFIX = 'data1/systemFiles/dbrm'
 MCS_EM_PATH = os.path.join(MCS_DATA_PATH, EM_PATH_SUFFIX)
 MCS_BRM_CURRENT_PATH = os.path.join(MCS_EM_PATH, 'BRM_saves_current')
 S3_BRM_CURRENT_PATH = os.path.join(EM_PATH_SUFFIX, 'BRM_saves_current')
+
 # keys file for CEJ password encryption\decryption
 # (CrossEngineSupport section in Columnstore.xml)
-MCS_SECRETS_FILE_PATH = os.path.join(MCS_DATA_PATH, '.secrets')
+MCS_SECRETS_FILENAME = '.secrets'
+MCS_SECRETS_FILE_PATH = os.path.join(MCS_DATA_PATH, MCS_SECRETS_FILENAME)
 
 # CMAPI SERVER
 CMAPI_CONFIG_FILENAME = 'cmapi_server.conf'

cmapi/cmapi_server/controllers/endpoints.py

@@ -19,6 +19,7 @@ from cmapi_server.constants import (
 )
 from cmapi_server.controllers.error import APIError
 from cmapi_server.handlers.cej import CEJError
+from cmapi_server.handlers.cej import CEJPasswordHandler
 from cmapi_server.handlers.cluster import ClusterHandler
 from cmapi_server.helpers import (
     cmapi_config_check, dequote, get_active_nodes, get_config_parser,
@@ -363,6 +364,7 @@ class ConfigController:
         sm_config_filename = request_body.get(
             'sm_config_filename', DEFAULT_SM_CONF_PATH
         )
+        secrets = request_body.get('secrets', None)
 
         if request_mode is None and request_config is None:
             raise_422_error(
@@ -391,6 +393,9 @@ class ConfigController:
             )
         request_response = {'timestamp': str(datetime.now())}
 
+        if secrets:
+            CEJPasswordHandler().save_secrets(secrets)
+
         node_config = NodeConfig()
         xml_config = request_body.get('config', None)
         sm_config = request_body.get('sm_config', None)

cmapi/cmapi_server/handlers/cej.py

@@ -1,12 +1,18 @@
 """Module contains all things related to working with .secrets file."""
+import binascii
 import json
 import logging
 import os
+import pwd
+import stat
+from shutil import copyfile
 
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.primitives import padding
 
-from cmapi_server.constants import MCS_SECRETS_FILE_PATH
+from cmapi_server.constants import (
+    MCS_DATA_PATH, MCS_SECRETS_FILENAME, MCS_SECRETS_FILE_PATH
+)
 from cmapi_server.exceptions import CEJError
 
 
@@ -20,7 +26,7 @@ class CEJPasswordHandler():
     """Handler for CrossEngineSupport password decryption."""
 
     @classmethod
-    def secretsfile_exists(cls):
+    def secretsfile_exists(cls) -> bool:
         """Check the .secrets file in MCS_SECRETS_FILE_PATH.
 
         :return: True if file exists and not empty.
@@ -43,7 +49,7 @@ class CEJPasswordHandler():
         return False
 
     @classmethod
-    def get_secrets_json(cls):
+    def get_secrets_json(cls) -> dict:
         """Get json from .secrets file.
 
         :raises CEJError: on empty\corrupted\wrong format .secrets file
@@ -68,7 +74,7 @@ class CEJPasswordHandler():
             return secrets_json
 
     @classmethod
-    def decrypt_password(cls, enc_data:str):
+    def decrypt_password(cls, enc_data: str) -> str:
         """Decrypt CEJ password if needed.
 
         :param enc_data: encrypted initialization vector + password in hex str
@@ -91,7 +97,7 @@ class CEJPasswordHandler():
             ) from value_error
 
         secrets_json = cls.get_secrets_json()
-        encryption_key_hex = secrets_json.get('encryption_key')
+        encryption_key_hex = secrets_json.get('encryption_key', None)
         if not encryption_key_hex:
             raise CEJError(
                 f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}'
@@ -117,3 +123,93 @@ class CEJPasswordHandler():
             unpadder.update(padded_passwd_bytes) + unpadder.finalize()
         )
         return passwd_bytes.decode()
+
+    @classmethod
+    def encrypt_password(cls, passwd: str) -> str:
+        iv = os.urandom(size=AES_IV_BIN_SIZE)
+
+        secrets_json = cls.get_secrets_json()
+        encryption_key_hex = secrets_json.get('encryption_key')
+        if not encryption_key_hex:
+            raise CEJError(
+                f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}'
+            )
+        try:
+            encryption_key = bytes.fromhex(encryption_key_hex)
+        except ValueError as value_error:
+            raise CEJError(
+                'Non-hexadecimal number found in encryption key from '
+                f'{MCS_SECRETS_FILE_PATH} file.'
+            ) from value_error
+        cipher = Cipher(
+            algorithms.AES(encryption_key),
+            modes.CBC(iv)
+        )
+
+        encryptor = cipher.encryptor()
+        padder = padding.PKCS7(algorithms.AES.block_size).padder()
+        padded_data = padder.update(passwd.encode()) + padder.finalize()
+
+        encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
+
+        return iv + encrypted_data
+
+    @classmethod
+    def generate_secrets_data(cls) -> dict:
+        """Generate secrets data for .secrets file.
+
+        :return: secrets data
+        :rtype: dict
+        """
+        key_length = algorithms.AES256.key_size // 8
+        encryption_key = os.urandom(size=key_length)
+        encryption_key_hex = binascii.hexlify(encryption_key).decode()
+        secrets_dict = {
+            'description': 'Columnstore CrossEngineSupport password encryption/decryption key',
+            'encryption_cipher': 'EVP_aes_256_cbc',
+            'encryption_key': encryption_key_hex
+        }
+
+        return secrets_dict
+
+    @classmethod
+    def save_secrets(
+        cls, secrets: dict, filepath: str = MCS_SECRETS_FILE_PATH,
+        owner: str = 'mysql'
+    ) -> None:
+        """Write secrets to .secrets file.
+
+        :param secrets: secrets dict
+        :type secrets: dict
+        :param filepath: path to the .secrets file
+        :type filepath: str, optional
+        :param owner: owner of the file
+        :type owner: str, optional
+        """
+        if cls.secretsfile_exists():
+            copyfile(
+                filepath,
+                os.path.join(
+                    os.path.dirname(filepath),
+                    f'{os.path.basename(filepath)}.cmapi.save'
+                )
+            )
+
+        try:
+            with open(
+                MCS_SECRETS_FILE_PATH, 'w', encoding='utf-8'
+            ) as secrets_file:
+                json.dump(secrets, secrets_file)
+        except Exception as exc:
+            raise CEJError(f'Write to .secrets file failed.') from exc
+
+        try:
+            os.chmod(MCS_SECRETS_FILE_PATH, stat.S_IRUSR)
+            userinfo = pwd.getpwnam(owner)
+            os.chown(MCS_SECRETS_FILE_PATH, userinfo.pw_uid, userinfo.pw_gid)
+            logging.debug(f'Permissions of .secrets file set to {owner}:read.')
+            logging.debug(f'Ownership of .secrets file given to {owner}.')
+        except Exception as exc:
+            raise CEJError(
+                f'Failed to set permissions or ownership for .secrets file.'
+            ) from exc

cmapi/cmapi_server/handlers/cluster.py

@@ -178,8 +178,7 @@ class ClusterHandler():
         update_revision_and_manager(
             input_config_filename=config, output_config_filename=config
         )
-
+        broadcast_new_config(config, distribute_secrets=True)
-        broadcast_new_config(config)
         logger.debug(f'Successfully finished adding node {node}.')
         return response
 

cmapi/cmapi_server/helpers.py

@@ -290,9 +290,10 @@ def broadcast_new_config(
     sm_config_filename: str = DEFAULT_SM_CONF_PATH,
     test_mode: bool = False,
     nodes: Optional[list] = None,
-    timeout: Optional[int] = None
+    timeout: Optional[int] = None,
+    distribute_secrets: bool = False
 ) -> None:
-    """Send new config to nodes in async way.
+    """Send new config to nodes. Now in async way.
 
     :param cs_config_filename: Columnstore.xml path,
                                defaults to DEFAULT_MCS_CONF_PATH
@@ -310,6 +311,8 @@ def broadcast_new_config(
     :param timeout: timeout passing to gracefully stop DMLProc TODO: for next
                     releases. Could affect all logic of broadcacting new config
     :type timeout: Optional[int], optional
+    :param distribute_secrets: flag to distribute secrets to nodes
+    :type distribute_secrets: bool
     :raises CMAPIBasicError: If Broadcasting config to nodes failed with errors
     """
 
@@ -338,6 +341,12 @@ def broadcast_new_config(
         'sm_config_filename': sm_config_filename,
         'sm_config': sm_config_text
     }
+
+    if distribute_secrets:
+        # TODO: do not restart cluster when put xml config only with
+        secrets = CEJPasswordHandler.get_secrets_json()
+        body['secrets'] = secrets
+
     # TODO: remove test mode here and replace it by mock in tests
     if test_mode:
         body['test'] = True

cmapi/mcs_cluster_tool/__main__.py

@@ -6,7 +6,7 @@ import typer
 
 from cmapi_server.logging_management import dict_config, add_logging_level, enable_console_logging
 from mcs_cluster_tool import (
-    cluster_app, cmapi_app, backup_commands, restore_commands
+    cluster_app, cmapi_app, backup_commands, restore_commands, tools_commands
 )
 from mcs_cluster_tool.constants import MCS_CLI_LOG_CONF_PATH
 
@@ -21,13 +21,25 @@ app = typer.Typer(
     rich_markup_mode='rich',
 )
 app.add_typer(cluster_app.app)
-# TODO: keep this only for potential backward compatibility
+# keep this only for potential backward compatibility and userfriendliness
 app.add_typer(cluster_app.app, name='cluster', hidden=True)
 app.add_typer(cmapi_app.app, name='cmapi')
-app.command('backup')(backup_commands.backup)
+app.command(
-app.command('dbrm_backup')(backup_commands.dbrm_backup)
+    'backup', rich_help_panel='Tools commands'
-app.command('restore')(restore_commands.restore)
+)(backup_commands.backup)
-app.command('dbrm_restore')(restore_commands.dbrm_restore)
+app.command(
+    'dbrm_backup', rich_help_panel='Tools commands'
+)(backup_commands.dbrm_backup)
+app.command(
+    'restore', rich_help_panel='Tools commands'
+)(restore_commands.restore)
+app.command(
+    'dbrm_restore', rich_help_panel='Tools commands'
+)(restore_commands.dbrm_restore)
+app.command('cskeys', rich_help_panel='Tools commands')(tools_commands.cskeys)
+app.command(
+    'cspasswd', rich_help_panel='Tools commands'
+)(tools_commands.cspasswd)
 
 
 @app.command(

cmapi/mcs_cluster_tool/backup_commands.py

@@ -2,9 +2,9 @@
 import logging
 import sys
 from datetime import datetime
-from typing_extensions import Annotated
 
 import typer
+from typing_extensions import Annotated
 
 from cmapi_server.process_dispatchers.base import BaseDispatcher
 from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH

cmapi/mcs_cluster_tool/cluster_app.py

@@ -7,7 +7,6 @@ import time
 from datetime import datetime, timedelta
 from typing import List, Optional
 
-import pyotp
 import requests
 import typer
 from typing_extensions import Annotated

cmapi/mcs_cluster_tool/restore_commands.py

@@ -1,9 +1,9 @@
 """Typer application for restore Columnstore data."""
 import logging
 import sys
-from typing_extensions import Annotated
 
 import typer
+from typing_extensions import Annotated
 
 from cmapi_server.process_dispatchers.base import BaseDispatcher
 from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH

cmapi/mcs_cluster_tool/tools_commands.py

@@ -0,0 +1,94 @@
+import logging
+import os
+
+import typer
+from typing_extensions import Annotated
+
+
+from cmapi_server.constants import MCS_SECRETS_FILE_PATH
+from cmapi_server.exceptions import CEJError
+from cmapi_server.handlers.cej import CEJPasswordHandler
+from mcs_cluster_tool.decorators import handle_output
+
+
+logger = logging.getLogger('mcs_cli')
+# pylint: disable=unused-argument, too-many-arguments, too-many-locals
+# pylint: disable=invalid-name, line-too-long
+
+
+@handle_output
+def cskeys(
+    filepath: Annotated[
+        str,
+        typer.Option(
+            '-f', '--filepath',
+            help='Path to the output file',
+        )
+    ] = MCS_SECRETS_FILE_PATH,
+    username: Annotated[
+        str,
+        typer.Option(
+            '-u', '--username',
+            help='Username for the key',
+        )
+    ] = 'mysql',
+):
+    if CEJPasswordHandler().secretsfile_exists():
+        typer.echo(
+            (
+                f'Secrets file "{filepath}" already exists. '
+                'Delete it before generating a new encryption key.'
+            ),
+            color='red',
+        )
+        raise typer.Exit(code=1)
+    elif not os.path.exists(os.path.dirname(filepath)):
+        typer.echo(
+            f'Directory "{os.path.dirname(filepath)}" does not exist.',
+            color='red'
+        )
+        raise typer.Exit(code=1)
+
+    new_secrets_data = CEJPasswordHandler().generate_secrets_data()
+    try:
+        CEJPasswordHandler().save_secrets(new_secrets_data, owner=username)
+    except CEJError as cej_error:
+        typer.echo(cej_error.message, color='red')
+        raise typer.Exit(code=2)
+    raise typer.Exit(code=0)
+
+
+@handle_output
+def cspasswd(
+    password: Annotated[
+        str,
+        typer.Option(
+            help='Password to encrypt/decrypt',
+            prompt=True, confirmation_prompt=True, hide_input=True
+        )
+    ],
+    decrypt: Annotated[
+        bool,
+        typer.Option(
+            '--decrypt',
+            help='Decrypt the provided password',
+        )
+    ] = False
+):
+    if decrypt:
+        try:
+            decrypted_password = CEJPasswordHandler().decrypt_password(
+                password
+            )
+        except CEJError as cej_error:
+            typer.echo(cej_error.message, color='red')
+            raise typer.Exit(code=1)
+        typer.echo(f'Decoded password: {decrypted_password}', color='green')
+    else:
+        try:
+            encoded_password = CEJPasswordHandler().encrypt_password(password)
+        except CEJError as cej_error:
+            typer.echo(cej_error.message, color='red')
+            raise typer.Exit(code=1)
+        typer.echo(f'Encoded password: {encoded_password}', color='green')
+    raise typer.Exit(code=0)