database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

feat(cmapi): MCOL-5019: review fixes.

Browse Source 
[fix] CEJPasswordHandler class methods to use directory for cskeys file
[fix] CEJPasswordHandler.encrypt_password to return password in hex format
[fix] CEJPasswordHandler key_length
[fix] CEJPasswordHandler os.urandom call typo
[upd] mcs cli README.md and man page
[upd] mcs cli README_DEV.md
[fix] mcs_cluster_tool/decorators.py to handle typer.Exit exception
[add] various docstrings
This commit is contained in:
mariadb-AlanMologorsky
2025-03-13 01:45:42 +03:00
committed by Alan Mologorsky
parent 215e4eea4d
commit aa57a7684c
8 changed files with 248 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
cmapi/cmapi_server/handlers/cej.py
View File

@ -10,9 +10,7 @@ from shutil import copyfile
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.primitives import padding
from cmapi_server.constants import (
MCS_DATA_PATH, MCS_SECRETS_FILENAME, MCS_SECRETS_FILE_PATH
)
from cmapi_server.constants import MCS_DATA_PATH, MCS_SECRETS_FILENAME
from cmapi_server.exceptions import CEJError
@ -26,16 +24,19 @@ class CEJPasswordHandler():
"""Handler for CrossEngineSupport password decryption."""
@classmethod
def secretsfile_exists(cls) -> bool:
"""Check the .secrets file in MCS_SECRETS_FILE_PATH.
def secretsfile_exists(cls, directory: str = MCS_DATA_PATH) -> bool:
"""Check the .secrets file in directory. Default MCS_SECRETS_FILE_PATH.
:param directory: path to the directory with .secrets file
:type directory: str, optional
:return: True if file exists and not empty.
:rtype: bool
"""
secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
try:
if (
os.path.isfile(MCS_SECRETS_FILE_PATH) and
os.path.getsize(MCS_SECRETS_FILE_PATH) > 0
os.path.isfile(secrets_file_path) and
os.path.getsize(secrets_file_path) > 0
):
return True
except Exception:
@ -49,40 +50,48 @@ class CEJPasswordHandler():
return False
@classmethod
def get_secrets_json(cls) -> dict:
def get_secrets_json(cls, directory: str = MCS_DATA_PATH) -> dict:
"""Get json from .secrets file.
:raises CEJError: on empty\corrupted\wrong format .secrets file
:param directory: path to the directory with .secrets file
:type directory: str, optional
:return: json from .secrets file
:rtype: dict
:raises CEJError: on empty\corrupted\wrong format .secrets file
"""
if not cls.secretsfile_exists():
raise CEJError(f'{MCS_SECRETS_FILE_PATH} file does not exist.')
with open(MCS_SECRETS_FILE_PATH) as secrets_file:
secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
if not cls.secretsfile_exists(directory=directory):
raise CEJError(f'{secrets_file_path} file does not exist.')
with open(secrets_file_path) as secrets_file:
try:
secrets_json = json.load(secrets_file)
except Exception:
logging.error(
'Something went wrong while loading json from '
f'{MCS_SECRETS_FILE_PATH}',
f'{secrets_file_path}',
exc_info=True
)
raise CEJError(
f'Looks like file {MCS_SECRETS_FILE_PATH} is corrupted or'
f'Looks like file {secrets_file_path} is corrupted or'
'has wrong format.'
) from None
return secrets_json
@classmethod
def decrypt_password(cls, enc_data: str) -> str:
def decrypt_password(
cls, enc_data: str, directory: str = MCS_DATA_PATH
) -> str:
"""Decrypt CEJ password if needed.
:param directory: path to the directory with .secrets file
:type directory: str, optional
:param enc_data: encrypted initialization vector + password in hex str
:type enc_data: str
:return: decrypted CEJ password
:rtype: str
"""
if not cls.secretsfile_exists():
secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
if not cls.secretsfile_exists(directory=directory):
logging.warning('Unencrypted CrossEngineSupport password used.')
return enc_data
@ -96,18 +105,18 @@ class CEJPasswordHandler():
'Non-hexadecimal number found in encrypted CEJ password.'
) from value_error
secrets_json = cls.get_secrets_json()
secrets_json = cls.get_secrets_json(directory=directory)
encryption_key_hex = secrets_json.get('encryption_key', None)
if not encryption_key_hex:
raise CEJError(
f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}'
f'Empty "encryption key" found in {secrets_file_path}'
)
try:
encryption_key = bytes.fromhex(encryption_key_hex)
except ValueError as value_error:
raise CEJError(
'Non-hexadecimal number found in encryption key from '
f'{MCS_SECRETS_FILE_PATH} file.'
f'{secrets_file_path} file.'
) from value_error
cipher = Cipher(
algorithms.AES(encryption_key),
@ -125,21 +134,34 @@ class CEJPasswordHandler():
return passwd_bytes.decode()
@classmethod
def encrypt_password(cls, passwd: str) -> str:
iv = os.urandom(size=AES_IV_BIN_SIZE)
def encrypt_password(
cls, passwd: str, directory: str = MCS_DATA_PATH
) -> str:
"""Encrypt CEJ password.
secrets_json = cls.get_secrets_json()
:param directory: path to the directory with .secrets file
:type directory: str, optional
:param passwd: CEJ password
:type passwd: str
:return: encrypted CEJ password in uppercase hex format
:rtype: str
"""
secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
iv = os.urandom(AES_IV_BIN_SIZE)
secrets_json = cls.get_secrets_json(directory=directory)
encryption_key_hex = secrets_json.get('encryption_key')
if not encryption_key_hex:
raise CEJError(
f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}'
f'Empty "encryption key" found in {secrets_file_path}'
)
try:
encryption_key = bytes.fromhex(encryption_key_hex)
except ValueError as value_error:
raise CEJError(
'Non-hexadecimal number found in encryption key from '
f'{MCS_SECRETS_FILE_PATH} file.'
f'{secrets_file_path} file.'
) from value_error
cipher = Cipher(
algorithms.AES(encryption_key),
@ -151,8 +173,8 @@ class CEJPasswordHandler():
padded_data = padder.update(passwd.encode()) + padder.finalize()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
return iv + encrypted_data
encrypted_passwd_bytes = iv + encrypted_data
return encrypted_passwd_bytes.hex().upper()
@classmethod
def generate_secrets_data(cls) -> dict:
@ -161,8 +183,8 @@ class CEJPasswordHandler():
:return: secrets data
:rtype: dict
"""
key_length = algorithms.AES256.key_size // 8
encryption_key = os.urandom(size=key_length)
key_length = 32 # AES256 key_size
encryption_key = os.urandom(key_length)
encryption_key_hex = binascii.hexlify(encryption_key).decode()
secrets_dict = {
'description': 'Columnstore CrossEngineSupport password encryption/decryption key',
@ -174,11 +196,13 @@ class CEJPasswordHandler():
@classmethod
def save_secrets(
cls, secrets: dict, filepath: str = MCS_SECRETS_FILE_PATH,
cls, secrets: dict, directory: str = MCS_DATA_PATH,
owner: str = 'mysql'
) -> None:
"""Write secrets to .secrets file.
:param directory: path to the directory with .secrets file
:type directory: str, optional
:param secrets: secrets dict
:type secrets: dict
:param filepath: path to the .secrets file
@ -186,29 +210,43 @@ class CEJPasswordHandler():
:param owner: owner of the file
:type owner: str, optional
"""
if cls.secretsfile_exists():
copyfile(
filepath,
os.path.join(
os.path.dirname(filepath),
f'{os.path.basename(filepath)}.cmapi.save'
secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
if cls.secretsfile_exists(directory=directory):
if cls.get_secrets_json(directory=directory) != secrets:
copyfile(
secrets_file_path,
os.path.join(
directory,
f'{MCS_SECRETS_FILENAME}.cmapi.save'
)
)
)
logging.warning(
f'Backup of {secrets_file_path} file created.'
)
else:
logging.debug(
f'No changes in {secrets_file_path} file detected.'
)
return
try:
with open(
MCS_SECRETS_FILE_PATH, 'w', encoding='utf-8'
secrets_file_path, 'w', encoding='utf-8'
) as secrets_file:
json.dump(secrets, secrets_file)
except Exception as exc:
raise CEJError(f'Write to .secrets file failed.') from exc
try:
os.chmod(MCS_SECRETS_FILE_PATH, stat.S_IRUSR)
os.chmod(secrets_file_path, stat.S_IRUSR)
userinfo = pwd.getpwnam(owner)
os.chown(MCS_SECRETS_FILE_PATH, userinfo.pw_uid, userinfo.pw_gid)
logging.debug(f'Permissions of .secrets file set to {owner}:read.')
logging.debug(f'Ownership of .secrets file given to {owner}.')
os.chown(secrets_file_path, userinfo.pw_uid, userinfo.pw_gid)
logging.debug(
f'Permissions of {secrets_file_path} file set to {owner}:read.'
)
logging.debug(
f'Ownership of {secrets_file_path} file given to {owner}.'
)
except Exception as exc:
raise CEJError(
f'Failed to set permissions or ownership for .secrets file.'

6
cmapi/cmapi_server/helpers.py
View File

@ -344,6 +344,10 @@ def broadcast_new_config(
if distribute_secrets:
# TODO: do not restart cluster when put xml config only with
# distribute secrets
if not CEJPasswordHandler.secretsfile_exists():
secrets_dict = CEJPasswordHandler.generate_secrets_data()
CEJPasswordHandler.save_secrets(secrets=secrets_dict)
secrets = CEJPasswordHandler.get_secrets_json()
body['secrets'] = secrets
@ -798,7 +802,7 @@ def get_cej_info(config_root):
'Columnstore.xml has an empty CrossEngineSupport.Password tag'
)
if CEJPasswordHandler.secretsfile_exists():
if CEJPasswordHandler.secretsfile_exists() and cej_password:
cej_password = CEJPasswordHandler.decrypt_password(cej_password)
return cej_host, cej_port, cej_username, cej_password

46
cmapi/mcs_cluster_tool/README.md
View File

@ -18,6 +18,8 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
* `dbrm_backup`: Columnstore DBRM Backup.
* `restore`: Restore Columnstore (and/or MariaDB) data.
* `dbrm_restore`: Restore Columnstore DBRM data.
* `cskeys`: Generates a random AES encryption key and init vector and writes them to disk.
* `cspasswd`: Encrypt a Columnstore plaintext password using the encryption key in the key file.
* `help-all`: Show help for all commands in man page style.
* `status`: Get status information.
* `stop`: Stop the Columnstore cluster.
@ -162,6 +164,50 @@ $ mcs dbrm_restore [OPTIONS]
* `-li, --list`: List backups.
* `--help`: Show this message and exit.
## `mcs cskeys`
This utility generates a random AES encryption key and init vector
and writes them to disk. The data is written to the file '.secrets',
in the specified directory. The key and init vector are used by
the utility 'cspasswd' to encrypt passwords used in Columnstore
configuration files, as well as by Columnstore itself to decrypt the
passwords.
WARNING: Re-creating the file invalidates all existing encrypted
passwords in the configuration files.
**Usage**:
```console
$ mcs cskeys [OPTIONS] [DIRECTORY]
```
**Arguments**:
* `[DIRECTORY]`: The directory where to store the file in. [default: /var/lib/columnstore]
**Options**:
* `-u, --user TEXT`: Designate the owner of the generated file. [default: mysql]
* `--help`: Show this message and exit.
## `mcs cspasswd`
Encrypt a Columnstore plaintext password using the encryption key in
the key file.
**Usage**:
```console
$ mcs cspasswd [OPTIONS]
```
**Options**:
* `--password TEXT`: Password to encrypt/decrypt [required]
* `--decrypt`: Decrypt an encrypted password instead.
* `--help`: Show this message and exit.
## `mcs help-all`
Show help for all commands in man page style.

10
cmapi/mcs_cluster_tool/README_DEV.md
View File

@ -7,6 +7,14 @@
```bash
typer mcs_cluster_tool/__main__.py utils docs --name mcs --output README.md
```
Optionally could be generated from installed package.
```bash
PYTHONPATH="/usr/share/columnstore/cmapi:/usr/share/columnstore/cmapi/deps" /usr/share/columnstore/cmapi/python/bin/python3 -m typer /usr/share/columnstore/cmapi/mcs_cluster_tool/__main__.py utils docs --name mcs --output ~/README.md
```
- dependencies for gem build (RHEL example)
```bash
sudo dnf install make gcc redhat-rpm-config -y
```
- install `md2man` (for now it's the only one tool that make convertation without any issues)
```bash
sudo yum install -y ruby ruby-devel
@ -14,6 +22,6 @@
```
- convert to perfect `.roff` file (`man` page)
```bash
md2man README.md > mcs.1
md2man-roff README.md > mcs.1
```
- enjoy =)

11
cmapi/mcs_cluster_tool/__main__.py
View File

@ -36,9 +36,16 @@ app.command(
app.command(
'dbrm_restore', rich_help_panel='Tools commands'
)(restore_commands.dbrm_restore)
app.command('cskeys', rich_help_panel='Tools commands')(tools_commands.cskeys)
app.command(
'cspasswd', rich_help_panel='Tools commands'
'cskeys', rich_help_panel='Tools commands',
short_help=(
'Generates a random AES encryption key and init vector and writes '
'them to disk.'
)
)(tools_commands.cskeys)
app.command(
'cspasswd', rich_help_panel='Tools commands',
short_help='Encrypt a Columnstore plaintext password.'
)(tools_commands.cspasswd)

5
cmapi/mcs_cluster_tool/decorators.py
View File

@ -25,11 +25,16 @@ def handle_output(func):
except typer.BadParameter as err:
logger.error('Bad command line parameter.')
raise err
except typer.Exit as err: # if some command used typer.Exit
#TODO: think about universal protocol to return json data and
# plain text results.
return_code = err.exit_code
except Exception:
logger.error(
'Undefined error during command execution',
exc_info=True
)
typer.echo('Unknown error, check the log file.', err=True)
raise typer.Exit(return_code)
return wrapper

59
cmapi/mcs_cluster_tool/mcs.1
View File

@ -27,6 +27,10 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
.IP \(bu 2
\fB\fCdbrm_restore\fR: Restore Columnstore DBRM data.
.IP \(bu 2
\fB\fCcskeys\fR: Generates a random AES encryption key and init vector and writes them to disk.
.IP \(bu 2
\fB\fCcspasswd\fR: Encrypt a Columnstore plaintext password using the encryption key in the key file.
.IP \(bu 2
\fB\fChelp\-all\fR: Show help for all commands in man page style.
.IP \(bu 2
\fB\fCstatus\fR: Get status information.
@ -252,6 +256,61 @@ $ mcs dbrm_restore [OPTIONS]
.IP \(bu 2
\fB\fC\-\-help\fR: Show this message and exit.
.RE
.SH \fB\fCmcs cskeys\fR
.PP
This utility generates a random AES encryption key and init vector
and writes them to disk. The data is written to the file \[aq]\&.secrets\[aq],
in the specified directory. The key and init vector are used by
the utility \[aq]cspasswd\[aq] to encrypt passwords used in Columnstore
configuration files, as well as by Columnstore itself to decrypt the
passwords.
.PP
WARNING: Re\-creating the file invalidates all existing encrypted
passwords in the configuration files.
.PP
\fBUsage\fP:
.PP
.RS
.nf
$ mcs cskeys [OPTIONS] [DIRECTORY]
.fi
.RE
.PP
\fBArguments\fP:
.RS
.IP \(bu 2
\fB\fC[DIRECTORY]\fR: The directory where to store the file in. [default: /var/lib/columnstore]
.RE
.PP
\fBOptions\fP:
.RS
.IP \(bu 2
\fB\fC\-u, \-\-user TEXT\fR: Designate the owner of the generated file. [default: mysql]
.IP \(bu 2
\fB\fC\-\-help\fR: Show this message and exit.
.RE
.SH \fB\fCmcs cspasswd\fR
.PP
Encrypt a Columnstore plaintext password using the encryption key in
the key file.
.PP
\fBUsage\fP:
.PP
.RS
.nf
$ mcs cspasswd [OPTIONS]
.fi
.RE
.PP
\fBOptions\fP:
.RS
.IP \(bu 2
\fB\fC\-\-password TEXT\fR: Password to encrypt/decrypt [required]
.IP \(bu 2
\fB\fC\-\-decrypt\fR: Decrypt an encrypted password instead.
.IP \(bu 2
\fB\fC\-\-help\fR: Show this message and exit.
.RE
.SH \fB\fCmcs help\-all\fR
.PP
Show help for all commands in man page style.

51
cmapi/mcs_cluster_tool/tools_commands.py
View File

@ -5,7 +5,9 @@ import typer
from typing_extensions import Annotated
from cmapi_server.constants import MCS_SECRETS_FILE_PATH
from cmapi_server.constants import (
MCS_DATA_PATH, MCS_SECRETS_FILENAME
)
from cmapi_server.exceptions import CEJError
from cmapi_server.handlers.cej import CEJPasswordHandler
from mcs_cluster_tool.decorators import handle_output
@ -18,22 +20,33 @@ logger = logging.getLogger('mcs_cli')
@handle_output
def cskeys(
filepath: Annotated[
user: Annotated[
str,
typer.Option(
'-f', '--filepath',
help='Path to the output file',
)
] = MCS_SECRETS_FILE_PATH,
username: Annotated[
str,
typer.Option(
'-u', '--username',
help='Username for the key',
'-u', '--user',
help='Designate the owner of the generated file.',
)
] = 'mysql',
directory: Annotated[
str,
typer.Argument(
help='The directory where to store the file in.',
)
] = MCS_DATA_PATH
):
if CEJPasswordHandler().secretsfile_exists():
"""
This utility generates a random AES encryption key and init vector
and writes them to disk. The data is written to the file '.secrets',
in the specified directory. The key and init vector are used by
the utility 'cspasswd' to encrypt passwords used in Columnstore
configuration files, as well as by Columnstore itself to decrypt the
passwords.
WARNING: Re-creating the file invalidates all existing encrypted
passwords in the configuration files.
"""
filepath = os.path.join(directory, MCS_SECRETS_FILENAME)
if CEJPasswordHandler().secretsfile_exists(directory=directory):
typer.echo(
(
f'Secrets file "{filepath}" already exists. '
@ -44,14 +57,18 @@ def cskeys(
raise typer.Exit(code=1)
elif not os.path.exists(os.path.dirname(filepath)):
typer.echo(
f'Directory "{os.path.dirname(filepath)}" does not exist.',
f'Directory "{directory}" does not exist.',
color='red'
)
raise typer.Exit(code=1)
new_secrets_data = CEJPasswordHandler().generate_secrets_data()
try:
CEJPasswordHandler().save_secrets(new_secrets_data, owner=username)
CEJPasswordHandler().save_secrets(
new_secrets_data, owner=user, directory=directory
)
typer.echo(f'Permissions of "{filepath}" set to owner:read.')
typer.echo(f'Ownership of "{filepath}" given to {user}.')
except CEJError as cej_error:
typer.echo(cej_error.message, color='red')
raise typer.Exit(code=2)
@ -71,10 +88,14 @@ def cspasswd(
bool,
typer.Option(
'--decrypt',
help='Decrypt the provided password',
help='Decrypt an encrypted password instead.',
)
] = False
):
"""
Encrypt a Columnstore plaintext password using the encryption key in
the key file.
"""
if decrypt:
try:
decrypted_password = CEJPasswordHandler().decrypt_password(