database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-10-28 19:54:55 +03:00
Code Activity
Files
bf8f58cc7c6750f25b4cdf9e063f7e1a48d8e8b5
mariadb-columnstore-engine/cmapi/cmapi_server/controllers/endpoints.py

1925 lines
73 KiB
Python
Raw Blame History

import logging
import hashlib
import socket
import subprocess
import time
from copy import deepcopy
from datetime import datetime
from pathlib import Path
import cherrypy
import pyotp
import requests
from mcs_node_control.models.dbrm import set_cluster_mode
from mcs_node_control.models.node_config import NodeConfig
from mcs_node_control.models.node_status import NodeStatus
from pydantic import ValidationError
from cmapi_server.constants import (
CMAPI_PACKAGE_NAME, CMAPI_PORT, DEFAULT_MCS_CONF_PATH,
DEFAULT_SM_CONF_PATH, EM_PATH_SUFFIX, MCS_BRM_CURRENT_PATH, MCS_EM_PATH,
MDB_CS_PACKAGE_NAME, MDB_SERVER_PACKAGE_NAME, REQUEST_TIMEOUT,
S3_BRM_CURRENT_PATH, SECRET_KEY,
)
from cmapi_server.controllers.api_clients import NodeControllerClient
from cmapi_server.controllers.error import APIError
from cmapi_server.exceptions import CMAPIBasicError, cmapi_error_to_422
from cmapi_server.controllers.request_models import (
ConfigPutRequestRootModel, StatefulConfigPutRequestModel,
)
from cmapi_server.handlers.cej import CEJError, CEJPasswordHandler
from cmapi_server.handlers.cluster import ClusterHandler
from cmapi_server.helpers import (
cmapi_config_check, dequote, get_active_nodes, get_config_parser,
get_current_key, get_dbroots, in_maintenance_state,
save_cmapi_conf_file, system_ready,
)
from cmapi_server.logging_management import change_loggers_level
from cmapi_server.managers.application import (
AppManager, AppStatefulConfig, StatefulConfigModel
)
from cmapi_server.managers.upgrade.packages import PackagesManager
from cmapi_server.managers.upgrade.repo import MariaDBESRepoManager
from cmapi_server.managers.backup_restore import PreUpgradeBackupRestoreManager
from cmapi_server.managers.process import MCSProcessManager, MDBProcessManager
from cmapi_server.managers.transaction import TransactionManager
from cmapi_server.node_manipulation import is_master, switch_node_maintenance
from cmapi_server.controllers.decorators import disable_json
# Bug in pylint https://github.com/PyCQA/pylint/issues/4584
requests.packages.urllib3.disable_warnings() # pylint: disable=no-member
module_logger = logging.getLogger('cmapi_server')
def log_begin(logger, func_name):
logger.debug(f"{func_name} starts")
def raise_422_error(
logger, func_name: str = '', err_msg: str = '', exc_info: bool = True
) -> None:
"""Function to log error and raise 422 api error.
:param logger: logger to use
:type logger: logging.Logger
:param func_name: function name where it called, defaults to ''
:type func_name: str, optional
:param err_msg: error message, defaults to ''
:type err_msg: str, optional
:param exc_info: write traceback to logs or not.
:type exc_info: bool
:raises APIError: everytime with custom error message
"""
# TODO: change:
# - func name to inspect.stack(0)[1][3]
# - make something to logger, seems passing here is useless
logger.error(f'{func_name} {err_msg}', exc_info=exc_info)
raise APIError(422, err_msg)
# TODO: Move somwhere else, eg. to helpers
def get_use_sudo(app_config: dict) -> bool:
"""Get value about using superuser or not from app config.
:param app_config: CherryPy application config
:type app_config: dict
:return: use_sudo config value
:rtype: bool
"""
privileges_section = app_config.get('Privileges', None)
if privileges_section is not None:
use_sudo = privileges_section.get('use_sudo', False)
else:
use_sudo = False
return use_sudo
@cherrypy.tools.register('before_handler', priority=80)
def validate_api_key():
"""Validate API key.
If no config file, create new one by coping from default. If no API key,
set api key from request headers.
"""
# TODO: simplify validation, using preload and may be class-controller
req = cherrypy.request
if 'X-Api-Key' not in req.headers:
error_message = 'No API key provided.'
module_logger.warning(error_message)
raise cherrypy.HTTPError(401, error_message)
# we thinking that api_key is the same with quoted api_key
request_api_key = dequote(req.headers.get('X-Api-Key', ''))
if not request_api_key:
error_message = 'Empty API key.'
module_logger.warning(error_message)
raise cherrypy.HTTPError(401, error_message)
# because of architecture of cherrypy config parser it makes from values
# python objects it causes some non standart behaviour
# - makes dequote of config values automatically if it is strings
# - config objects always gives a dict object
# - strings with only integers inside will be always converted to int type
inmemory_api_key = str(
req.app.config.get('Authentication', {}).get('x-api-key', '')
)
if not inmemory_api_key:
module_logger.warning(
'No API key in the configuration. Adding it into the config.'
)
req.app.config.update(
{'Authentication': {'x-api-key': request_api_key}}
)
# update the cmapi server config file
config_filepath = req.app.config['config']['path']
cmapi_config_check(config_filepath)
cfg_parser = get_config_parser(config_filepath)
if not cfg_parser.has_section('Authentication'):
cfg_parser.add_section('Authentication')
# TODO: Do not store api key in cherrypy config.
# It causes some overhead on custom ini file and handling it.
# For cherrypy config file values have to be python objects.
# So string have to be quoted.
cfg_parser['Authentication']['x-api-key'] = f"'{request_api_key}'"
save_cmapi_conf_file(cfg_parser, config_filepath)
return
if inmemory_api_key != request_api_key:
module_logger.warning(f'Incorrect API key [ {request_api_key} ]')
raise cherrypy.HTTPError(401, 'Incorrect API key')
@cherrypy.tools.register("before_handler", priority=81)
def active_operation():
app = cherrypy.request.app
txn_section = app.config.get('txn', None)
txn_manager_address = None
if txn_section is not None:
txn_manager_address = app.config['txn'].get('manager_address', None)
if txn_manager_address is not None and len(txn_manager_address) > 0:
raise_422_error(
module_logger, 'active_operation', 'There is an active operation.'
)
@cherrypy.tools.register('before_handler', priority=82)
def has_active_nodes():
"""Check if there are any active nodes in the cluster.
TODO: Remove in next releases due to never used.
Now TransactionManager has this check inside.
Before removing, have to check all API endpoints without transaction
mechanics to potential use of this handler.
"""
active_nodes = get_active_nodes()
if len(active_nodes) == 0:
raise_422_error(
module_logger, 'has_active_nodes',
'No active nodes in the cluster.'
)
class TimingTool(cherrypy.Tool):
"""Tool to measure imncoming requests processing time."""
def __init__(self):
# if before_handler used we got 500 on each error in request body
# (eg wrong or no content in PUT requests):
# - wrong request body
# - never happened handler
# - no before_handler event
# - never add cherrypy.request._time
# - got error at before_finalize event getting cherrypy.request._time
# - return 500 instead of 415 error
super().__init__('before_request_body', self.start_timer, priority=90)
def _setup(self):
"""Method to call by CherryPy when the tool is applied."""
super()._setup()
cherrypy.request.hooks.attach(
'before_finalize', self.end_timer, priority=5
)
def start_timer(self):
"""Save time and log information about incoming request."""
cherrypy.request._time = time.time()
logger = logging.getLogger('access_logger')
request = cherrypy.request
remote = request.remote.name or request.remote.ip
logger.info(
f'Got incoming {request.method} request from "{remote}" '
f'to "{request.path_info}". uid: {request.unique_id}'
)
def end_timer(self):
"""Calculate request processing duration and leave a log message."""
duration = time.time() - cherrypy.request._time
logger = logging.getLogger('access_logger')
request = cherrypy.request
remote = request.remote.name or request.remote.ip
logger.info(
f'Finished processing incoming {request.method} '
f'request from "{remote}" to "{request.path_info}" in '
f'{duration:.4f} seconds. uid: {request.unique_id}'
)
cherrypy.tools.timeit = TimingTool()
class StatusController:
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@disable_json(input=True)
def get_status(self):
"""
Handler for /status (GET)
"""
func_name = 'get_status'
log_begin(module_logger, func_name)
node_status = NodeStatus()
hostname = (
cherrypy.request.headers.get('Host', '').split(':')[0] or
socket.gethostname()
)
#TODO: add localhost condition check and another way to get FQDN
node_fqdn = socket.gethostbyaddr(hostname)[0]
status_response = {
'timestamp': str(datetime.now()),
'uptime': node_status.get_host_uptime(),
'dbrm_mode': node_status.get_dbrm_status(),
'cluster_mode': node_status.get_cluster_mode(),
'dbroots': sorted(get_dbroots(node_fqdn)),
'module_id': int(node_status.get_module_id()),
'services': MCSProcessManager.get_running_mcs_procs(),
}
module_logger.debug(f'{func_name} returns {str(status_response)}')
return status_response
@cherrypy.tools.timeit()
@disable_json(input=True)
def get_primary(self):
"""
Handler for /primary (GET)
..WARNING: do not add api key validation here, this may cause
mcs-loadbrm.py (in MCS engine repo) failure
"""
func_name = 'get_primary'
log_begin(module_logger, func_name)
# TODO: convert this value to json bool (remove str() invoke here)
# to do so loadbrm and save brm have to be fixed
# + check other places
get_master_response = {'is_primary': str(NodeConfig().is_primary_node())}
module_logger.debug(f'{func_name} returns {str(get_master_response)}')
return get_master_response
@cherrypy.tools.timeit()
@disable_json(input=True)
def get_new_primary(self):
"""
Handler for /new_primary (GET)
"""
func_name = 'get_new_primary'
log_begin(module_logger, func_name)
try:
get_master_response = {'is_primary': is_master()}
except CEJError as cej_error:
raise_422_error(
module_logger, func_name, cej_error.message
)
module_logger.debug(f'{func_name} returns {str(get_master_response)}')
return get_master_response
class ConfigController:
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@disable_json(input=True)
def get_config(self):
"""
Handler for /config (GET)
"""
func_name = 'get_config'
log_begin(module_logger, func_name)
mcs_config = NodeConfig()
config_response = {'timestamp': str(datetime.now()),
'config': mcs_config.get_current_config(),
'sm_config': mcs_config.get_current_sm_config(),
}
if (module_logger.isEnabledFor(logging.DEBUG)):
dbg_config_response = deepcopy(config_response)
dbg_config_response.pop('config')
dbg_config_response['config'] = 'config was removed to reduce logs.'
dbg_config_response['sm_config'] = 'config was removed to reduce logs.'
module_logger.debug(
f'{func_name} returns {str(dbg_config_response)}'
)
return config_response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_config(self):
"""
Handler for /config (PUT)
"""
func_name = 'put_config'
log_begin(module_logger, func_name)
app = cherrypy.request.app
txn_section = app.config.get('txn', None)
if txn_section is None:
raise_422_error(
module_logger, func_name,
'PUT /config called outside of an operation.'
)
try:
wrapper = ConfigPutRequestRootModel.model_validate(cherrypy.request.json)
# the actual StatefulConfigPutRequestModel or FullConfigPutRequestModel or
# PutConfigSetModeRequestModel
req_model = wrapper.root
except ValidationError as exp:
raise_422_error(
module_logger, func_name, f'Mandatory attribute is missing: {exp.errors()}'
)
req = cherrypy.request
use_sudo = get_use_sudo(req.app.config)
#TODO: remove is_test
# is_test = True means this should not save
# the config file or apply the changes
is_test = req_model.test
if req_model.type == 'set_mode':
# TODO: move it to separate endpoint
request_timeout = req_model.timeout
request_cluster_mode = req_model.cluster_mode
current_mode = set_cluster_mode(request_cluster_mode)
if current_mode == request_cluster_mode:
# Normal exit
request_response = {'timestamp': str(datetime.now())}
module_logger.debug(
f'{func_name} returns {str(request_response)}'
)
return request_response
else:
raise_422_error(
module_logger, func_name,
(
f'Error occured setting cluster to "{request_cluster_mode}" '
f'mode, got "{current_mode}"'
)
)
# if stateful config is provided, we just need to fast apply only stateful config
success = AppStatefulConfig.apply_update(req_model.stateful_config_dict)
if not success:
logging.info('Stateful config update was stale.')
else:
logging.info(
f'Stateful config updated with term {req_model.stateful_config_dict.version.term} '
f'and seq {req_model.stateful_config_dict.version.seq}.'
)
if isinstance(req_model, StatefulConfigPutRequestModel):
return {'timestamp': str(datetime.now()), 'success': success}
request_mode = req_model.cluster_mode
xml_config = req_model.config
sm_config = req_model.sm_config
mcs_config_filename = req_model.mcs_config_filename
sm_config_filename = req_model.sm_config_filename
secrets = req_model.secrets
request_timeout = req_model.timeout
operation_params = (request_mode, xml_config, secrets)
# if no operation to apply, return 422
if not any(operation_params):
raise_422_error(module_logger, func_name, 'Mandatory operation attribute is missing.')
request_headers = cherrypy.request.headers
request_manager_address = request_headers.get('Remote-Addr', None)
if request_manager_address is None:
raise_422_error(
module_logger, func_name,
'Cannot get Cluster manager IP address.'
)
txn_manager_address = app.config['txn'].get('manager_address', None)
if txn_manager_address is None or len(txn_manager_address) == 0:
raise_422_error(
module_logger, func_name,
'PUT /config called outside of an operation.'
)
txn_manager_address = dequote(txn_manager_address).lower()
request_manager_address = dequote(request_manager_address).lower()
if request_manager_address in ['127.0.0.1', 'localhost', '::1']:
request_manager_address = socket.gethostbyname(
socket.gethostname()
)
request_response = {'timestamp': str(datetime.now())}
if secrets:
#TODO: validate incoming secrets?
CEJPasswordHandler().save_secrets(secrets)
node_config = NodeConfig()
if is_test:
return request_response
if xml_config is not None:
node_config.apply_config(
config_filename=mcs_config_filename,
xml_string=xml_config,
sm_config_filename=sm_config_filename,
sm_config_string=sm_config
)
# TODO: change stop/start to restart option.
try:
MCSProcessManager.stop_node(
is_primary=node_config.is_primary_node(),
use_sudo=use_sudo,
timeout=request_timeout,
)
except CMAPIBasicError as err:
raise_422_error(
module_logger, func_name,
f'Error while stopping node. Details: {err.message}.',
exc_info=False
)
# if not in the list of active nodes,
# then do not start the services
new_root = node_config.get_current_config_root(
mcs_config_filename
)
if in_maintenance_state():
module_logger.info(
'Maintenance state is active in new config. '
'MCS processes should not be started.'
)
cherrypy.engine.publish('failover', False)
# skip all other operations below
return request_response
else:
cherrypy.engine.publish('failover', True)
if node_config.in_active_nodes(new_root):
try:
MCSProcessManager.start_node(
is_primary=node_config.is_primary_node(),
use_sudo=use_sudo,
is_read_replica=node_config.am_i_read_replica(),
)
except CMAPIBasicError as err:
raise_422_error(
module_logger, func_name,
(
'Error while starting node. '
f'Details: {err.message}'
),
exc_info=False
)
else:
module_logger.info(
'This node is not in the current ActiveNodes section. '
'Not starting Columnstore processes.'
)
attempts = 0
# TODO: FIX IT. If got (False, False) result, for eg in case
# when special CEJ user is not set, this check loop
# is useless and does nothing.
try:
ready, retry = system_ready(mcs_config_filename)
except CEJError as cej_error:
raise_422_error(
module_logger, func_name, cej_error.message
)
while not ready:
if retry:
attempts +=1
if attempts >= 10:
module_logger.debug(
'Timed out waiting for this node to become ready.'
)
break
time.sleep(1)
else:
break
try:
ready, retry = system_ready(mcs_config_filename)
except CEJError as cej_error:
raise_422_error(
module_logger, func_name, cej_error.message
)
else:
module_logger.debug(f'Node is ready to accept queries.')
app.config['txn']['config_changed'] = True
# We might want to raise error
return request_response
# Unexpected exit
raise_422_error(module_logger, func_name, 'Unknown error.')
class BeginController:
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@cherrypy.tools.active_operation() # pylint: disable=no-member
def put_begin(self):
"""
Handler for /begin (PUT)
"""
func_name = 'put_begin'
log_begin(module_logger, func_name)
app = cherrypy.request.app
request_body = cherrypy.request.json
txn_id = request_body.get('id', None)
txn_timeout = request_body.get('timeout', None)
request_headers = cherrypy.request.headers
txn_manager_address = request_headers.get('Remote-Addr', None)
module_logger.debug(f'{func_name} JSON body {str(request_body)}')
if txn_manager_address is None:
raise_422_error(module_logger, func_name, "Cannot get Cluster Manager \
IP address.")
txn_manager_address = dequote(txn_manager_address).lower()
if txn_manager_address in ['127.0.0.1', 'localhost', '::1']:
txn_manager_address = socket.gethostbyname(socket.gethostname())
if txn_id is None or txn_timeout is None or txn_manager_address is None:
raise_422_error(module_logger, func_name, "id or timeout is not set.")
app.config.update({
'txn': {
'id': txn_id,
'timeout': int(datetime.now().timestamp()) + txn_timeout,
'manager_address': txn_manager_address,
'config_changed': False,
},
})
begin_response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(begin_response)}')
return begin_response
class CommitController:
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_commit(self):
"""
Handler for /commit (PUT)
"""
func_name = 'put_commit'
log_begin(module_logger, func_name)
commit_response = {'timestamp': str(datetime.now())}
app = cherrypy.request.app
txn_section = app.config.get('txn', None)
if txn_section is None:
raise_422_error(module_logger, func_name, "No operation to commit.")
request_headers = cherrypy.request.headers
request_manager_address = request_headers.get('Remote-Addr', None)
if request_manager_address is None:
raise_422_error(module_logger, func_name, "Cannot get Cluster\
Manager IP address.")
txn_manager_address = app.config['txn'].get('manager_address', None)
if txn_manager_address is None or len(txn_manager_address) == 0:
raise_422_error(module_logger, func_name, "No operation to commit.")
txn_manager_address = dequote(txn_manager_address).lower()
request_manager_address = dequote(request_manager_address).lower()
if request_manager_address in ['127.0.0.1', 'localhost', '::1']:
request_manager_address = socket.gethostbyname(socket.gethostname())
# txn is active
app.config['txn']['id'] = 0
app.config['txn']['timeout'] = 0
app.config['txn']['manager_address'] = ''
app.config['txn']['config_changed'] = False
module_logger.debug(f'{func_name} returns {str(commit_response)}')
return commit_response
class RollbackController:
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_rollback(self):
"""
Handler for /rollback (PUT)
"""
rollback_response = {'timestamp': str(datetime.now())}
app = cherrypy.request.app
txn_section = app.config.get('txn', None)
if txn_section is None:
raise APIError(422, 'No operation to rollback.')
request_headers = cherrypy.request.headers
request_manager_address = request_headers.get('Remote-Addr', None)
if request_manager_address is None:
raise APIError(422, 'Cannot get Cluster Manager IP address.')
txn_manager_address = app.config['txn'].get('manager_address', None)
if txn_manager_address is None or len(txn_manager_address) == 0:
raise APIError(422, 'No operation to rollback.')
txn_manager_address = dequote(txn_manager_address).lower()
request_manager_address = dequote(request_manager_address).lower()
if request_manager_address in ['127.0.0.1', 'localhost', '::1']:
request_manager_address = socket.gethostbyname(socket.gethostname())
#TODO: add restart processes flag?
# txn is active
txn_config_changed = app.config['txn'].get('config_changed', None)
if txn_config_changed is True:
node_config = NodeConfig()
node_config.rollback_config()
# TODO: do we need to restart node here?
node_config.apply_config(
xml_string=node_config.get_current_config()
)
app.config['txn']['id'] = 0
app.config['txn']['timeout'] = 0
app.config['txn']['manager_address'] = ''
app.config['txn']['config_changed'] = False
return rollback_response
class StartController:
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_start(self):
func_name = 'put_start'
log_begin(module_logger, func_name)
req = cherrypy.request
use_sudo = get_use_sudo(req.app.config)
node_config = NodeConfig()
try:
MCSProcessManager.start_node(
is_primary=node_config.is_primary_node(),
use_sudo=use_sudo,
is_read_replica=node_config.am_i_read_replica(),
)
except CMAPIBasicError as err:
raise_422_error(
module_logger, func_name,
f'Error while starting node processes. Details: {err.message}',
exc_info=False
)
# TODO: should we change config revision here? Seem to be no.
# Do we need to change flag in a one node maintenance?
switch_node_maintenance(False)
cherrypy.engine.publish('failover', True)
start_response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(start_response)}')
return start_response
class ShutdownController:
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_shutdown(self):
func_name = 'put_shutdown'
log_begin(module_logger, func_name)
req = cherrypy.request
use_sudo = get_use_sudo(req.app.config)
request_body = cherrypy.request.json
timeout = request_body.get('timeout', 0)
node_config = NodeConfig()
try:
MCSProcessManager.stop_node(
is_primary=node_config.is_primary_node(),
use_sudo=use_sudo,
timeout=timeout,
)
except CMAPIBasicError as err:
raise_422_error(
module_logger, func_name,
f'Error while stopping node processes. Details: {err.message}',
exc_info=False
)
# TODO: should we change config revision here? Seem to be no.
# Do we need to change flag in a one node maintenance?
switch_node_maintenance(True)
cherrypy.engine.publish('failover', False)
shutdown_response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(shutdown_response)}')
return shutdown_response
class ExtentMapController:
def get_brm_bytes(self, element:str):
func_name = 'get_brm_bytes'
log_begin(module_logger, func_name)
node_config = NodeConfig()
result = b''
# there must be sm available
if node_config.s3_enabled():
success = False
retry_count = 0
while not success and retry_count < 10:
module_logger.debug(f'{func_name} returns {element} from S3.')
# TODO: Remove conditional once container dispatcher
# uses non-root by default
if MCSProcessManager.dispatcher_name == 'systemd':
args = [
'su', '-s', '/bin/sh', '-c',
f'smcat {S3_BRM_CURRENT_PATH}', 'mysql'
]
else:
args = ['smcat', S3_BRM_CURRENT_PATH]
ret = subprocess.run(args, stdout=subprocess.PIPE)
if ret.returncode != 0:
module_logger.warning(f"{func_name} got error code {ret.returncode} from smcat, retrying")
time.sleep(1)
retry_count += 1
continue
elem_current_suffix = ret.stdout.decode("utf-8").rstrip()
suffix_for_file = elem_current_suffix
# The journal is always in the current directory, strip trailing A/B from suffix
if element == 'journal' and suffix_for_file.endswith(('A', 'B')):
suffix_for_file = suffix_for_file[:-1]
elem_current_filename = f'{EM_PATH_SUFFIX}/{suffix_for_file}_{element}'
# TODO: Remove conditional once container dispatcher
# uses non-root by default
if MCSProcessManager.dispatcher_name == 'systemd':
args = [
'su', '-s', '/bin/sh', '-c',
f'smcat {elem_current_filename}', 'mysql'
]
else:
args = ['smcat', elem_current_filename]
ret = subprocess.run(args, stdout=subprocess.PIPE)
if ret.returncode != 0:
module_logger.warning(f"{func_name} got error code {ret.returncode} from smcat, retrying")
time.sleep(1)
retry_count += 1
continue
result = ret.stdout
success = True
else:
module_logger.debug(
f'{func_name} returns {element} from local storage.'
)
elem_current_name = Path(MCS_BRM_CURRENT_PATH)
elem_current_filename = elem_current_name.read_text().rstrip()
suffix_for_file = elem_current_filename
# The journal is always in the current directory, strip trailing A/B from suffix
if element == 'journal' and suffix_for_file.endswith(('A', 'B')):
suffix_for_file = suffix_for_file[:-1]
elem_current_file = Path(
f'{MCS_EM_PATH}/{suffix_for_file}_{element}'
)
result = elem_current_file.read_bytes()
module_logger.debug(f'{func_name} returns.')
return result
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@disable_json(input=True, output=True)
def get_em(self):
return self.get_brm_bytes('em')
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@disable_json(input=True, output=True)
def get_journal(self):
return self.get_brm_bytes('journal')
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@disable_json(input=True, output=True)
def get_vss(self):
return self.get_brm_bytes('vss')
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@disable_json(input=True, output=True)
def get_vbbm(self):
return self.get_brm_bytes('vbbm')
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@disable_json(input=True)
def get_footprint(self):
# Dummy footprint
result = {'em': '00f62e18637e1708b080b076ea6aa9b0',
'journal': '00f62e18637e1708b080b076ea6aa9b0',
'vss': '00f62e18637e1708b080b076ea6aa9b0',
'vbbm': '00f62e18637e1708b080b076ea6aa9b0',
}
return result
class ClusterController:
_cp_config = {
"request.methods_with_bodies": ("POST", "PUT", "PATCH", "DELETE")
}
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_start(self):
func_name = 'put_start'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
config = request_body.get('config', DEFAULT_MCS_CONF_PATH)
in_transaction = request_body.get('in_transaction', False)
try:
if not in_transaction:
with TransactionManager():
response = ClusterHandler.start(config)
else:
response = ClusterHandler.start(config)
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_shutdown(self):
func_name = 'put_shutdown'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
timeout = request_body.get('timeout', None)
force = request_body.get('force', False)
config = request_body.get('config', DEFAULT_MCS_CONF_PATH)
in_transaction = request_body.get('in_transaction', False)
try:
if not in_transaction:
with TransactionManager():
response = ClusterHandler.shutdown(config, timeout)
else:
response = ClusterHandler.shutdown(config)
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def start_mariadb(self):
"""Handler for /cluster/start-mariadb (PUT) endpoint."""
func_name = 'put_start_mariadb'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
# TODO: Is transaction really needed here.
timeout = request_body.get('timeout', None)
in_transaction = request_body.get('in_transaction', False)
active_nodes = get_active_nodes()
all_responses: dict = dict()
for node in active_nodes:
logging.debug(f'Starting MariaDB server on "{node}".')
client = NodeControllerClient(
request_timeout=REQUEST_TIMEOUT,
base_url=f'https://{node}:{CMAPI_PORT}'
)
node_response = client.start_mariadb()
logging.debug(f'MariaDB server started on {node}')
all_responses[node] = node_response
response = {
'timestamp': str(datetime.now()),
**all_responses
}
logging.debug(
'Successfully finished starting MariaDB server on all nodes.'
)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def stop_mariadb(self):
"""Handler for /cluster/stop-mariadb (PUT) endpoint."""
func_name = 'put_stop_mariadb'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
# TODO: Is transaction really needed here.
timeout = request_body.get('timeout', None)
in_transaction = request_body.get('in_transaction', False)
active_nodes = get_active_nodes()
all_responses: dict = dict()
for node in active_nodes:
logging.debug(f'Stopping MariaDB server on "{node}".')
client = NodeControllerClient(
request_timeout=REQUEST_TIMEOUT,
base_url=f'https://{node}:{CMAPI_PORT}'
)
node_response = client.stop_mariadb()
logging.debug(f'MariaDB server stopped on {node}')
all_responses[node] = node_response
response = {
'timestamp': str(datetime.now()),
**all_responses
}
logging.debug(
'Successfully finished stopping MariaDB server on all nodes.'
)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_mode_set(self):
func_name = 'put_mode_set'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
mode = request_body.get('mode', 'readonly')
config = request_body.get('config', DEFAULT_MCS_CONF_PATH)
in_transaction = request_body.get('in_transaction', False)
try:
if not in_transaction:
with TransactionManager():
response = ClusterHandler.set_mode(mode, config=config)
else:
response = ClusterHandler.set_mode(mode, config=config)
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_add_node(self):
func_name = 'add_node'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
node = request_body.get('node', None)
config = request_body.get('config', DEFAULT_MCS_CONF_PATH)
in_transaction = request_body.get('in_transaction', False)
read_replica = bool(request_body.get('read_replica', False))
if node is None:
raise_422_error(module_logger, func_name, 'missing node argument')
with cmapi_error_to_422(module_logger, func_name):
if not in_transaction:
with TransactionManager(extra_nodes=[node]):
response = ClusterHandler.add_node(node, config, read_replica)
else:
response = ClusterHandler.add_node(node, config, read_replica)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def delete_remove_node(self):
func_name = 'remove_node'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
node = request_body.get('node', None)
config = request_body.get('config', DEFAULT_MCS_CONF_PATH)
in_transaction = request_body.get('in_transaction', False)
#TODO: add arguments verification decorator
if node is None:
raise_422_error(module_logger, func_name, 'missing node argument')
with cmapi_error_to_422(module_logger, func_name):
if not in_transaction:
with TransactionManager(remove_nodes=[node]):
response = ClusterHandler.remove_node(node, config)
else:
response = ClusterHandler.remove_node(node, config)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_scan_for_attached_dbroots(self):
'''TODO: Based on doc, endpoint not exposed'''
func_name = 'put_scan_for_attached_dbroots'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = cherrypy.request.json
node = request_body.get('node', None)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_failover_master(self):
'''TODO: Based on doc, endpoint not exposed'''
func_name = 'put_failover_master'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = cherrypy.request.json
source = request_body.get('from', None)
dest = request_body.get('to', None)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_move_dbroot(self):
'''TODO: Based on doc, endpoint not exposed'''
func_name = 'put_move_dbroot'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = cherrypy.request.json
source = request_body.get('from', None)
dest = request_body.get('to', None)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_decommission_node(self):
'''TODO: Based on doc, endpoint not exposed'''
func_name = 'put_decommission_node'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = cherrypy.request.json
node = request_body.get('node', None)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def get_status(self):
func_name = 'get_status'
log_begin(module_logger, func_name)
try:
response = ClusterHandler.status()
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
def get_versions(self):
"""Handler for /cluster/versions (GET) endpoint."""
func_name = 'cluster_get_versions'
log_begin(module_logger, func_name)
# Get versions of packages from all active nodes.
# If no active nodes found, get versions from localhost.
active_nodes = get_active_nodes()
active_nodes_count = len(active_nodes)
all_versions: dict = dict()
if not active_nodes:
logging.debug(
'No active nodes found, getting versions from localhost.'
)
active_nodes.append('localhost')
for node in active_nodes:
logging.debug(f'Getting packages versions from "{node}".')
client = NodeControllerClient(
request_timeout=REQUEST_TIMEOUT,
base_url=f'https://{node}:{CMAPI_PORT}'
)
node_versions = client.get_versions()
logging.debug(
f'Node: {node} has installed versions: {node_versions}'
)
all_versions[node] = node_versions
versions_set: set = set()
for versions in all_versions.values():
for version in versions.values():
versions_set.add(version)
if set(versions_set) != set(all_versions[active_nodes[0]].values()):
# Nodes have different versions of packages.
raise_422_error(
logger=module_logger, func_name='get_versions',
err_msg=(
'Nodes have different versions of packages. '
f'Active nodes count: {active_nodes_count}. '
f'Active nodes: {active_nodes}. '
f'Packages versions: {all_versions}'
)
)
response = {
'timestamp': str(datetime.now()),
**all_versions[active_nodes[0]],
}
logging.debug(
'Successfully finished getting package versions from all nodes.'
)
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def install_repo(self):
"""Handler for /cluster/install-repo (PUT) endpoint.
Installs ES repository on all active nodes.
"""
func_name = 'cluster_install_repo'
log_begin(module_logger, func_name)
active_nodes = get_active_nodes()
request = cherrypy.request
request_body = request.json
token = request_body.get('token', None)
mariadb_version = request_body.get('mariadb_version', None)
if not token or not mariadb_version:
raise_422_error(
module_logger, func_name,
'Missing required arguments: token, mariadb_version.'
)
if not active_nodes:
logging.debug(
'No active nodes found, installing repo on localhost.'
)
active_nodes.append('localhost')
all_responses: dict = dict()
for node in active_nodes:
logging.debug(f'Installing repo on "{node}".')
client = NodeControllerClient(
base_url=f'https://{node}:{CMAPI_PORT}'
)
node_response = client.install_repo(
token=token,
mariadb_version=mariadb_version
)
logging.debug(f'ES repo installed on {node}')
all_responses[node] = node_response
response = {
'timestamp': str(datetime.now()),
**all_responses
}
logging.debug(
'Successfully finished installing repo on all nodes.'
)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def preupgrade_backup(self):
"""Handler for /cluster/preupgrade-backup (PUT) endpoint."""
func_name = 'cluster_preupgrade_backup'
log_begin(module_logger, func_name)
active_nodes = get_active_nodes()
all_responses: dict = dict()
for node in active_nodes:
logging.debug(
f'Backuping DBRM and configs before upgrade on "{node}".'
)
client = NodeControllerClient(
base_url=f'https://{node}:{CMAPI_PORT}'
)
node_response = client.preupgrade_backup()
logging.debug(f'PreUpgrade backup completed on {node}')
all_responses[node] = node_response
response = {
'timestamp': str(datetime.now()),
**all_responses
}
logging.debug(
'Successfully finished PreUpgrade backup on all nodes.'
)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def upgrade_mdb_mcs(self):
"""Handler for /cluster/upgrade-mdb-mcs (PUT) endpoint."""
func_name = 'cluster_upgrade_mdb_mcs'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
mdb_version = request_body.get('mariadb_version', None)
mcs_version = request_body.get('columnstore_version', None)
if not mdb_version or not mcs_version:
raise_422_error(
module_logger, func_name,
'Missing required arguments: mdb_version, mcs_version.'
)
active_nodes = get_active_nodes()
all_responses: dict = dict()
for node in active_nodes:
logging.debug(
f'Upgrading MDB and MCS on "{node}".'
)
client = NodeControllerClient(
base_url=f'https://{node}:{CMAPI_PORT}'
)
node_response = client.upgrade_mdb_mcs(
mariadb_version=mdb_version, columnstore_version=mcs_version
)
logging.debug(f'Upgrade MDB and MCS completed on {node}')
all_responses[node] = node_response
response = {
'timestamp': str(datetime.now()),
**all_responses
}
logging.debug(
'Successfully finished upgrading MDB and MCS on all nodes.'
)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def upgrade_cmapi(self):
"""Handler for /cluster/upgrade-cmapi (PUT) endpoint."""
func_name = 'cluster_upgrade_cmapi'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
target_version = request_body.get('version', None)
if not target_version:
raise_422_error(
module_logger, func_name,
'Missing required argument target_version.'
)
active_nodes = get_active_nodes()
all_responses: dict = dict()
for node in active_nodes:
logging.debug(
f'Kicking CMAPI to upgrade on "{node}".'
)
client = NodeControllerClient(
base_url=f'https://{node}:{CMAPI_PORT}'
)
node_response = client.kick_cmapi_upgrade(version=target_version)
all_responses[node] = node_response
response = {
'timestamp': str(datetime.now()),
**all_responses
}
logging.debug(
'Started CMAPI upgrade on all nodes.'
)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def get_health(self):
func_name = 'get_health'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
timeout = request_body.get('timeout', None)
in_transaction = request_body.get('in_transaction', False)
try:
if not in_transaction:
with TransactionManager():
# TODO: just a placeholder for now
# response = ClusterHandler.health()
response = {'status': 'ok'}
else:
# response = ClusterHandler.health()
response = {'status': 'ok'}
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
def set_api_key(self):
"""Handler for /cluster/apikey-set (PUT)
Only for cli tool usage.
"""
func_name = 'cluster_set_api_key'
module_logger.debug('Start setting API key to all nodes in cluster.')
request = cherrypy.request
request_body = request.json
new_api_key = dequote(request_body.get('api_key', ''))
totp_key = request_body.get('verification_key', '')
if not totp_key or not new_api_key:
# not show which arguments in error message because endpoint for
# cli tool or internal usage only
raise_422_error(
module_logger, func_name, 'Missing required arguments.'
)
totp = pyotp.TOTP(SECRET_KEY)
if not totp.verify(totp_key):
raise_422_error(
module_logger, func_name, 'Wrong verification key.'
)
with cmapi_error_to_422(module_logger, func_name):
response = ClusterHandler.set_api_key(new_api_key, totp_key)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
def set_log_level(self):
"""Handler for /cluster/log-level (PUT)
Only for develop purposes.
"""
func_name = 'cluster_set_log_level'
module_logger.debug(
'Start setting new log level to all nodes in cluster.'
)
request = cherrypy.request
request_body = request.json
new_level = request_body.get('level', None)
if not new_level:
raise_422_error(
module_logger, func_name, 'Missing required level argument.'
)
module_logger.info(f'Start setting new logging level "{new_level}".')
try:
response = ClusterHandler.set_log_level(new_level)
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def check_shared_storage(self):
"""Handler for /cluster/check-shared-storage/ (PUT) endpoint."""
func_name = 'check_shared_storage'
log_begin(module_logger, func_name)
# Optional skip list provided by caller (e.g., failover monitor)
request = cherrypy.request
request_body = request.json or {}
skip_nodes = request_body.get('skip_nodes', [])
try:
response = ClusterHandler.check_shared_storage(skip_nodes)
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
except Exception:
raise_422_error(
module_logger, func_name,
'Undefined error happened while checking shared storage.'
)
module_logger.debug(f'{func_name} returns {str(response)}')
return response
class ApiKeyController:
@cherrypy.tools.timeit()
def set_api_key(self):
"""Handler for /node/apikey-set (PUT)
Only for cli tool usage.
"""
func_name = 'node_set_api_key'
module_logger.debug('Start setting new node API key.')
request = cherrypy.request
request_body = request.json
new_api_key = dequote(request_body.get('api_key', ''))
totp_key = request_body.get('verification_key', '')
if not totp_key or not new_api_key:
# not show which arguments in error message because endpoint for
# internal usage only
raise_422_error(
module_logger, func_name, 'Missing required arguments.'
)
totp = pyotp.TOTP(SECRET_KEY)
if not totp.verify(totp_key):
raise_422_error(
module_logger, func_name, 'Wrong verification key.'
)
config_filepath = request.app.config['config']['path']
cmapi_config_check(config_filepath)
cfg_parser = get_config_parser(config_filepath)
config_api_key = get_current_key(cfg_parser)
if config_api_key != new_api_key:
if not cfg_parser.has_section('Authentication'):
cfg_parser.add_section('Authentication')
# TODO: Do not store api key in cherrypy config.
# It causes some overhead on custom ini file and handling it.
# For cherrypy config file values have to be python objects.
# So string have to be quoted.
cfg_parser['Authentication']['x-api-key'] = f"'{new_api_key}'"
save_cmapi_conf_file(cfg_parser, config_filepath)
else:
module_logger.info(
'API key in config file is the same with new one.'
)
# anyway update inmemory api key
request.app.config.update(
{'Authentication': {'x-api-key': new_api_key}}
)
module_logger.info('API key successfully updated.')
return {'timestamp': str(datetime.now())}
class LoggingConfigController:
@cherrypy.tools.timeit()
def set_log_level(self):
"""Handler for /node/log-level (PUT)
Only for develop purposes.
"""
func_name = 'node_put_log_level'
request = cherrypy.request
request_body = request.json
new_level = request_body.get('level', None)
if not new_level:
raise_422_error(
module_logger, func_name, 'Missing required level argument.'
)
module_logger.info(f'Start setting new logging level "{new_level}".')
try:
change_loggers_level(new_level)
except ValueError as exc:
raise_422_error(
module_logger, func_name, str(exc)
)
except Exception:
raise_422_error(
module_logger, func_name, 'Unknown error'
)
module_logger.debug(
f'Finished setting new logging level "{new_level}".'
)
return {'new_level': new_level}
class AppController():
@disable_json(input=True)
def ready(self):
if AppManager.started:
return {'started': True}
else:
raise APIError(503, 'CMAPI not ready to handle requests.')
class NodeProcessController():
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_stop_dmlproc(self):
"""Handler for /node/stop_dmlproc (PUT) endpoint."""
# TODO: make it works only from cli tool like set_api_key made
func_name = 'put_stop_dmlproc'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
timeout = request_body.get('timeout', 10)
force = request_body.get('force', False)
if force:
module_logger.debug(
f'Calling DMLproc to force stop after timeout={timeout}.'
)
MCSProcessManager.stop(
name='DMLProc', is_primary=True, use_sudo=True, timeout=timeout
)
else:
module_logger.debug('Callling stop DMLproc gracefully.')
try:
MCSProcessManager.gracefully_stop_dmlproc()
except (ConnectionRefusedError, RuntimeError):
raise_422_error(
logger=module_logger, func_name=func_name,
err_msg='Couldn\'t stop DMlproc gracefully'
)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
@disable_json(input=True)
def get_process_running(self, process_name):
"""Handler for /node/is_process_running (GET) endpoint."""
func_name = 'get_process_running'
log_begin(module_logger, func_name)
process_running = MCSProcessManager.is_service_running(process_name)
response = {
'timestamp': str(datetime.now()),
'process_name': process_name,
'running': process_running
}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
class NodeController:
@cherrypy.tools.timeit()
def get_versions(self):
"""Handler for /node/versions (GET) endpoint."""
func_name = 'get_node_versions'
log_begin(module_logger, func_name)
columnstore_ver = AppManager.get_columnstore_version()
cmapi_short_ver = AppManager.version
# cmapi version currently is just a part of columnstore version excluding MDB version part
# so canonicalize it
cmapi_ver = columnstore_ver if cmapi_short_ver in columnstore_ver else cmapi_short_ver
node_versions = {
'cmapi_version': cmapi_ver,
'columnstore_version': columnstore_ver,
'server_version': AppManager.get_mdb_version(),
}
response = {
'timestamp': str(datetime.now()),
**node_versions
}
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def latest_mdb_version(self):
"""Handler for /node/latest-mdb-version (GET) endpoint."""
func_name = 'get_latest_mdb_version'
log_begin(module_logger, func_name)
try:
version = MariaDBESRepoManager.get_latest_tested_mdb_version()
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
response = {
'timestamp': str(datetime.now()),
'latest_mdb_version': version
}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def validate_mdb_version(self, token, mariadb_version):
"""Handler for /node/validate-mdb-version (GET) endpoint."""
func_name = 'get_validate_mdb_version'
log_begin(module_logger, func_name)
if not token or not mariadb_version:
raise_422_error(
module_logger, func_name,
'Missing required arguments: token, mariadb_version.'
)
os_name, os_version = AppManager.get_distro_info()
arch = AppManager.get_architecture()
repo_manager = MariaDBESRepoManager(
token=token, arch=arch, os_type=os_name, os_version=os_version,
mariadb_version=mariadb_version
)
try:
repo_manager.check_mdb_version_exists()
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def validate_es_token(self, token):
"""Handler for /node/validate-es-token (GET) endpoint."""
func_name = 'get_validate_es_token'
log_begin(module_logger, func_name)
if not token:
raise_422_error(
module_logger, func_name,
'Missing required argument token.'
)
try:
MariaDBESRepoManager.verify_token(token)
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def start_mariadb(self):
"""Handler for /node/start_mariadb (PUT) endpoint."""
func_name = 'node_start_mariadb'
log_begin(module_logger, func_name)
req = cherrypy.request
use_sudo = get_use_sudo(req.app.config)
try:
MDBProcessManager.start(use_sudo=use_sudo)
except CMAPIBasicError as err:
raise_422_error(
module_logger, func_name,
(
'Error while starting mariadb process. '
f'Details: {err.message}'
),
exc_info=False
)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def stop_mariadb(self):
"""Handler for /node/stop_mariadb (PUT) endpoint."""
func_name = 'node_stop_mariadb'
log_begin(module_logger, func_name)
req = cherrypy.request
use_sudo = get_use_sudo(req.app.config)
try:
MDBProcessManager.stop(use_sudo=use_sudo)
except CMAPIBasicError as err:
raise_422_error(
module_logger, func_name,
(
'Error while stopping mariadb process. '
f'Details: {err.message}'
),
exc_info=False
)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def install_repo(self):
"""Handler for /node/install-repo (PUT) endpoint."""
func_name = 'node_install_repo'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
token = request_body.get('token', None)
mariadb_version = request_body.get('mariadb_version', None)
if not token or not mariadb_version:
raise_422_error(
module_logger, func_name,
'Missing required arguments: token, mariadb_version.'
)
os_name, os_version = AppManager.get_distro_info()
arch = AppManager.get_architecture()
repo_manager = MariaDBESRepoManager(
token=token, arch=arch, os_type=os_name, os_version=os_version,
mariadb_version=mariadb_version
)
try:
repo_manager.setup_repo()
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def repo_pkg_versions(self):
"""Handler for /node/repo-pkg-versions (GET) endpoint."""
func_name = 'get_repo_pkg_versions'
log_begin(module_logger, func_name)
os_name, _ = AppManager.get_distro_info()
mdb_pkg_name: str
mcs_pkg_name: str
cmapi_pkg_name: str
if os_name in ['ubuntu', 'debian']:
mdb_pkg_name = MDB_SERVER_PACKAGE_NAME.deb
mcs_pkg_name = MDB_CS_PACKAGE_NAME.deb
cmapi_pkg_name = CMAPI_PACKAGE_NAME.deb
elif os_name in ['centos', 'rhel', 'rocky']:
mdb_pkg_name = MDB_SERVER_PACKAGE_NAME.rhel
mcs_pkg_name = MDB_CS_PACKAGE_NAME.rhel
cmapi_pkg_name = CMAPI_PACKAGE_NAME.rhel
else:
raise_422_error(
module_logger, func_name, f'Unsupported OS type: {os_name}'
)
try:
repo_versions = {
'cmapi_version': MariaDBESRepoManager.get_ver_of(
cmapi_pkg_name, os_name
),
'columnstore_version': MariaDBESRepoManager.get_ver_of(
mcs_pkg_name, os_name
),
'server_version': MariaDBESRepoManager.get_ver_of(
mdb_pkg_name, os_name
),
}
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
response = {
'timestamp': str(datetime.now()),
**repo_versions
}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def preupgrade_backup(self):
"""Handler for /node/preupgrade-backup (PUT) endpoint."""
func_name = 'node_preupgrade_backup'
log_begin(module_logger, func_name)
os_name, _ = AppManager.get_distro_info()
try:
PreUpgradeBackupRestoreManager.backup_dbrm()
PreUpgradeBackupRestoreManager.backup_configs(distro_name=os_name)
except CMAPIBasicError as err:
raise_422_error(
module_logger, func_name,
f'Error while PreUpgrade backup. Details: {err.message}',
exc_info=False
)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def upgrade_mdb_mcs(self):
"""Handler for /node/upgrade-mdb-mcs (PUT) endpoint."""
func_name = 'node_upgrade_mdb_mcs'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
mdb_version = request_body.get('mariadb_version', None)
mcs_version = request_body.get('columnstore_version', None)
if not mdb_version or not mcs_version:
raise_422_error(
module_logger, func_name,
'Missing required arguments: mdb_version, mcs_version.'
)
os_name, _ = AppManager.get_distro_info()
try:
packages_manager = PackagesManager(
os_name=os_name, mdb_version=mdb_version,
mcs_version=mcs_version
)
packages_manager.upgrade_mdb_and_mcs()
except CMAPIBasicError as err:
raise_422_error(
module_logger, func_name,
(
'Error while Upgrading MDB and MCS packages. '
f'Details: {err.message}'
),
exc_info=False
)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def kick_cmapi_upgrade(self):
"""Handler for /node/kick-cmapi-upgrade (PUT) endpoint."""
func_name = 'node_kick_cmapi_upgrade'
log_begin(module_logger, func_name)
request = cherrypy.request
request_body = request.json
target_version = request_body.get('version', None)
if target_version is None:
raise_422_error(
module_logger, func_name, 'Missing required version argument.'
)
try:
PackagesManager.kick_cmapi_upgrade(cmapi_version=target_version)
except CMAPIBasicError as err:
raise_422_error(module_logger, func_name, err.message)
response = {'timestamp': str(datetime.now())}
module_logger.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def check_shared_file(self, file_path, check_sum):
func_name = 'check_shared_file'
log_begin(module_logger, func_name)
ACCEPTED_PATHS = (
'/var/lib/columnstore/data1/',
'/var/lib/columnstore/storagemanager/metadata/data1/'
)
if not file_path.startswith(ACCEPTED_PATHS):
raise_422_error(module_logger, func_name, 'Not acceptable file_path.')
success = True
file_path_obj = Path(file_path)
logging.debug(f'Checking shared file at {file_path} with md5 {check_sum}.')
if not file_path_obj.exists():
success = False
logging.debug(f'Shared file {file_path} does not exist.')
else:
with file_path_obj.open(mode='rb') as file_to_check:
data = file_to_check.read()
calculated_md5 = hashlib.md5(data).hexdigest()
if calculated_md5 != check_sum:
logging.debug(
f'Shared file at {file_path} md5 {calculated_md5} does not match given md5 {check_sum}.'
)
success = False
if success:
logging.debug(f'Shared file {file_path} md5 matches {check_sum}.')
response = {
'timestamp': str(datetime.now()),
'success': success
}
logging.debug(f'{func_name} returns {str(response)}')
return response
@cherrypy.tools.timeit()
@cherrypy.tools.validate_api_key() # pylint: disable=no-member
def put_stateful_config(self):
"""Handler for /node/stateful-config (PUT) endpoint.
#TODO: for next releases.
"""
func_name = 'put_stateful_config'
log_begin(module_logger, func_name)
request_body = cherrypy.request.json
try:
request_stateful_config = StatefulConfigModel.model_validate(
request_body.get('stateful_config_dict')
)
except ValidationError as exp:
raise_422_error(module_logger, func_name,f'Invalid request body: {exp.errors()}')
success = AppStatefulConfig.apply_update(request_stateful_config)
if not success:
logging.info('Stateful config update was stale.')
else:
logging.info(
f'Stateful config updated with term {request_stateful_config.version.term} '
f'and seq {request_stateful_config.version.seq}.'
)
return {'timestamp': str(datetime.now()), 'success': success}
View Git Blame Copy Permalink