feat(upgrade): MCOL-6028 upgrade MVP with repo-managed flow, prechecks, and other enhancements

Implements the initial upgrade capability across CMAPI and the CLI, including
repository setup, package operations, environment prechecks, and coordinated
cluster steps with progress reporting.

Details:
- CMAPI upgrade manager:
  - Add `cmapi/cmapi_server/managers/upgrade/` modules:
    - `repo.py`, `packages.py`, `preinstall.py`, `upgrade.py`, `utils.py` and `__init__.py`
  - Extend endpoints and routing to expose upgrade operations and status:
    - `cmapi_server/controllers/{endpoints.py, dispatcher.py, api_clients.py}`
    - `cmapi_server/managers/{application.py, process.py}`
    - Add improved constants and helpers for upgrade flow
- Backup/restore and safety:
  - Add `cmapi_server/managers/backup_restore.py`
  - Fix pre-upgrade backup regressions (due to `mcs_backup_manager.sh 3.17 changes`)
  - Improve cluster version validation; add `ignore_missmatch` override
- CLI enhancements:
  - Progress UI and richer feedback (`mcs_cluster_tool/tools_commands.py`, `README.md`, `mcs.1`)
  - Add steps to start MDB and start MCS during/after upgrade
  - Improved error surfacing for version validation
- Platform and packaging:
  - Ubuntu and Rocky Linux support
  - RHEL/DNF dry-run support
  - Distro detection and platform-dependent logic hardened
  - Logging improvements
- Updater service:
  - Add `cmapi/updater/cmapi_updater.service.template` and `cmapi_updater.sh` to make CMAPI update itself
- Docs:
  - Update mcs cli README and mcs.1 man file
  - Add `cmapi/updater/README.md`

cmapi/CMakeLists.txt

@@ -74,6 +74,7 @@ configure_file(conffiles.template conffiles)
 configure_file(mcs.template mcs)
 configure_file(mcs_aws.template mcs_aws)
 configure_file(mcs_gsutil.template mcs_gsutil)
+configure_file(updater/cmapi_updater.service.template cmapi_updater.service)
 
 install(
     DIRECTORY python
@@ -97,7 +98,7 @@ install(
     DESTINATION ${CMAPI_DIR}
 )
 install(FILES cmapi_server/cmapi_server.conf systemd.env DESTINATION ${ETC_DIR})
-install(FILES ${SYSTEMD_UNIT_NAME}.service DESTINATION ${SYSTEMD_UNIT_DIR})
+install(FILES ${SYSTEMD_UNIT_NAME}.service cmapi_updater.service DESTINATION ${SYSTEMD_UNIT_DIR})
 install(
     FILES mcs
     PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
@@ -118,8 +119,14 @@ install(
     PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
     DESTINATION ${BIN_DIR}
 )
+install(
+    FILES updater/cmapi_updater.sh
+    PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
+    DESTINATION ${BIN_DIR}
+)
 install(FILES mcs_cluster_tool/mcs.1 DESTINATION ${MAN_DIR})
 
+
 option(RPM "Build an RPM" OFF)
 if(RPM)
     set(CPACK_GENERATOR "RPM")

cmapi/cmapi_server/constants.py

@@ -3,6 +3,7 @@
 TODO: move main constant paths here and replace in files in next releases.
 """
 import os
+from dataclasses import dataclass
 from enum import Enum
 from typing import NamedTuple
 
@@ -106,9 +107,72 @@ MCS_LOG_PATH = '/var/log/mariadb/columnstore'
 
 # BRM shmem lock inspection/reset tool
 SHMEM_LOCKS_PATH = os.path.join(MCS_INSTALL_BIN, 'mcs-shmem-locks')
+# mcs and cmapi constanst shared
+MCS_BACKUP_MANAGER_SH = os.path.join(MCS_INSTALL_BIN, 'mcs_backup_manager.sh')
 
 # client constants
 CMAPI_PORT = 8640  #TODO: use it in all places
 CURRENT_NODE_CMAPI_URL = f'https://localhost:{CMAPI_PORT}'
 REQUEST_TIMEOUT: float = 30.0
-TRANSACTION_TIMEOUT: float = 300.0  # 5 minutes
+TRANSACTION_TIMEOUT: float = 300.0  # 5 minutes
+
+# API version
+_version = '0.4.0'
+
+# constants for packages and repositories
+SUPPORTED_DISTROS = (
+    'ubuntu',
+    'debian',
+    'centos',
+    'rhel',
+    'rocky',
+)
+SUPPORTED_ARCHITECTURES = ('x86_64', 'amd64', 'aarch64', 'arm64')
+
+@dataclass(frozen=True)
+class MultiDistroNamer:
+    rhel: str
+    deb: str
+
+MDB_SERVER_PACKAGE_NAME = MultiDistroNamer(
+    rhel='MariaDB-server',
+    deb='mariadb-server'
+)
+MDB_CS_PACKAGE_NAME = MultiDistroNamer(
+    rhel='MariaDB-columnstore-engine',
+    deb='mariadb-plugin-columnstore'
+)
+CMAPI_PACKAGE_NAME = MultiDistroNamer(
+    rhel='MariaDB-columnstore-cmapi',
+    deb='mariadb-columnstore-cmapi'
+)
+ES_REPO = MultiDistroNamer(
+    rhel=(
+'''[mariadb-es-main]
+name = MariaDB Enterprise Server
+baseurl = https://dlm.mariadb.com/repo/{token}/mariadb-enterprise-server/{mdb_version}/rpm/rhel/{os_major_version}/{arch}
+gpgkey = {gpg_key_url}
+gpgcheck = 1
+enabled = 1
+module_hotfixes = 1
+'''
+    ),
+    deb='deb [arch=amd64,arm64] https://dlm.mariadb.com/repo/{token}/mariadb-enterprise-server/{mdb_version}/deb {os_version} main'
+)
+ES_REPO_PRIORITY_PREFS = '''
+Package: *
+Pin: origin dlm.mariadb.com
+Pin-Priority: 1700
+'''
+ES_VERIFY_URL = MultiDistroNamer(
+    rhel='https://dlm.mariadb.com/repo/{token}/mariadb-enterprise-server/{mdb_version}/rpm/rhel/{os_major_version}/{arch}/repodata/repomd.xml',
+    deb='https://dlm.mariadb.com/repo/{token}/mariadb-enterprise-server/{mdb_version}/deb/dists/{os_version}/Release'
+)
+MDB_GPG_KEY_URL = 'https://supplychain.mariadb.com/MariaDB-Enterprise-GPG-KEY'
+ES_TOKEN_VERIFY_URL = 'https://dlm.mariadb.com/browse/{token}/mariadb_enterprise_server/'
+MDB_LATEST_TESTED_MAJOR = '10.6'
+MDB_LATEST_RELEASES_URL = 'https://dlm.mariadb.com/rest/releases/mariadb_enterprise_server/'
+PKG_GET_VER_CMD = MultiDistroNamer(
+    rhel="rpm -q --queryformat '%{{VERSION}}' {package_name}",
+    deb="dpkg-query -f '${{Version}}' -W {package_name}"
+)

cmapi/cmapi_server/controllers/api_clients.py

@@ -5,26 +5,29 @@ import pyotp
 import requests
 
 from cmapi_server.constants import (
-    CMAPI_CONF_PATH,
+    CMAPI_CONF_PATH, CURRENT_NODE_CMAPI_URL, SECRET_KEY, _version
-    CURRENT_NODE_CMAPI_URL,
-    SECRET_KEY,
 )
-from cmapi_server.controllers.dispatcher import _version
 from cmapi_server.exceptions import CMAPIBasicError
 from cmapi_server.helpers import get_config_parser, get_current_key
 from tracing.traced_session import get_traced_session
 
 
-class ClusterControllerClient:
+class BaseClient:
+    """Base class for API clients.
+
+    This class is not intended to be used directly, but rather as a
+    base class for other API clients. It provides a common interface
+    for making requests to the API and handling responses.
+    WARNING: This class only handles the API requests, it does not
+             handle the transaction management. So it should be started
+             at level above using TransactionManager (decorator or context
+             manager).
+    """
     def __init__(
             self, base_url: str = CURRENT_NODE_CMAPI_URL,
             request_timeout: Optional[float] = None
     ):
-        """Initialize the ClusterControllerClient with the base URL.
+        """Initialize the BaseClient with the base URL.
-
-        WARNING: This class only handles the API requests, it does not
-        handle the transaction management. So it should be started
-        at level above using TransactionManager (decorator or context manager).
 
         :param base_url: The base URL for the API endpoints,
                          defaults to CURRENT_NODE_CMAPI_URL
@@ -34,6 +37,99 @@ class ClusterControllerClient:
         """
         self.base_url = base_url
         self.request_timeout = request_timeout
+        self.cmd_url = None
+
+    def _request(
+        self, method: str, endpoint: str,
+        data: Optional[Dict[str, Any]] = None,
+        throw_real_exp: bool = False
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Make a request to the API.
+
+        :param method: The HTTP method to use.
+        :param endpoint: The API endpoint to call.
+        :param data: The data to send with the request.
+        :return: The response from the API.
+        """
+        url = f'{self.cmd_url}/{endpoint}'
+        cmapi_cfg_parser = get_config_parser(CMAPI_CONF_PATH)
+        key = get_current_key(cmapi_cfg_parser)
+        headers = {'x-api-key': key}
+        if method in ['PUT', 'POST', 'DELETE']:
+            headers['Content-Type'] = 'application/json'
+            data = {'in_transaction': True, **(data or {})}
+        try:
+            response = requests.request(
+                method, url, headers=headers,
+                params=data if method == 'GET' else None,
+                json=data if method in ('PUT', 'POST') else None,
+                timeout=self.request_timeout, verify=False
+            )
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.ConnectionError as exc:
+            message = (
+                f'API client could not connect to {url}. '
+                'Is cmapi server running and reachable?'
+            )
+            logging.error(message)
+            if throw_real_exp:
+                raise exc
+            raise CMAPIBasicError(message)
+        # TODO: different handler for timeout exception?
+        except requests.exceptions.HTTPError as exc:
+            resp = exc.response
+            error_msg = str(exc)
+            if resp.status_code == 422:
+                # in this case we think cmapi server returned some value but
+                # had error during running endpoint handler code
+                try:
+                    resp_json = resp.json()
+                    error_msg = resp_json.get('error', resp_json)
+                except requests.exceptions.JSONDecodeError:
+                    error_msg = resp.text
+            message = (
+                f'API client got an exception in request to {exc.request.url} '
+                f'with code {resp.status_code if resp is not None else "NA"} '
+                f'and error: {error_msg}'
+            )
+            logging.error(message)
+            if throw_real_exp:
+                raise exc
+            raise CMAPIBasicError(message)
+        except requests.exceptions.RequestException as exc:
+            status_code = exc.response.status_code if exc.response else 'NA'
+            message = (
+                'API client got an undefined error in request to '
+                f'{exc.request.url} with code {status_code!r} and '
+                f'error: {str(exc)}'
+            )
+            logging.error(message)
+            if throw_real_exp:
+                raise exc
+            raise CMAPIBasicError(message)
+
+
+class ClusterControllerClient(BaseClient):
+    """Client for the ClusterController API.
+    This class provides methods for interacting with the cluster
+    management API, including starting and stopping the cluster,
+    adding and removing nodes, and getting the cluster status.
+    """
+    def __init__(
+            self, base_url: str = CURRENT_NODE_CMAPI_URL,
+            request_timeout: Optional[float] = None
+    ):
+        """Initialize the ClusterControllerClient with the base URL.
+
+        :param base_url: The base URL for the API endpoints,
+                         defaults to CURRENT_NODE_CMAPI_URL
+        :type base_url: str, optional
+        :param request_timeout: request timeout, defaults to None
+        :type request_timeout: Optional[float], optional
+        """
+        super().__init__(base_url, request_timeout)
+        self.cmd_url = f'{self.base_url}/cmapi/{_version}/cluster'
 
     def start_cluster(
             self, extra: Dict[str, Any] = dict()
@@ -68,7 +164,7 @@ class ClusterControllerClient:
     ) -> Union[Dict[str, Any], Dict[str, str]]:
         """Add a node to the cluster.
 
-        :param node_info: Information about the node to add.
+        :param node_info: Information about a node to add.
         :return: The response from the API.
         """
         #TODO: fix interface as in remove_node used or think about universal
@@ -91,6 +187,16 @@ class ClusterControllerClient:
         """
         return self._request('GET', 'status')
 
+    def get_health(
+            self, extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """
+        Get the health of the cluster.
+
+        :return: The response from the API.
+        """
+        return self._request('GET', 'health', extra)
+
     def set_api_key(
             self, api_key: str
     ) -> Union[Dict[str, Any], Dict[str, str]]:
@@ -104,7 +210,7 @@ class ClusterControllerClient:
             'api_key': api_key,
             'verification_key': totp.now()
         }
-        return self._request('put', 'apikey-set', payload)
+        return self._request('PUT', 'apikey-set', payload)
 
     def set_log_level(
             self, log_level: str
@@ -114,7 +220,7 @@ class ClusterControllerClient:
         :param log_level: The log level to set.
         :return: The response from the API.
         """
-        return self._request('put', 'log-level', {'log_level': log_level})
+        return self._request('PUT', 'log-level', {'log_level': log_level})
 
     def load_s3data(
             self, s3data_info: Dict[str, Any]
@@ -124,58 +230,263 @@ class ClusterControllerClient:
         :param s3data_info: Information about the S3 data to load.
         :return: The response from the API.
         """
-        return self._request('put', 'load_s3data', s3data_info)
+        return self._request('PUT', 'load_s3data', s3data_info)
 
-    def _request(
+    def get_versions(
-            self, method: str, endpoint: str,
+            self, extra: Dict[str, Any] = dict()
-            data: Optional[Dict[str, Any]] = None
     ) -> Union[Dict[str, Any], Dict[str, str]]:
-        """Make a request to the API.
+        """Get packages versions.
-
-        :param method: The HTTP method to use.
-        :param endpoint: The API endpoint to call.
-        :param data: The data to send with the request.
         :return: The response from the API.
         """
-        url = f'{self.base_url}/cmapi/{_version}/cluster/{endpoint}'
+        return self._request('GET', 'versions', extra)
-        cmapi_cfg_parser = get_config_parser(CMAPI_CONF_PATH)
+
-        key = get_current_key(cmapi_cfg_parser)
+    def start_mariadb(
-        headers = {'x-api-key': key}
+            self, extra: Dict[str, Any] = dict()
-        if method in ['PUT', 'POST', 'DELETE']:
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
-            headers['Content-Type'] = 'application/json'
+        """Start MariaDB server service on each node in cluster.
-            data = {'in_transaction': True, **(data or {})}
+
-        try:
+        :return: The response from the API.
-            response = get_traced_session().request(
+        """
-                method, url, headers=headers, json=data,
+        return self._request('PUT', 'start-mariadb', extra)
-                timeout=self.request_timeout, verify=False
+
-            )
+    def stop_mariadb(
-            response.raise_for_status()
+            self, extra: Dict[str, Any] = dict()
-            return response.json()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
-        # TODO: different handler for timeout exception?
+        """Stop MariaDB server service on each node in cluster.
-        except requests.HTTPError as exc:
+
-            resp = exc.response
+        :return: The response from the API.
-            error_msg = str(exc)
+        """
-            if resp is not None and resp.status_code == 422:
+        return self._request('PUT', 'stop-mariadb', extra)
-                # in this case we think cmapi server returned some value but
+
-                # had error during running endpoint handler code
+    def install_repo(
-                try:
+            self, token: str, mariadb_version: str,
-                    resp_json = resp.json()
+            extra: Dict[str, Any] = dict()
-                    error_msg = resp_json.get('error', resp_json)
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
-                except requests.exceptions.JSONDecodeError:
+        """Install ES repository on each node in cluster.
-                    error_msg = resp.text
+
-            message = (
+        :return: The response from the API.
-                f'API client got an exception in request to {exc.request.url if exc.request else url} '
+        """
-                f'with code {resp.status_code if resp is not None else "?"} and error: {error_msg}'
+        data = {
-            )
+            'token': token,
-            logging.error(message)
+            'mariadb_version': mariadb_version
-            raise CMAPIBasicError(message)
+        }
-        except requests.exceptions.RequestException as exc:
+        return self._request('PUT', 'install-repo', {**data, **extra})
-            request_url = getattr(exc.request, 'url', url)
+
-            response_status = getattr(getattr(exc, 'response', None), 'status_code', '?')
+    def preupgrade_backup(
-            message = (
+            self, extra: Dict[str, Any] = dict()
-                'API client got an undefined error in request to '
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
-                f'{request_url} with code {response_status} and '
+        """Backup DBRM and configs on each node in cluster.
-                f'error: {str(exc)}'
+
-            )
+        :return: The response from the API.
-            logging.error(message)
+        """
-            raise CMAPIBasicError(message)
+        return self._request('PUT', 'preupgrade-backup', extra)
+
+    def upgrade_mdb_mcs(
+            self, mariadb_version: str, columnstore_version: str,
+            extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Upgrade MariaDB and Columnstore on each node in cluster.
+
+        :return: The response from the API.
+        """
+        data = {
+            'mariadb_version': mariadb_version,
+            'columnstore_version': columnstore_version
+        }
+        return self._request('PUT', 'upgrade-mdb-mcs', {**data, **extra})
+
+    def upgrade_cmapi(
+            self, version: str,
+            extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Starts cmapi_updater.service on each node and waits for new cmapi.
+
+        :return: The response from the API.
+        """
+        return self._request(
+            'PUT', 'upgrade-cmapi', {'version': version, **extra}
+        )
+
+
+class NodeControllerClient(BaseClient):
+    """Client for the NodeController API.
+
+    This class provides methods for interacting with a node management
+    API.
+    """
+    def __init__(
+            self, base_url: str = CURRENT_NODE_CMAPI_URL,
+            request_timeout: Optional[float] = None
+    ):
+        """Initialize the NodeControllerClient with the base URL.
+
+        :param base_url: The base URL for the API endpoints,
+                         defaults to CURRENT_NODE_CMAPI_URL
+        :type base_url: str, optional
+        :param request_timeout: request timeout, defaults to None
+        :type request_timeout: Optional[float], optional
+        """
+        super().__init__(base_url, request_timeout)
+        self.cmd_url = f'{self.base_url}/cmapi/{_version}/node'
+
+    def get_versions(
+            self, extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Get packages versions installed on a node.
+
+        :return: The response from the API.
+        """
+        return self._request('GET', 'versions', extra)
+
+    def get_latest_mdb_version(
+        self, extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Get latest tested MDB version from repo.
+
+        :return: The response from the API.
+        """
+        return self._request('GET', 'latest-mdb-version', extra)
+
+    def validate_mdb_version(
+            self, token: str, mariadb_version: str,
+            extra: Dict[str, Any] = dict(),
+            **kwargs
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Verify MariaDB ES version.
+
+        :param token: valid ES token
+        :type token: str
+        :param mariadb_version: MariaDB version to verify
+        :type mariadb_version: str
+        :return: The response from the API
+        :rtype: Union[Dict[str, Any], Dict[str, str]]
+        """
+        data = {
+            'token': token,
+            'mariadb_version': mariadb_version
+        }
+        return self._request('GET', 'validate-mdb-version', {**data, **extra}, **kwargs)
+
+    def validate_es_token(
+            self, token: str,
+            extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Verify MariaDB ES token is correct.
+
+        :param token: ES token to verify
+        :type token: str
+        :return: The response from the API.
+        """
+        return self._request(
+            'GET', 'validate-es-token', {'token': token, **extra}
+        )
+
+    def start_mariadb(
+            self, extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Start MariaDB-server service on a node.
+
+        :return: The response from the API.
+        """
+        return self._request('PUT', 'start-mariadb', extra)
+
+    def stop_mariadb(
+            self, extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Stop MariaDB-server service on a node.
+
+        :return: The response from the API.
+        """
+        return self._request('PUT', 'stop-mariadb', extra)
+
+    def repo_pkg_versions(
+            self, extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Get available packages versions from the repo on a node.
+
+        :return: The response from the API.
+        """
+        return self._request('GET', 'repo-pkg-versions', extra)
+
+    def install_repo(
+            self, token: str, mariadb_version: str,
+            extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Install the repository on a node.
+
+        :param token: valid ES token
+        :type token: str
+        :param mariadb_version: MariaDB version to verify
+        :type mariadb_version: str
+        :return: The response from the API.
+        """
+        data = {
+            'token': token,
+            'mariadb_version': mariadb_version
+        }
+        return self._request('PUT', 'install-repo', {**data, **extra})
+
+
+    def preupgrade_backup(
+            self, extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Backup DBRM and configs on a node.
+
+        :return: The response from the API.
+        """
+        return self._request('PUT', 'preupgrade-backup', extra)
+
+    def upgrade_mdb_mcs(
+            self, mariadb_version: str, columnstore_version: str,
+            extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Upgrade MariaDB and Columnstore on a node.
+
+        :return: The response from the API.
+        """
+        data = {
+            'mariadb_version': mariadb_version,
+            'columnstore_version': columnstore_version
+        }
+        return self._request('PUT', 'upgrade-mdb-mcs', {**data, **extra})
+
+    def kick_cmapi_upgrade(
+            self, version: str,
+            extra: Dict[str, Any] = dict()
+    ) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Starting cmapi_updater.service on a node.
+
+        :return: The response from the API.
+        """
+        return self._request(
+            'PUT', 'kick-cmapi-upgrade', {'version': version, **extra}
+        )
+
+
+class AppControllerClient(BaseClient):
+    """Client for the AppController API.
+
+    This class provides methods for interacting with a cmapi special management
+    API.
+    """
+    def __init__(
+            self, base_url: str = CURRENT_NODE_CMAPI_URL,
+            request_timeout: Optional[float] = None
+    ):
+        """Initialize the NodeControllerClient with the base URL.
+
+        :param base_url: The base URL for the API endpoints,
+                         defaults to CURRENT_NODE_CMAPI_URL
+        :type base_url: str, optional
+        :param request_timeout: request timeout, defaults to None
+        :type request_timeout: Optional[float], optional
+        """
+        super().__init__(base_url, request_timeout)
+        self.cmd_url = f'{self.base_url}/cmapi/'
+
+    def get_ready(self) -> Union[Dict[str, Any], Dict[str, str]]:
+        """Get CMAPI ready or not.
+
+        :return: The response from the API.
+        """
+        return self._request('GET', 'ready', None, throw_real_exp=True)

cmapi/cmapi_server/controllers/dispatcher.py

@@ -3,16 +3,16 @@ import logging
 
 import cherrypy
 
+from cmapi_server.constants import _version
 from cmapi_server.controllers.endpoints import (
-    StatusController, ConfigController, BeginController, CommitController,
+    ApiKeyController, AppController, BeginController, ClusterController,
-    RollbackController, StartController, ShutdownController,
+    CommitController, ConfigController, ExtentMapController,
-    ExtentMapController, ClusterController, ApiKeyController,
+    LoggingConfigController, NodeController, NodeProcessController,
-    LoggingConfigController, AppController, NodeProcessController
+    RollbackController, ShutdownController, StartController, StatusController,
 )
-
 from cmapi_server.controllers.s3dataload import S3DataLoadController
 
-_version = '0.4.0'
+
 dispatcher = cherrypy.dispatch.RoutesDispatcher()
 logger = logging.getLogger(__name__)
 
@@ -253,7 +253,7 @@ dispatcher.connect(
 )
 
 
-# /_version/node/is_process_running/ (PUT)
+# /_version/node/is_process_running/ (GET)
 dispatcher.connect(
     name = 'is_process_running',
     route = f'/cmapi/{_version}/node/is_process_running',
@@ -263,6 +263,196 @@ dispatcher.connect(
 )
 
 
+# /_version/cluster/health/ (PUT)
+dispatcher.connect(
+    name = 'cluster_get_health',
+    route = f'/cmapi/{_version}/cluster/health',
+    action = 'get_health',
+    controller = ClusterController(),
+    conditions = {'method': ['GET']}
+)
+
+
+# /_version/node/versions (GET)
+dispatcher.connect(
+    name = 'node_get_versions',
+    route = f'/cmapi/{_version}/node/versions',
+    action = 'get_versions',
+    controller = NodeController(),
+    conditions = {'method': ['GET']}
+)
+
+
+# /_version/cluster/version (GET)
+dispatcher.connect(
+    name = 'cluster_get_versions',
+    route = f'/cmapi/{_version}/cluster/versions',
+    action = 'get_versions',
+    controller = ClusterController(),
+    conditions = {'method': ['GET']}
+)
+
+
+# /_version/node/latest-mdb-version (GET)
+dispatcher.connect(
+    name = 'get_latest_mdb_version',
+    route = f'/cmapi/{_version}/node/latest-mdb-version',
+    action = 'latest_mdb_version',
+    controller = NodeController(),
+    conditions = {'method': ['GET']}
+)
+
+
+# /_version/node/validate-mdb-version (GET)
+dispatcher.connect(
+    name = 'get_validate_mdb_version',
+    route = f'/cmapi/{_version}/node/validate-mdb-version',
+    action = 'validate_mdb_version',
+    controller = NodeController(),
+    conditions = {'method': ['GET']}
+)
+
+
+# /_version/node/validate-es-token (GET)
+dispatcher.connect(
+    name = 'get_validate_es_token',
+    route = f'/cmapi/{_version}/node/validate-es-token',
+    action = 'validate_es_token',
+    controller = NodeController(),
+    conditions = {'method': ['GET']}
+)
+
+
+# /_version/node/stop-mariadb (PUT)
+dispatcher.connect(
+    name = 'node_stop_mariadb',
+    route = f'/cmapi/{_version}/node/stop-mariadb',
+    action = 'stop_mariadb',
+    controller = NodeController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/cluster/stop-mariadb (PUT)
+dispatcher.connect(
+    name = 'cluster_stop_mariadb',
+    route = f'/cmapi/{_version}/cluster/stop-mariadb',
+    action = 'stop_mariadb',
+    controller = ClusterController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/node/start-mariadb (PUT)
+dispatcher.connect(
+    name = 'node_start_mariadb',
+    route = f'/cmapi/{_version}/node/start-mariadb',
+    action = 'start_mariadb',
+    controller = NodeController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/cluster/start-mariadb (PUT)
+dispatcher.connect(
+    name = 'cluster_start_mariadb',
+    route = f'/cmapi/{_version}/cluster/start-mariadb',
+    action = 'start_mariadb',
+    controller = ClusterController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/node/install-repo (PUT)
+dispatcher.connect(
+    name = 'node_install_repo',
+    route = f'/cmapi/{_version}/node/install-repo',
+    action = 'install_repo',
+    controller = NodeController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/cluster/install-repo (PUT)
+dispatcher.connect(
+    name = 'cluster_install_repo',
+    route = f'/cmapi/{_version}/cluster/install-repo',
+    action = 'install_repo',
+    controller = ClusterController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/node/repo-pkg-versions (GET)
+dispatcher.connect(
+    name = 'get_repo_pkg_versions',
+    route = f'/cmapi/{_version}/node/repo-pkg-versions',
+    action = 'repo_pkg_versions',
+    controller = NodeController(),
+    conditions = {'method': ['GET']}
+)
+
+
+# /_version/node/preupgrade-backup (PUT)
+dispatcher.connect(
+    name = 'node_preupgrade_backup',
+    route = f'/cmapi/{_version}/node/preupgrade-backup',
+    action = 'preupgrade_backup',
+    controller = NodeController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/cluster/preupgrade-backup (PUT)
+dispatcher.connect(
+    name = 'cluster_preupgrade_backup',
+    route = f'/cmapi/{_version}/cluster/preupgrade-backup',
+    action = 'preupgrade_backup',
+    controller = ClusterController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/node/upgrade-mdb-mcs (PUT)
+dispatcher.connect(
+    name = 'node_upgrade_mdb_mcs',
+    route = f'/cmapi/{_version}/node/upgrade-mdb-mcs',
+    action = 'upgrade_mdb_mcs',
+    controller = NodeController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/cluster/upgrade-mdb-mcs (PUT)
+dispatcher.connect(
+    name = 'cluster_upgrade_mdb_mcs',
+    route = f'/cmapi/{_version}/cluster/upgrade-mdb-mcs',
+    action = 'upgrade_mdb_mcs',
+    controller = ClusterController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/node/kick-cmapi-upgrade (PUT)
+dispatcher.connect(
+    name = 'node_kick_cmapi_upgrade',
+    route = f'/cmapi/{_version}/node/kick-cmapi-upgrade',
+    action = 'kick_cmapi_upgrade',
+    controller = NodeController(),
+    conditions = {'method': ['PUT']}
+)
+
+
+# /_version/cluster/upgrade-cmapi (PUT)
+dispatcher.connect(
+    name = 'cluster_upgrade_cmapi',
+    route = f'/cmapi/{_version}/cluster/upgrade-cmapi',
+    action = 'upgrade_cmapi',
+    controller = ClusterController(),
+    conditions = {'method': ['PUT']}
+)
+
+
 def jsonify_error(status, message, traceback, version): \
     # pylint: disable=unused-argument
     """JSONify all CherryPy error responses (created by raising the

cmapi/cmapi_server/controllers/endpoints.py

@@ -14,32 +14,27 @@ from mcs_node_control.models.node_config import NodeConfig
 from mcs_node_control.models.node_status import NodeStatus
 
 from cmapi_server.constants import (
-    DEFAULT_MCS_CONF_PATH,
+    CMAPI_PACKAGE_NAME, CMAPI_PORT, DEFAULT_MCS_CONF_PATH,
-    DEFAULT_SM_CONF_PATH,
+    DEFAULT_SM_CONF_PATH, EM_PATH_SUFFIX, MCS_BRM_CURRENT_PATH, MCS_EM_PATH,
-    EM_PATH_SUFFIX,
+    MDB_CS_PACKAGE_NAME, MDB_SERVER_PACKAGE_NAME, REQUEST_TIMEOUT,
-    MCS_BRM_CURRENT_PATH,
+    S3_BRM_CURRENT_PATH, SECRET_KEY,
-    MCS_EM_PATH,
-    S3_BRM_CURRENT_PATH,
-    SECRET_KEY,
 )
+from cmapi_server.controllers.api_clients import NodeControllerClient
 from cmapi_server.controllers.error import APIError
 from cmapi_server.exceptions import CMAPIBasicError, cmapi_error_to_422
 from cmapi_server.handlers.cej import CEJError, CEJPasswordHandler
 from cmapi_server.handlers.cluster import ClusterHandler
 from cmapi_server.helpers import (
-    cmapi_config_check,
+    cmapi_config_check, dequote, get_active_nodes, get_config_parser,
-    dequote,
+    get_current_key, get_dbroots, in_maintenance_state,
-    get_active_nodes,
+    save_cmapi_conf_file, system_ready,
-    get_config_parser,
-    get_current_key,
-    get_dbroots,
-    in_maintenance_state,
-    save_cmapi_conf_file,
-    system_ready,
 )
 from cmapi_server.logging_management import change_loggers_level
 from cmapi_server.managers.application import AppManager
-from cmapi_server.managers.process import MCSProcessManager
+from cmapi_server.managers.upgrade.packages import PackagesManager
+from cmapi_server.managers.upgrade.repo import MariaDBESRepoManager
+from cmapi_server.managers.backup_restore import PreUpgradeBackupRestoreManager
+from cmapi_server.managers.process import MCSProcessManager, MDBProcessManager
 from cmapi_server.managers.transaction import TransactionManager
 from cmapi_server.node_manipulation import is_master, switch_node_maintenance
 
@@ -890,6 +885,80 @@ class ClusterController:
         module_logger.debug(f'{func_name} returns {str(response)}')
         return response
 
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def start_mariadb(self):
+        """Handler for /cluster/start-mariadb (PUT) endpoint."""
+        func_name = 'put_start_mariadb'
+        log_begin(module_logger, func_name)
+
+        request = cherrypy.request
+        request_body = request.json
+        # TODO: Is transaction really needed here.
+        timeout = request_body.get('timeout', None)
+        in_transaction = request_body.get('in_transaction', False)
+
+        active_nodes = get_active_nodes()
+        all_responses: dict = dict()
+        for node in active_nodes:
+            logging.debug(f'Starting MariaDB server on "{node}".')
+            client = NodeControllerClient(
+                request_timeout=REQUEST_TIMEOUT,
+                base_url=f'https://{node}:{CMAPI_PORT}'
+            )
+            node_response = client.start_mariadb()
+            logging.debug(f'MariaDB server started on {node}')
+            all_responses[node] = node_response
+        response = {
+            'timestamp': str(datetime.now()),
+            **all_responses
+        }
+        logging.debug(
+            'Successfully finished starting MariaDB server on all nodes.'
+        )
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def stop_mariadb(self):
+        """Handler for /cluster/stop-mariadb (PUT) endpoint."""
+        func_name = 'put_stop_mariadb'
+        log_begin(module_logger, func_name)
+
+        request = cherrypy.request
+        request_body = request.json
+        # TODO: Is transaction really needed here.
+        timeout = request_body.get('timeout', None)
+        in_transaction = request_body.get('in_transaction', False)
+
+        active_nodes = get_active_nodes()
+        all_responses: dict = dict()
+        for node in active_nodes:
+            logging.debug(f'Stopping MariaDB server on "{node}".')
+            client = NodeControllerClient(
+                request_timeout=REQUEST_TIMEOUT,
+                base_url=f'https://{node}:{CMAPI_PORT}'
+            )
+            node_response = client.stop_mariadb()
+            logging.debug(f'MariaDB server stopped on {node}')
+            all_responses[node] = node_response
+        response = {
+            'timestamp': str(datetime.now()),
+            **all_responses
+        }
+        logging.debug(
+            'Successfully finished stopping MariaDB server on all nodes.'
+        )
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
     @cherrypy.tools.timeit()
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
@@ -1057,6 +1126,246 @@ class ClusterController:
         module_logger.debug(f'{func_name} returns {str(response)}')
         return response
 
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_out()
+    def get_versions(self):
+        """Handler for /cluster/versions (GET) endpoint."""
+        func_name = 'cluster_get_versions'
+        log_begin(module_logger, func_name)
+        # Get versions of packages from all active nodes.
+        # If no active nodes found, get versions from localhost.
+        active_nodes = get_active_nodes()
+        active_nodes_count = len(active_nodes)
+        all_versions: dict = dict()
+
+        if not active_nodes:
+            logging.debug(
+                'No active nodes found, getting versions from localhost.'
+            )
+            active_nodes.append('localhost')
+        for node in active_nodes:
+            logging.debug(f'Getting packages versions from "{node}".')
+            client = NodeControllerClient(
+                request_timeout=REQUEST_TIMEOUT,
+                base_url=f'https://{node}:{CMAPI_PORT}'
+            )
+            node_versions = client.get_versions()
+            logging.debug(
+                f'Node: {node} has installed versions: {node_versions}'
+            )
+            all_versions[node] = node_versions
+
+        versions_set: set = set()
+        for versions in all_versions.values():
+            for version in versions.values():
+                versions_set.add(version)
+
+        if set(versions_set) != set(all_versions[active_nodes[0]].values()):
+            # Nodes have different versions of packages.
+            raise_422_error(
+                logger=module_logger, func_name='get_versions',
+                err_msg=(
+                    'Nodes have different versions of packages. '
+                    f'Active nodes count: {active_nodes_count}. '
+                    f'Active nodes: {active_nodes}. '
+                    f'Packages versions: {all_versions}'
+                )
+            )
+        response = {
+            'timestamp': str(datetime.now()),
+            **all_versions[active_nodes[0]],
+        }
+        logging.debug(
+            'Successfully finished getting package versions from all nodes.'
+        )
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def install_repo(self):
+        """Handler for /cluster/install-repo (PUT) endpoint.
+
+        Installs ES repository on all active nodes.
+        """
+        func_name = 'cluster_install_repo'
+        log_begin(module_logger, func_name)
+        active_nodes = get_active_nodes()
+        request = cherrypy.request
+        request_body = request.json
+        token = request_body.get('token', None)
+        mariadb_version = request_body.get('mariadb_version', None)
+
+        if not token or not mariadb_version:
+            raise_422_error(
+                module_logger, func_name,
+                'Missing required arguments: token, mariadb_version.'
+            )
+        if not active_nodes:
+            logging.debug(
+                'No active nodes found, installing repo on localhost.'
+            )
+            active_nodes.append('localhost')
+        all_responses: dict = dict()
+        for node in active_nodes:
+            logging.debug(f'Installing repo on "{node}".')
+            client = NodeControllerClient(
+                base_url=f'https://{node}:{CMAPI_PORT}'
+            )
+            node_response = client.install_repo(
+                token=token,
+                mariadb_version=mariadb_version
+            )
+            logging.debug(f'ES repo installed on {node}')
+            all_responses[node] = node_response
+        response = {
+            'timestamp': str(datetime.now()),
+            **all_responses
+        }
+        logging.debug(
+            'Successfully finished installing repo on all nodes.'
+        )
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def preupgrade_backup(self):
+        """Handler for /cluster/preupgrade-backup (PUT) endpoint."""
+        func_name = 'cluster_preupgrade_backup'
+        log_begin(module_logger, func_name)
+
+        active_nodes = get_active_nodes()
+        all_responses: dict = dict()
+        for node in active_nodes:
+            logging.debug(
+                f'Backuping DBRM and configs before upgrade on "{node}".'
+            )
+            client = NodeControllerClient(
+                base_url=f'https://{node}:{CMAPI_PORT}'
+            )
+            node_response = client.preupgrade_backup()
+            logging.debug(f'PreUpgrade backup completed on {node}')
+            all_responses[node] = node_response
+        response = {
+            'timestamp': str(datetime.now()),
+            **all_responses
+        }
+        logging.debug(
+            'Successfully finished PreUpgrade backup on all nodes.'
+        )
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def upgrade_mdb_mcs(self):
+        """Handler for /cluster/upgrade-mdb-mcs (PUT) endpoint."""
+        func_name = 'cluster_upgrade_mdb_mcs'
+        log_begin(module_logger, func_name)
+        request = cherrypy.request
+        request_body = request.json
+        mdb_version = request_body.get('mariadb_version', None)
+        mcs_version = request_body.get('columnstore_version', None)
+        if not mdb_version or not mcs_version:
+            raise_422_error(
+                module_logger, func_name,
+                'Missing required arguments: mdb_version, mcs_version.'
+            )
+        active_nodes = get_active_nodes()
+        all_responses: dict = dict()
+        for node in active_nodes:
+            logging.debug(
+                f'Upgrading MDB and MCS on "{node}".'
+            )
+            client = NodeControllerClient(
+                base_url=f'https://{node}:{CMAPI_PORT}'
+            )
+            node_response = client.upgrade_mdb_mcs(
+                mariadb_version=mdb_version, columnstore_version=mcs_version
+            )
+            logging.debug(f'Upgrade MDB and MCS completed on {node}')
+            all_responses[node] = node_response
+        response = {
+            'timestamp': str(datetime.now()),
+            **all_responses
+        }
+        logging.debug(
+            'Successfully finished upgrading MDB and MCS on all nodes.'
+        )
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def upgrade_cmapi(self):
+        """Handler for /cluster/upgrade-cmapi (PUT) endpoint."""
+        func_name = 'cluster_upgrade_cmapi'
+        log_begin(module_logger, func_name)
+        request = cherrypy.request
+        request_body = request.json
+        target_version = request_body.get('version', None)
+        if not target_version:
+            raise_422_error(
+                module_logger, func_name,
+                'Missing required argument target_version.'
+            )
+        active_nodes = get_active_nodes()
+        all_responses: dict = dict()
+        for node in active_nodes:
+            logging.debug(
+                f'Kicking CMAPI to upgrade on "{node}".'
+            )
+            client = NodeControllerClient(
+                base_url=f'https://{node}:{CMAPI_PORT}'
+            )
+            node_response = client.kick_cmapi_upgrade(version=target_version)
+            all_responses[node] = node_response
+        response = {
+            'timestamp': str(datetime.now()),
+            **all_responses
+        }
+        logging.debug(
+            'Started CMAPI upgrade on all nodes.'
+        )
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def get_health(self):
+        func_name = 'get_health'
+        log_begin(module_logger, func_name)
+
+        request = cherrypy.request
+        request_body = request.json
+        timeout = request_body.get('timeout', None)
+        in_transaction = request_body.get('in_transaction', False)
+
+        try:
+            if not in_transaction:
+                with TransactionManager():
+                    # TODO: just a placeholder for now
+                    # response = ClusterHandler.health()
+                    response = {'status': 'ok'}
+            else:
+                # response = ClusterHandler.health()
+                response = {'status': 'ok'}
+        except CMAPIBasicError as err:
+            raise_422_error(module_logger, func_name, err.message)
+
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
     @cherrypy.tools.timeit()
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
@@ -1275,3 +1584,307 @@ class NodeProcessController():
         }
         module_logger.debug(f'{func_name} returns {str(response)}')
         return response
+
+
+class NodeController:
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_out()
+    def get_versions(self):
+        """Handler for /node/versions (GET) endpoint."""
+        func_name = 'get_node_versions'
+        log_begin(module_logger, func_name)
+        columnstore_ver = AppManager.get_columnstore_version()
+        cmapi_short_ver = AppManager.version
+        # cmapi version currently is just a part of columnstore version excluding MDB version part
+        # so canonicalize it
+        cmapi_ver = columnstore_ver if cmapi_short_ver in columnstore_ver else cmapi_short_ver
+        node_versions = {
+            'cmapi_version': cmapi_ver,
+            'columnstore_version': columnstore_ver,
+            'server_version': AppManager.get_mdb_version(),
+        }
+        response = {
+            'timestamp': str(datetime.now()),
+            **node_versions
+        }
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def latest_mdb_version(self):
+        """Handler for /node/latest-mdb-version (GET) endpoint."""
+        func_name = 'get_latest_mdb_version'
+        log_begin(module_logger, func_name)
+        try:
+            version = MariaDBESRepoManager.get_latest_tested_mdb_version()
+        except CMAPIBasicError as err:
+            raise_422_error(module_logger, func_name, err.message)
+        response = {
+            'timestamp': str(datetime.now()),
+            'latest_mdb_version': version
+        }
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def validate_mdb_version(self, token, mariadb_version):
+        """Handler for /node/validate-mdb-version (GET) endpoint."""
+        func_name = 'get_validate_mdb_version'
+        log_begin(module_logger, func_name)
+        if not token or not mariadb_version:
+            raise_422_error(
+                module_logger, func_name,
+                'Missing required arguments: token, mariadb_version.'
+            )
+        os_name, os_version = AppManager.get_distro_info()
+        arch = AppManager.get_architecture()
+        repo_manager = MariaDBESRepoManager(
+            token=token, arch=arch, os_type=os_name, os_version=os_version,
+            mariadb_version=mariadb_version
+        )
+
+        try:
+            repo_manager.check_mdb_version_exists()
+        except CMAPIBasicError as err:
+            raise_422_error(module_logger, func_name, err.message)
+        response = {'timestamp': str(datetime.now())}
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def validate_es_token(self, token):
+        """Handler for /node/validate-es-token (GET) endpoint."""
+        func_name = 'get_validate_es_token'
+        log_begin(module_logger, func_name)
+
+        if not token:
+            raise_422_error(
+                module_logger, func_name,
+                'Missing required argument token.'
+            )
+        try:
+            MariaDBESRepoManager.verify_token(token)
+        except CMAPIBasicError as err:
+            raise_422_error(module_logger, func_name, err.message)
+        response = {'timestamp': str(datetime.now())}
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def start_mariadb(self):
+        """Handler for /node/start_mariadb (PUT) endpoint."""
+        func_name = 'node_start_mariadb'
+        log_begin(module_logger, func_name)
+        req = cherrypy.request
+        use_sudo = get_use_sudo(req.app.config)
+        try:
+            MDBProcessManager.start(use_sudo=use_sudo)
+        except CMAPIBasicError as err:
+            raise_422_error(
+                module_logger, func_name,
+                (
+                    'Error while starting mariadb process. '
+                    f'Details: {err.message}'
+                ),
+                exc_info=False
+            )
+        response = {'timestamp': str(datetime.now())}
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def stop_mariadb(self):
+        """Handler for /node/stop_mariadb (PUT) endpoint."""
+        func_name = 'node_stop_mariadb'
+        log_begin(module_logger, func_name)
+        req = cherrypy.request
+        use_sudo = get_use_sudo(req.app.config)
+        try:
+            MDBProcessManager.stop(use_sudo=use_sudo)
+        except CMAPIBasicError as err:
+            raise_422_error(
+                module_logger, func_name,
+                (
+                    'Error while stopping mariadb process. '
+                    f'Details: {err.message}'
+                ),
+                exc_info=False
+            )
+        response = {'timestamp': str(datetime.now())}
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def install_repo(self):
+        """Handler for /node/install-repo (PUT) endpoint."""
+        func_name = 'node_install_repo'
+        log_begin(module_logger, func_name)
+
+        request = cherrypy.request
+        request_body = request.json
+        token = request_body.get('token', None)
+        mariadb_version = request_body.get('mariadb_version', None)
+
+        if not token or not mariadb_version:
+            raise_422_error(
+                module_logger, func_name,
+                'Missing required arguments: token, mariadb_version.'
+            )
+        os_name, os_version = AppManager.get_distro_info()
+        arch = AppManager.get_architecture()
+        repo_manager = MariaDBESRepoManager(
+            token=token, arch=arch, os_type=os_name, os_version=os_version,
+            mariadb_version=mariadb_version
+        )
+        try:
+            repo_manager.setup_repo()
+        except CMAPIBasicError as err:
+            raise_422_error(module_logger, func_name, err.message)
+        response = {'timestamp': str(datetime.now())}
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def repo_pkg_versions(self):
+        """Handler for /node/repo-pkg-versions (GET) endpoint."""
+        func_name = 'get_repo_pkg_versions'
+        log_begin(module_logger, func_name)
+        os_name, _ = AppManager.get_distro_info()
+        mdb_pkg_name: str
+        mcs_pkg_name: str
+        cmapi_pkg_name: str
+        if os_name in ['ubuntu', 'debian']:
+            mdb_pkg_name = MDB_SERVER_PACKAGE_NAME.deb
+            mcs_pkg_name = MDB_CS_PACKAGE_NAME.deb
+            cmapi_pkg_name = CMAPI_PACKAGE_NAME.deb
+        elif os_name in ['centos', 'rhel', 'rocky']:
+            mdb_pkg_name = MDB_SERVER_PACKAGE_NAME.rhel
+            mcs_pkg_name = MDB_CS_PACKAGE_NAME.rhel
+            cmapi_pkg_name = CMAPI_PACKAGE_NAME.rhel
+        else:
+            raise_422_error(
+                module_logger, func_name, f'Unsupported OS type: {os_name}'
+            )
+
+        try:
+            repo_versions = {
+                'cmapi_version': MariaDBESRepoManager.get_ver_of(
+                    cmapi_pkg_name, os_name
+                ),
+                'columnstore_version': MariaDBESRepoManager.get_ver_of(
+                    mcs_pkg_name, os_name
+                ),
+                'server_version': MariaDBESRepoManager.get_ver_of(
+                    mdb_pkg_name, os_name
+                ),
+            }
+        except CMAPIBasicError as err:
+            raise_422_error(module_logger, func_name, err.message)
+        response = {
+            'timestamp': str(datetime.now()),
+            **repo_versions
+        }
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def preupgrade_backup(self):
+        """Handler for /node/preupgrade-backup (PUT) endpoint."""
+        func_name = 'node_preupgrade_backup'
+        log_begin(module_logger, func_name)
+        os_name, _ = AppManager.get_distro_info()
+        try:
+            PreUpgradeBackupRestoreManager.backup_dbrm()
+            PreUpgradeBackupRestoreManager.backup_configs(distro_name=os_name)
+        except CMAPIBasicError as err:
+            raise_422_error(
+                module_logger, func_name,
+                f'Error while PreUpgrade backup. Details: {err.message}',
+                exc_info=False
+            )
+        response = {'timestamp': str(datetime.now())}
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def upgrade_mdb_mcs(self):
+        """Handler for /node/upgrade-mdb-mcs (PUT) endpoint."""
+        func_name = 'node_upgrade_mdb_mcs'
+        log_begin(module_logger, func_name)
+        request = cherrypy.request
+        request_body = request.json
+        mdb_version = request_body.get('mariadb_version', None)
+        mcs_version = request_body.get('columnstore_version', None)
+        if not mdb_version or not mcs_version:
+            raise_422_error(
+                module_logger, func_name,
+                'Missing required arguments: mdb_version, mcs_version.'
+            )
+        os_name, _ = AppManager.get_distro_info()
+        try:
+            packages_manager = PackagesManager(
+                os_name=os_name, mdb_version=mdb_version,
+                mcs_version=mcs_version
+            )
+            packages_manager.upgrade_mdb_and_mcs()
+        except CMAPIBasicError as err:
+            raise_422_error(
+                module_logger, func_name,
+                (
+                    'Error while Upgrading MDB and MCS packages. '
+                    f'Details: {err.message}'
+                ),
+                exc_info=False
+            )
+        response = {'timestamp': str(datetime.now())}
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response
+
+    @cherrypy.tools.timeit()
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.validate_api_key()  # pylint: disable=no-member
+    def kick_cmapi_upgrade(self):
+        """Handler for /node/kick-cmapi-upgrade (PUT) endpoint."""
+        func_name = 'node_kick_cmapi_upgrade'
+        log_begin(module_logger, func_name)
+        request = cherrypy.request
+        request_body = request.json
+        target_version = request_body.get('version', None)
+        if target_version is None:
+            raise_422_error(
+                module_logger, func_name, 'Missing required version argument.'
+            )
+        try:
+            PackagesManager.kick_cmapi_upgrade(cmapi_version=target_version)
+        except CMAPIBasicError as err:
+            raise_422_error(module_logger, func_name, err.message)
+
+        response = {'timestamp': str(datetime.now())}
+        module_logger.debug(f'{func_name} returns {str(response)}')
+        return response

cmapi/cmapi_server/helpers.py

@@ -28,7 +28,7 @@ requests.packages.urllib3.disable_warnings()  # pylint: disable=no-member
 
 from cmapi_server.constants import (
     CMAPI_CONF_PATH, CMAPI_DEFAULT_CONF_PATH, DEFAULT_MCS_CONF_PATH,
-    DEFAULT_SM_CONF_PATH, LOCALHOSTS
+    DEFAULT_SM_CONF_PATH, LOCALHOSTS, _version
 )
 from cmapi_server.handlers.cej import CEJPasswordHandler
 from cmapi_server.managers.process import MCSProcessManager
@@ -576,7 +576,6 @@ def get_current_key(config_parser):
 
 
 def get_version():
-    from cmapi_server.controllers.dispatcher import _version
     return _version
 
 

cmapi/cmapi_server/logging_management.py

@@ -117,6 +117,8 @@ def add_logging_level(level_name, level_num, method_name=None):
 def enable_console_logging(logger: logging.Logger) -> None:
     """Enable logging to console for passed logger by adding a StreamHandler to it"""
     console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.DEBUG)
+    console_handler.setFormatter(logger.handlers[0].formatter)
     logger.addHandler(console_handler)
 
 
@@ -159,7 +161,7 @@ def change_loggers_level(level: str):
         logger.setLevel(level)
 
 def disable_unwanted_loggers():
-    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger('urllib3').setLevel(logging.WARNING)
 
 
 class JsonFormatter(logging.Formatter):

cmapi/cmapi_server/managers/application.py

@@ -1,7 +1,15 @@
 import logging
+import platform
 from typing import Optional, Tuple, Dict
 
-from cmapi_server.constants import VERSION_PATH
+import distro
+
+from cmapi_server.constants import (
+    MDB_CS_PACKAGE_NAME, MDB_SERVER_PACKAGE_NAME, PKG_GET_VER_CMD,
+    SUPPORTED_DISTROS, SUPPORTED_ARCHITECTURES, VERSION_PATH, MultiDistroNamer
+)
+from cmapi_server.exceptions import CMAPIBasicError
+from cmapi_server.process_dispatchers.base import BaseDispatcher
 
 
 class AppManager:
@@ -65,3 +73,125 @@ class AppManager:
         if release:
             version = f"{version}.{release}"
         return version, revision
+
+    @classmethod
+    def get_architecture(cls) -> str:
+        """Get system architecture.
+
+        :return: system architecture
+        :rtype: str
+        """
+        arch = platform.machine().lower()
+        if arch not in SUPPORTED_ARCHITECTURES:
+            message = f'Unsupported architecture: {arch}'
+            logging.error(message)
+            raise CMAPIBasicError(message)
+        if arch in ['x86_64', 'amd64']:
+            arch = 'x86_64'
+        elif arch in ['aarch64', 'arm64']:
+            arch = 'aarch64'
+        return arch
+
+    @classmethod
+    def get_distro_info(cls) -> tuple[str, str]:
+        """Get OS name and version.
+
+        :return: OS name and version
+        :rtype: tuple[str, str]
+        """
+        platform_name = platform.system().lower()
+        if platform_name == 'linux':
+            distro_name = distro.id().lower()
+            distro_version = distro.version().lower()
+            if distro_name == 'debian':
+                if distro_version.startswith('11'):
+                    distro_version = 'bullseye'
+                elif distro_version.startswith('12'):
+                    distro_version = 'bookworm'
+                else:
+                    message = (
+                        f'Unsupported Debian version: {distro_version}. '
+                        'Supported versions are 11 (bullseye) and 12 '
+                        '(bookworm).'
+                    )
+                    logging.error(message)
+                    raise CMAPIBasicError(message)
+            if distro_name == 'ubuntu':
+                if distro_version.startswith('20.04'):
+                    distro_version = 'focal'
+                elif distro_version.startswith('22.04'):
+                    distro_version = 'jammy'
+                elif distro_version.startswith('24.04'):
+                    distro_version = 'noble'
+                else:
+                    message = (
+                        f'Unsupported Ubuntu version: {distro_version}. '
+                        'Supported versions are 20.04 (focal), 22.04 (jammy), '
+                        'and 24.04 (noble).'
+                    )
+                    logging.error(message)
+                    raise CMAPIBasicError(message)
+            if distro_name not in SUPPORTED_DISTROS:
+                message = (
+                    f'Unsupported Linux distribution: {distro_name}. '
+                )
+                logging.error(message)
+                raise CMAPIBasicError(message)
+        else:
+            message = f'Unsupported platform: {platform_name}'
+            logging.error(message)
+            raise CMAPIBasicError(message)
+        return distro_name, distro_version
+
+    @classmethod
+    def get_installed_pkg_ver(cls, pkg_namer: MultiDistroNamer) -> str:
+        """Get package version with given package name.
+
+        :param pkg_namer: object that contains package name for several ditros
+        :type pkg_namer: MultiDistroNamer
+        :raises CMAPIBasicError: if failed getting version
+        :return: package version
+        :rtype: str
+        """
+        distro_name, _ = cls.get_distro_info()
+        cmd: str = ''
+        package_name: str = ''
+        if distro_name in ['ubuntu', 'debian']:
+            package_name = pkg_namer.deb
+            cmd = PKG_GET_VER_CMD.deb.format(package_name=package_name)
+        elif distro_name in ['centos', 'rhel', 'rocky', 'almalinux']:
+            package_name = pkg_namer.rhel
+            cmd = PKG_GET_VER_CMD.rhel.format(package_name=package_name)
+        success, result_raw = BaseDispatcher.exec_command(cmd)
+        if not success:
+            message = (
+                f'Failed to get {package_name} version with result: '
+                f'{result_raw}'
+            )
+            logging.error(message)
+            raise CMAPIBasicError(message)
+        version_clean = result_raw
+        if distro_name in ['ubuntu', 'debian']:
+            # remove prefix before : (epoch)
+            result_raw = result_raw.split(':', 1)[1]
+            # remove suffix after first '+'
+            version_clean = result_raw.split('+', 1)[0]
+        return version_clean
+
+    @classmethod
+    def get_mdb_version(cls) -> str:
+        """Get MDB version.
+
+        :return: MDB version
+        :rtype: str
+        """
+        return cls.get_installed_pkg_ver(MDB_SERVER_PACKAGE_NAME)
+
+    @classmethod
+    def get_columnstore_version(cls) -> str:
+        """Get Columnstore version.
+
+        :return: Columnstore version
+        :rtype: str
+        """
+        return cls.get_installed_pkg_ver(MDB_CS_PACKAGE_NAME)

cmapi/cmapi_server/managers/backup_restore.py

@@ -0,0 +1,115 @@
+import logging
+import glob
+import shutil
+import os
+from datetime import datetime
+
+from cmapi_server.exceptions import CMAPIBasicError
+from cmapi_server.process_dispatchers.base import BaseDispatcher
+from cmapi_server.constants import MCS_BACKUP_MANAGER_SH
+
+
+class BackupRestoreManager:
+    @classmethod
+    def backup_dbrm(cls, args: dict) -> None:
+        """Make DBRM backup.
+
+        :raises CMAPIBasicError: If dbrm backup is unsuccesfull.
+        """
+        cmd: str = (
+            f'{MCS_BACKUP_MANAGER_SH} dbrm_backup '
+            f'{" ".join([f"{k} {v}" if v else k for k,v in args.items()])}'
+        )
+        success, cmd_output = BaseDispatcher.exec_command(cmd)
+        if not success:
+            err_message: str = ''
+            if not cmd_output:
+                err_message = f'Can\'t start DBRM backup using cmd: "{cmd}"'
+                logging.error(err_message, exc_info=True)
+            else:
+                logging.error(err_message)
+            raise CMAPIBasicError(err_message)
+
+
+class PreUpgradeBackupRestoreManager(BackupRestoreManager):
+    @classmethod
+    def backup_dbrm(cls) -> None:  #pylint: disable=arguments-differ
+        """PreUpgrade dbrm backup.
+
+        :raises CMAPIBasicError: If dbrm backup is unsuccesfull.
+        """
+        args = {
+            '-r': '9999',
+            '-nb': 'preupgrade_dbrm_backup',
+            '--quiet': '',
+            '--skip-locks': ''
+        }
+        super().backup_dbrm(args)
+
+
+    @classmethod
+    def _copy_files(cls, source_pattern: str, destination_dir: str):
+        """Copy files using pattern path to a given destination.
+
+        :param source_pattern: source pattern could be just a str path
+        :type source_pattern: str
+        :param destination_dir: destination dir
+        :type destination_dir: str
+        """
+        files = glob.glob(source_pattern)
+        if not files:
+            logging.warning(f'No files matched: {source_pattern}')
+            return
+
+        for file_path in files:
+            try:
+                logging.debug(f'Copying: {file_path} -> {destination_dir}')
+                shutil.copy(file_path, destination_dir)
+            except Exception:
+                logging.warning(
+                    f'Failed to copy {file_path}',
+                    exc_info=True
+                )
+
+    @classmethod
+    def backup_configs(cls, distro_name: str):
+        """Backup config files.
+
+        :param distro_name: node distro name
+        :type distro_name: str
+        """
+        timestamp = datetime.now().strftime('%m-%d-%Y-%H%M')
+        pre_upgrade_config_directory = (
+            f'/tmp/preupgrade-configurations-{timestamp}'
+        )
+        os.makedirs(pre_upgrade_config_directory, exist_ok=True)
+        cls._copy_files(
+            '/etc/columnstore/Columnstore.xml', pre_upgrade_config_directory
+        )
+        cls._copy_files(
+            '/etc/columnstore/storagemanager.cnf', pre_upgrade_config_directory
+        )
+        cls._copy_files(
+            '/etc/columnstore/cmapi_server.conf', pre_upgrade_config_directory
+        )
+
+        if distro_name in ['centos', 'rhel', 'rocky', 'almalinux']:
+            cls._copy_files(
+                '/etc/my.cnf.d/server.cnf', pre_upgrade_config_directory
+            )
+        elif distro_name in ['ubuntu', 'debian']:
+            cls._copy_files(
+                '/etc/mysql/mariadb.conf.d/*server.cnf',
+                pre_upgrade_config_directory
+            )
+        else:
+            logging.error(f'Unknown distro: {distro_name}')
+
+    @staticmethod
+    def restore_dbrm():
+        #TODO: implement restore logic
+        pass
+
+    @staticmethod
+    def restore_configs():
+        pass

cmapi/cmapi_server/managers/process.py

@@ -482,3 +482,46 @@ class MCSProcessManager:
         if cls.get_running_mcs_procs():
             cls.stop_node(is_primary, use_sudo)
         cls.start_node(is_primary, use_sudo, is_read_replica)
+
+
+class MDBProcessManager:
+    """TODO: not working with a non systemd installations need to implement."""
+    process_dispatcher = SystemdDispatcher
+
+    @classmethod
+    def is_service_running(cls, use_sudo: bool = True) -> bool:
+        """Check if MariaDB process is running.
+
+        :param use_sudo: use sudo or not, defaults to True
+        :type use_sudo: bool, optional
+        :return: True if MariaDB process is running, otherwise False
+        :rtype: bool
+        """
+        return cls.process_dispatcher.is_service_running(
+            'mariadb', use_sudo=use_sudo
+        )
+
+    @classmethod
+    def start(cls, use_sudo: bool) -> bool:
+        """Start MariaDB process.
+
+        :type use_sudo: bool
+        :return: True if process started successfully
+        :rtype: bool
+        """
+        if not cls.is_service_running():
+            return cls.process_dispatcher.start('mariadb', use_sudo)
+        return True
+
+    @classmethod
+    def stop(cls, use_sudo: bool = True) -> bool:
+        """Stop MariaDB process.
+
+        :type use_sudo: bool
+        :return: True if process stopped
+          successfully
+        :rtype: bool
+        """
+        if cls.is_service_running():
+            return cls.process_dispatcher.stop('mariadb', use_sudo)
+        return True

cmapi/cmapi_server/managers/upgrade/packages.py

@@ -0,0 +1,144 @@
+import logging
+
+from cmapi_server.constants import (
+    MDB_CS_PACKAGE_NAME, MDB_SERVER_PACKAGE_NAME
+)
+from cmapi_server.exceptions import CMAPIBasicError
+from cmapi_server.process_dispatchers.base import BaseDispatcher
+
+
+class PackagesManager:
+    """
+    This class is responsible for managing the installation of packages.
+    It provides methods to install, uninstall packages.
+    """
+    def __init__(self, os_name: str, mdb_version: str, mcs_version: str):
+        self.os_name = os_name
+        self.mdb_version = mdb_version
+        self.mcs_version = mcs_version
+        self.pkg_manager: str = ''
+        if self.os_name in ['ubuntu', 'debian']:
+            # Prefer apt-get in scripts for stability and noninteractive support
+            self.pkg_manager = 'apt-get'
+            self.mdb_pkg_name = MDB_SERVER_PACKAGE_NAME.deb
+            self.mcs_pkg_name = MDB_CS_PACKAGE_NAME.deb
+        elif self.os_name in ['centos', 'rhel', 'rocky']:
+            self.pkg_manager = 'yum'
+            self.mdb_pkg_name = MDB_SERVER_PACKAGE_NAME.rhel
+            self.mcs_pkg_name = MDB_CS_PACKAGE_NAME.rhel
+        else:
+            raise CMAPIBasicError(f'Unsupported OS type: {self.os_name}')
+
+    def install_package(self, package_name: str, version: str = 'latest', dry_run: bool = False):
+        """Install a package by its name.
+
+        :param package_name: the name of the package to install.
+        :param version: version to install
+        :param dry_run: if True, simulate the transaction without making changes
+        """
+        extra_args = ''
+        dry_run_flag = ''
+        conf_opt = ''
+        env_vars = None
+        package = package_name
+        if self.os_name in ['ubuntu', 'debian']:
+            if version != 'latest':
+                package = f'{package_name}={version}'
+                # Allow downgrades explicitly when version is pinned
+                extra_args = '--allow-downgrades'
+            # Noninteractive mode and ALWAYS keep current configs
+            env_vars = {'DEBIAN_FRONTEND': 'noninteractive'}
+            conf_opt = '-o Dpkg::Options::=--force-confold'
+            dry_run_flag = '-s' if dry_run else ''
+        elif self.os_name in ['centos', 'rhel', 'rocky']:
+            if version != 'latest':
+                package = f'{package_name}-{version}'
+            # For yum, use --assumeno as a safe preview of the transaction
+            # but it exits non-zero; prefer tsflags=test for a clean dry-run
+            dry_run_flag = '--setopt=tsflags=test' if dry_run else ''
+
+        cmd = (
+            f"{self.pkg_manager} install -y {dry_run_flag} "
+            f"{conf_opt} {extra_args} {package}"
+        ).strip()
+        success, result = BaseDispatcher.exec_command(cmd, env=env_vars)
+        if not success:
+            message = (
+                f'Failed to install {package} using command {cmd} with '
+                f'result: {result}'
+            )
+            logging.error(message)
+            raise CMAPIBasicError(message)
+
+    def remove_package(self, package_name: str, *, dry_run: bool = False) -> bool:
+        """Uninstall a package by its name.
+
+        :param package_name: The name of the package to remove.
+        :param dry_run: if True, simulate the transaction without making changes
+        """
+        env_vars = None
+        dry_flag = ''
+        if self.os_name in ['ubuntu', 'debian']:
+            env_vars = {'DEBIAN_FRONTEND': 'noninteractive'}
+            dry_flag = '-s' if dry_run else ''
+        elif self.os_name in ['centos', 'rhel', 'rocky']:
+            # use tsflags=test to simulate with zero exit code
+            dry_flag = '--setopt=tsflags=test' if dry_run else ''
+
+        cmd = f'{self.pkg_manager} remove -y {dry_flag} {package_name}'.strip()
+        success, result = BaseDispatcher.exec_command(cmd, env=env_vars)
+        if not success:
+            message = (
+                f'Failed to remove {package_name} using command {cmd} with '
+                f'result: {result}'
+            )
+            logging.error(message)
+            raise CMAPIBasicError(message)
+
+    def upgrade_mdb_and_mcs(self, *, precheck: bool = True):
+        """Remove packages and then install newer or older versions.
+
+        The function can perform a dry-run of all steps first. If any of the
+        simulated transactions fail, the actual removal/installation will not
+        be executed.
+
+        :param precheck: when True, simulate remove/install before real actions
+        """
+        if precheck:
+            # 1) Simulate removals; fail fast if not possible
+            self.remove_package(self.mcs_pkg_name, dry_run=True)
+            self.remove_package(self.mdb_pkg_name, dry_run=True)
+
+        # 2) Perform actual removals
+        self.remove_package(self.mcs_pkg_name)
+        self.remove_package(self.mdb_pkg_name)
+
+        if precheck:
+            # 3) Now that packages are removed, simulate installs to validate deps
+            self.install_package(self.mdb_pkg_name, self.mdb_version, dry_run=True)
+            self.install_package(self.mcs_pkg_name, self.mcs_version, dry_run=True)
+
+        # 4) Perform actual installs
+        self.install_package(self.mdb_pkg_name, self.mdb_version)
+        self.install_package(self.mcs_pkg_name, self.mcs_version)
+
+    @classmethod
+    def kick_cmapi_upgrade(cls, cmapi_version: str):
+        """Starts the one-shot cmapi_updater.service.
+
+        :param cmapi_version: target CMAPI version to install
+        :type cmapi_version: str
+        """
+        with open('/tmp/cmapi_updater.conf', 'w+', encoding='utf-8') as file:
+            file.write(f'CMAPI_VERSION={cmapi_version}')
+        cmd = 'systemctl start cmapi_updater.service'
+        success, result = BaseDispatcher.exec_command(cmd, daemonize=True)
+        # Note: this likely never reports an error in practice, but we still check.
+        if not success:
+            message = (
+                f'Failed to start cmapi_updater.serice using command {cmd} '
+                f'with result: {result}'
+            )
+            logging.error(message)
+            raise CMAPIBasicError(message)
+        logging.info('Started cmapi_updater.service to upgrade CMAPI.')

cmapi/cmapi_server/managers/upgrade/preinstall.py

@@ -0,0 +1,54 @@
+import logging
+import re
+
+from cmapi_server.exceptions import CMAPIBasicError
+from cmapi_server.process_dispatchers.base import BaseDispatcher
+
+
+class PreInstallManager:
+
+    @staticmethod
+    def check_gtid_strict_mode():
+        """
+        Check if gtid_strict_mode is enabled in MariaDB/MySQL configuration.
+        Throw an error if gtid_strict_mode=1 is found.
+
+        TODO: seems to be useless if set dynamically using
+              SET GLOBAL gtid_strict_mode = 1;
+              Better solution is to use query SELECT @@global.gtid_strict_mode;
+              But need to investigate how to implement it if no crossengine
+              user set, may be check it and fallback or just throw an error.
+        """
+        cmd: str = 'my_print_defaults --mysqld'
+        success, cmd_output = BaseDispatcher.exec_command(cmd)
+        if not success:
+            if not cmd_output:
+                logging.debug(
+                    'my_print_defaults not found. Ensure gtid_strict_mode=0.'
+                )
+            else:
+                logging.debug(
+                    'my_print_defaults --mysqld command call returns an '
+                    f'error: {cmd_output}. Ensure gtid_strict_mode=0.'
+                )
+        else:
+            # Search for gtid_strict_mode or gtid-strict-mode patterns
+            gtid_pattern = re.compile(r"gtid[-_]strict[-_]mode")
+            strict_mode_lines = [
+                line for line in cmd_output.splitlines()
+                if gtid_pattern.search(line)
+            ]
+            if strict_mode_lines:
+                # Check if any line shows gtid_strict_mode=1
+                for line in strict_mode_lines:
+                    line = line.strip()
+                    if (
+                        line == '--gtid_strict_mode=1' or
+                        line == '--gtid_strict_mode=ON'
+                    ):
+                        message = (
+                            'gtid strick mode is ON, need to be off before '
+                            'upgrade/downgrade.'
+                        )
+                        logging.error(message)
+                        raise CMAPIBasicError(message)

cmapi/cmapi_server/managers/upgrade/repo.py

@@ -0,0 +1,288 @@
+import logging
+import os
+import re
+import subprocess
+
+import requests
+
+from cmapi_server.constants import (
+    ES_REPO, ES_REPO_PRIORITY_PREFS, ES_TOKEN_VERIFY_URL, ES_VERIFY_URL, MDB_GPG_KEY_URL,
+    MDB_LATEST_RELEASES_URL, MDB_LATEST_TESTED_MAJOR, REQUEST_TIMEOUT,
+)
+from cmapi_server.exceptions import CMAPIBasicError
+from cmapi_server.managers.upgrade.utils import ComparableVersion
+from cmapi_server.process_dispatchers.base import BaseDispatcher
+
+
+class MariaDBESRepoManager:
+    def __init__(
+            self, token: str, arch: str, os_type: str, os_version: str,
+            mariadb_version: str = 'latest',
+    ):
+        self.token = token
+        self.arch = arch
+        self.os_type = os_type
+        self.os_version = os_version
+        if mariadb_version == 'latest':
+            self.mariadb_version = self.get_latest_tested_mdb_version()
+        else:
+            self.mariadb_version = mariadb_version
+
+    def _import_mariadb_keyring(self):
+        """
+        Download and place the MariaDB keyring into /etc/apt/trusted.gpg.d.
+        """
+        key_url = 'https://supplychain.mariadb.com/mariadb-keyring-2019.gpg'
+        keyring_path = '/etc/apt/trusted.gpg.d/mariadb-keyring-2019.gpg'
+
+        try:
+            # Download the keyring file
+            response = requests.get(key_url)
+            response.raise_for_status()
+
+            # Write the keyring file to the specified path
+            with open(keyring_path, 'wb') as key_file:
+                key_file.write(response.content)
+
+            # Set permissions to 644
+            os.chmod(keyring_path, 0o644)
+            logging.debug(
+                f'Keyring successfully downloaded and placed at {keyring_path}'
+            )
+        except requests.RequestException as exc:
+            raise CMAPIBasicError(
+                f'Failed to download keyring from {key_url}: {exc}'
+            )
+        except OSError as exc:
+            raise CMAPIBasicError(
+                f'Failed to write keyring to {keyring_path}: {exc}'
+            )
+
+    def check_mdb_version_exists(self):
+        """Check if passed MDB version exists in the repo.
+
+        :raises CMAPIBasicError: unsupported OS type
+        :raises CMAPIBasicError: wrong MDB version passed
+        :raises CMAPIBasicError: some other request/response errors
+        """
+        verify_url: str = ''
+        if self.os_type in ['ubuntu', 'debian']:
+            # get only two first numbers from version to build repo link
+            verify_url = ES_VERIFY_URL.deb.format(
+                token=self.token,
+                mdb_version=self.mariadb_version,
+                os_version=self.os_version
+            )
+        elif self.os_type in ['centos', 'rhel', 'rocky']:
+            verify_url = ES_VERIFY_URL.rhel.format(
+                token=self.token,
+                mdb_version=self.mariadb_version,
+                os_major_version=self.os_version.split('.', maxsplit=1)[0],
+                arch=self.arch
+            )
+        else:
+            raise CMAPIBasicError(f'Unsupported OS type: {self.os_type}')
+        try:
+            # Download the keyring file
+            response = requests.get(verify_url, timeout=REQUEST_TIMEOUT)
+            if response.status_code in (403, 404):
+                raise CMAPIBasicError(
+                    'MariaDB Enterprise Server version '
+                    f'{self.mariadb_version} is not working for your OS '
+                    'version or OS type.\nPlease verify that it is correct.\n '
+                    'Not all releases of MariaDB are available on all '
+                    'distributions.'
+                )
+            elif response.ok:
+                logging.debug(
+                    'MariaDB Enterprise Server version '
+                    f'{self.mariadb_version} is valid.'
+                )
+            else:
+                response.raise_for_status()
+        except requests.RequestException:
+            raise CMAPIBasicError(
+                'Failed to check MDB version exists from '
+                f'{verify_url}'
+            )
+
+    @staticmethod
+    def verify_token(token: str):
+        """Verify ES token.
+
+        :param token: es token to verify
+        :type token: str
+        :raises CMAPIBasicError: Invalid token format
+        :raises CMAPIBasicError: Invalid token
+        :raises CMAPIBasicError: Other request errors
+        """
+        # Check token format UUID
+        valid_format = re.fullmatch(
+            r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
+            token
+        )
+        if not valid_format:
+            err_message = (
+                f'Invalid token format: "{token}". The token should be '
+                'of the form ########-####-####-####-############'
+            )
+            raise CMAPIBasicError(err_message)
+
+        verify_url = ES_TOKEN_VERIFY_URL.format(token=token)
+
+        try:
+            response = requests.head(
+                verify_url, allow_redirects=True, timeout=REQUEST_TIMEOUT
+            )
+            if response.status_code in (403, 404):
+                raise CMAPIBasicError(
+                    'Invalid token. Please verify that it is correct.'
+                )
+            elif response.ok:
+                logging.debug('MariaDB ES Token is valid.')
+            else:
+                response.raise_for_status()
+        except requests.RequestException:
+            raise CMAPIBasicError(
+                'Problem encountered while trying to verify ES token.'
+            )
+
+    @classmethod
+    def get_latest_tested_mdb_version(cls) -> str:
+        """Get latest tested MDB version from repo.
+
+        :raises CMAPIBasicError: no latest version matched with latest tested
+        :raises CMAPIBasicError: if request error
+        :return: latest MDB version matched with latest tested major
+        :rtype: str
+        """
+        try:
+            # Download the keyring file
+            response = requests.get(
+                MDB_LATEST_RELEASES_URL, timeout=REQUEST_TIMEOUT
+            )
+            response.raise_for_status()
+            latest_version_nums = [
+                ver
+                for ver in response.text.split(' ')
+                if MDB_LATEST_TESTED_MAJOR in ver
+            ]
+            if not latest_version_nums:
+                raise CMAPIBasicError(
+                    'Failed to get latest MDB version number matched latest '
+                    f'tested major {MDB_LATEST_TESTED_MAJOR}'
+            )
+            latest_version_num = sorted(latest_version_nums, reverse=True)[0]
+            logging.debug(
+                'Succesfully got latest MBD version number: '
+                f'{latest_version_num}'
+            )
+        except requests.RequestException as exc:
+            raise CMAPIBasicError(
+                'Failed to get latest MDB version numbers from '
+                f'{MDB_LATEST_RELEASES_URL}'
+            )
+        return latest_version_num
+
+    @classmethod
+    def get_ver_of(cls, package_name: str, os_type: str,) -> str:
+        """Get version of package in a repo.
+
+        :param package_name: name of package to get version
+        :type package_name: str
+        :param os_type: os name
+        :type os_type: str
+        :raises CMAPIBasicError: if os type isn't supported
+        :raises CMAPIBasicError: if failed getting package info
+        :raises CMAPIBasicError: if couldn't find any package with given name
+        :return: latest available package version
+        :rtype: str
+        """
+        latest_version: str = ''
+        cmd: str = ''
+        if os_type in ['ubuntu', 'debian']:
+            cmd = f'apt show {package_name}'
+        elif os_type in ['centos', 'rhel', 'rocky']:
+            cmd = f'yum info --showduplicates --available {package_name}'
+        else:
+            raise CMAPIBasicError(f'Unsupported OS type: {os_type}')
+
+        success, result = BaseDispatcher.exec_command(cmd)
+        if not success:
+            message = (
+                f'Failed to get {package_name} package information using '
+                f'command {cmd} with result: {result}'
+            )
+            logging.error(message)
+            raise CMAPIBasicError(message)
+
+        matches = re.findall(r'\bVersion\s*:\s+(\S+)', result)
+        if matches:
+            latest_version = max(matches, key=ComparableVersion)
+        else:
+            raise CMAPIBasicError(
+                'Could not find any version for package with name '
+                f'{package_name}'
+            )
+        return latest_version
+
+    def check_repo(self):
+        """Check repo installed and have needed version of MDB.
+
+        :raises CMAPIBasicError: if unsupported os type detected
+        :raises CMAPIBasicError: could not find package matching the version
+        """
+        pkg_ver: str = ''
+        mdb_pkg_mgr_version = self.get_ver_of('mariadb-server', self.os_type)
+        if self.os_type in ['ubuntu', 'debian']:
+            # for deb packages it's just a part of version
+            # eg: 10.6.22.18 and 1:10.6.22.18+maria~ubu2204
+            pkg_ver = self.mariadb_version.replace('-', '.')
+        elif self.os_type in ['centos', 'rhel', 'rocky']:
+            # in rhel distros it's full version
+            pkg_ver = self.mariadb_version.replace('-', '_')
+        else:
+            raise CMAPIBasicError(f'Unsupported OS type: {self.os_type}')
+        if pkg_ver not in mdb_pkg_mgr_version:
+            raise CMAPIBasicError(
+                'Could not find mariadb-server package matched with version '
+                f'{pkg_ver}'
+            )
+
+    def setup_repo(self):
+        """Set up the MariaDB Enterprise repository based on OS type."""
+        self.check_mdb_version_exists()
+        if self.os_type in ['ubuntu', 'debian']:
+            # get only two first numbers from version to build repo link
+            repo_data = ES_REPO.deb.format(
+                token=self.token,
+                mdb_version=self.mariadb_version,
+                os_version=self.os_version
+            )
+            repo_file = '/etc/apt/sources.list.d/mariadb.list'
+            with open(repo_file, 'w', encoding='utf-8') as f:
+                f.write(repo_data)
+            # Set permissions to 640
+            os.chmod(repo_file, 0o640)
+
+            pref_file = '/etc/apt/preferences'
+            with open(pref_file, 'w', encoding='utf-8') as f:
+                f.write(ES_REPO_PRIORITY_PREFS)
+
+            self._import_mariadb_keyring()
+            subprocess.run(['apt-get', 'update'], check=True)
+        elif self.os_type in ['centos', 'rhel', 'rocky']:
+            repo_data = ES_REPO.rhel.format(
+                token=self.token,
+                mdb_version=self.mariadb_version,
+                os_major_version=self.os_version.split('.', maxsplit=1)[0],
+                arch=self.arch,
+                gpg_key_url=MDB_GPG_KEY_URL
+            )
+            repo_file = '/etc/yum.repos.d/mariadb.repo'
+            with open(repo_file, 'w', encoding='utf-8') as f:
+                f.write(repo_data)
+            subprocess.run(['rpm', '--import', MDB_GPG_KEY_URL], check=True)
+        else:
+            raise CMAPIBasicError(f'Unsupported OS type: {self.os_type}')
+        self.check_repo()

cmapi/cmapi_server/managers/upgrade/upgrade.py

@@ -0,0 +1,26 @@
+    def updowngrade(self):
+        check_gtid_strict_mode()
+        # check_mariadb_versions
+
+        # Stop All
+        init_cs_down
+        wait_cs_down
+        stop_mariadb
+        stop_cmapi
+
+        # Make backups of configurations, dbrms
+        pre_upgrade_dbrm_backup
+        pre_upgrade_configuration_backup
+
+        # Upgrade
+        do_enterprise_upgrade
+
+        # Start All
+        printf "\nStartup\n"
+        start_mariadb
+        start_cmapi
+        init_cs_up
+
+        # Post Upgrade
+        confirm_dbrmctl_ok
+        run_mariadb_upgrade

cmapi/cmapi_server/managers/upgrade/utils.py

@@ -0,0 +1,44 @@
+import re
+from dataclasses import dataclass
+from functools import total_ordering
+
+
+@total_ordering
+@dataclass
+class ComparableVersion:
+    version: str
+
+    def __post_init__(self):
+        self.version_nums = self._split_version(self.version)
+
+    def _split_version(self, version: str) -> list[int]:
+        # Drop epoch if present (debian style)
+        if ':' in version:
+            version = version.split(':', 1)[1]
+
+        # I'm not sure if ~ could be the first, but seems to be possible in some debians
+        # Trim at first suffix marker (+, ~) if any
+        for marker in ('+', '~'):
+            if marker in version:
+                version = version.split(marker, 1)[0]
+                break
+        # Remove leading .0s and split by . _ -
+        parts = re.split(r'[._-]', version)
+        return [int(p.lstrip('0') or '0') for p in parts]
+
+    def _normalized(self, other) -> tuple[list, list]:
+        """Return two zero-padded version lists of equal length."""
+        v1 = self.version_nums.copy()
+        v2 = other.version_nums.copy()
+        max_len = max(len(v1), len(v2))
+        v1 += [0] * (max_len - len(v1))
+        v2 += [0] * (max_len - len(v2))
+        return v1, v2
+
+    def __eq__(self, other):
+        v1, v2 = self._normalized(other)
+        return v1 == v2
+
+    def __lt__(self, other):
+        v1, v2 = self._normalized(other)
+        return v1 < v2

cmapi/cmapi_server/test/config_apply_example.py

@@ -4,7 +4,7 @@ import configparser
 from pathlib import Path
 from datetime import datetime
 
-from cmapi_server.controllers.dispatcher import _version
+from cmapi_server.constants import _version
 
 config_filename = './cmapi_server/cmapi_server.conf'
 

cmapi/cmapi_server/test/test_cluster.py

@@ -6,8 +6,7 @@ from shutil import copyfile
 
 import requests
 
-from cmapi_server.constants import MCSProgs
+from cmapi_server.constants import MCSProgs, _version
-from cmapi_server.controllers.dispatcher import _version
 from cmapi_server.managers.process import MCSProcessManager
 from cmapi_server.test.unittest_global import (
     COPY_MCS_CONFIG_FILEPATH,

cmapi/cmapi_server/test/test_em_endpoints.py

@@ -11,11 +11,9 @@ import requests
 requests.packages.urllib3.disable_warnings()
 
 from cmapi_server.constants import (
-    EM_PATH_SUFFIX, MCS_EM_PATH, MCS_BRM_CURRENT_PATH, S3_BRM_CURRENT_PATH
+    EM_PATH_SUFFIX, MCS_EM_PATH, MCS_BRM_CURRENT_PATH, S3_BRM_CURRENT_PATH, _version
-)
-from cmapi_server.controllers.dispatcher import (
-    dispatcher, jsonify_error,_version
 )
+from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error
 from cmapi_server.managers.certificate import CertificateManager
 from cmapi_server.test.unittest_global import (
     cmapi_config_filename, tmp_cmapi_config_filename

cmapi/mcs_cluster_tool/README.md

@@ -15,14 +15,15 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 
 **Commands**:
 
-* `backup`: Backup Columnstore and/or MariaDB server data.
+* `backup`: Backup Columnstore and/or MariaDB data.
 * `dbrm_backup`: Columnstore DBRM Backup.
-* `restore`: Restore Columnstore (and/or MariaDB server) data.
+* `restore`: Restore Columnstore (and/or MariaDB) data.
 * `dbrm_restore`: Restore Columnstore DBRM data.
 * `cskeys`: Generate a random AES encryption key and init vector and write them to disk.
 * `cspasswd`: Encrypt a Columnstore plaintext password.
 * `bootstrap-single-node`: Bootstrap a single node (localhost)...
 * `review`: Provides useful functions to review and troubleshoot the MCS cluster.
+* `install_es`: [Beta] Install the specified MDB ES version.
 * `help-all`: Show help for all commands in man page style.
 * `status`: Get status information.
 * `stop`: Stop the Columnstore cluster.
@@ -32,6 +33,7 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 * `set`: Set cluster parameters.
 * `cluster`: MariaDB Columnstore cluster management...
 * `cmapi`: Commands related to CMAPI itself.
+* `sentry`: Manage Sentry DSN configuration for error...
 
 ## `mcs backup`
 
@@ -247,10 +249,10 @@ $ mcs review [OPTIONS]
 
 * `--version`: Only show the header with version information.
 * `--logs`: Create a compressed archive of logs for MariaDB Support Ticket
-* `--path`: Define the path for where to save files/tarballs and outputs of this script.
+* `--path TEXT`: Define the path for where to save files/tarballs and outputs of this script.
 * `--backupdbrm`: Takes a compressed backup of extent map files in dbrm directory.
 * `--testschema`: Creates a test schema, tables, imports, queries, drops schema.
-* `--testschemakeep`: creates a test schema, tables, imports, queries, does not drop.
+* `--testschemakeep`: Creates a test schema, tables, imports, queries, does not drop.
 * `--ldlischema`: Using ldli, creates test schema, tables, imports, queries, drops schema.
 * `--ldlischemakeep`: Using ldli, creates test schema, tables, imports, queries, does not drop.
 * `--emptydirs`: Searches /var/lib/columnstore for empty directories.
@@ -267,6 +269,26 @@ $ mcs review [OPTIONS]
 * `--color TEXT`: print headers in color. Options:  prefix color with l for light.
 * `--help`: Show this message and exit.
 
+## `mcs install_es`
+
+[Beta]
+Install the specified MDB ES version.
+If the version is 'latest', it will upgrade to the latest tested version
+available.
+
+**Usage**:
+
+```console
+$ mcs install_es [OPTIONS]
+```
+
+**Options**:
+
+* `--token TEXT`: ES API Token to use for the upgrade.  [required]
+* `-v, --version TEXT`: ES version to upgdate.
+* `--ignore-mismatch`: Proceed even if nodes report different installed package versions (use majority as baseline).
+* `--help`: Show this message and exit.
+
 ## `mcs help-all`
 
 Show help for all commands in man page style.
@@ -367,6 +389,7 @@ $ mcs node add [OPTIONS]
 **Options**:
 
 * `--node TEXT`: node IP, name or FQDN. Can be used multiple times to add several nodes at a time.  [required]
+* `--read-replica`: Add node (or nodes, if more than one is passed) as read replicas.
 * `--help`: Show this message and exit.
 
 ### `mcs node remove`
@@ -566,6 +589,7 @@ $ mcs cluster node add [OPTIONS]
 **Options**:
 
 * `--node TEXT`: node IP, name or FQDN. Can be used multiple times to add several nodes at a time.  [required]
+* `--read-replica`: Add node (or nodes, if more than one is passed) as read replicas.
 * `--help`: Show this message and exit.
 
 #### `mcs cluster node remove`
@@ -684,3 +708,70 @@ $ mcs cmapi is-ready [OPTIONS]
 
 * `--node TEXT`: Which node to check the CMAPI is ready to handle requests.  [default: 127.0.0.1]
 * `--help`: Show this message and exit.
+
+## `mcs sentry`
+
+Manage Sentry DSN configuration for error tracking.
+
+**Usage**:
+
+```console
+$ mcs sentry [OPTIONS] COMMAND [ARGS]...
+```
+
+**Options**:
+
+* `--help`: Show this message and exit.
+
+**Commands**:
+
+* `show`: Show current Sentry DSN configuration.
+* `enable`: Enable Sentry error tracking with the...
+* `disable`: Disable Sentry error tracking by removing...
+
+### `mcs sentry show`
+
+Show current Sentry DSN configuration.
+
+**Usage**:
+
+```console
+$ mcs sentry show [OPTIONS]
+```
+
+**Options**:
+
+* `--help`: Show this message and exit.
+
+### `mcs sentry enable`
+
+Enable Sentry error tracking with the provided DSN.
+
+**Usage**:
+
+```console
+$ mcs sentry enable [OPTIONS] DSN
+```
+
+**Arguments**:
+
+* `DSN`: Sentry DSN URL to enable for error tracking.  [required]
+
+**Options**:
+
+* `-e, --environment TEXT`: Sentry environment name (default: development).  [default: development]
+* `--help`: Show this message and exit.
+
+### `mcs sentry disable`
+
+Disable Sentry error tracking by removing the configuration.
+
+**Usage**:
+
+```console
+$ mcs sentry disable [OPTIONS]
+```
+
+**Options**:
+
+* `--help`: Show this message and exit.

cmapi/mcs_cluster_tool/README_DEV.md

@@ -9,7 +9,7 @@
     ```
     Optionally could be generated from installed package.
     ```bash
-    PYTHONPATH="/usr/share/columnstore/cmapi:/usr/share/columnstore/cmapi/deps" /usr/share/columnstore/cmapi/python/bin/python3 -m  typer /usr/share/columnstore/cmapi/mcs_cluster_tool/__main__.py utils docs --name mcs --output ~/README.md
+    PYTHONPATH="/usr/share/columnstore/cmapi:/usr/share/columnstore/cmapi/deps" /usr/share/columnstore/cmapi/python/bin/python3 -m typer /usr/share/columnstore/cmapi/mcs_cluster_tool/__main__.py utils docs --name mcs --output ~/README.md
     ```
 - dependencies for gem build (RHEL example)
     ```bash

cmapi/mcs_cluster_tool/__main__.py

@@ -59,6 +59,11 @@ app.command(
 app.add_typer(
     tools_commands.sentry_app, name='sentry', rich_help_panel='Tools commands', hidden=True
 )
+app.command(
+    'install_es', rich_help_panel='Tools commands',
+)(tools_commands.install_es)
+
+
 @app.command(
         name='help-all', help='Show help for all commands in man page style.',
         add_help_option=False
@@ -67,6 +72,7 @@ def help_all():
     # Open the man page in interactive mode
     subprocess.run(['man', 'mcs'])
 
+
 @app.callback()
 def main(verbose: bool = typer.Option(False, '--verbose', '-v', help='Enable verbose logging to console')):
     '''Add a -v option and setup logging in every subcommand'''
@@ -78,9 +84,9 @@ def setup_logging(verbose: bool = False) -> None:
     dict_config(MCS_CLI_LOG_CONF_PATH)
     if verbose:
         for logger_name in ("", "mcs_cli"):
-            logger = logging.getLogger(logger_name)
+            current_logger = logging.getLogger(logger_name)
-            logger.setLevel(logging.DEBUG)
+            current_logger.setLevel(logging.DEBUG)
-            enable_console_logging(logger)
+            enable_console_logging(current_logger)
 
 
 if __name__ == '__main__':

cmapi/mcs_cluster_tool/backup_commands.py

@@ -7,7 +7,7 @@ import typer
 from typing_extensions import Annotated
 
 from cmapi_server.process_dispatchers.base import BaseDispatcher
-from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH
+from cmapi_server.constants import MCS_BACKUP_MANAGER_SH
 from mcs_cluster_tool.decorators import handle_output
 from mcs_cluster_tool.helpers import cook_sh_arg
 

cmapi/mcs_cluster_tool/constants.py

@@ -6,7 +6,8 @@ from cmapi_server.constants import MCS_INSTALL_BIN
 MCS_CLI_ROOT_PATH = os.path.dirname(__file__)
 MCS_CLI_LOG_CONF_PATH =  os.path.join(MCS_CLI_ROOT_PATH, 'mcs_cli_log.conf')
 
-MCS_BACKUP_MANAGER_SH = os.path.join(MCS_INSTALL_BIN, 'mcs_backup_manager.sh')
 MCS_COLUMNSTORE_REVIEW_SH = os.path.join(
     MCS_INSTALL_BIN, 'columnstore_review.sh'
-)
+)
+
+INSTALL_ES_LOG_FILEPATH = '/var/tmp/mcs_cli_install_es.log'

cmapi/mcs_cluster_tool/mcs.1

@@ -21,7 +21,7 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 \fBCommands\fP:
 .RS
 .IP \(bu 2
-\fB\fCbackup\fR: Backup Columnstore and/or MariDB data.
+\fB\fCbackup\fR: Backup Columnstore and/or MariaDB data.
 .IP \(bu 2
 \fB\fCdbrm_backup\fR: Columnstore DBRM Backup.
 .IP \(bu 2
@@ -29,7 +29,7 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 .IP \(bu 2
 \fB\fCdbrm_restore\fR: Restore Columnstore DBRM data.
 .IP \(bu 2
-\fB\fCcskeys\fR: Generates a random AES encryption key and init vector and writes them to disk.
+\fB\fCcskeys\fR: Generate a random AES encryption key and init vector and write them to disk.
 .IP \(bu 2
 \fB\fCcspasswd\fR: Encrypt a Columnstore plaintext password.
 .IP \(bu 2
@@ -37,6 +37,8 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 .IP \(bu 2
 \fB\fCreview\fR: Provides useful functions to review and troubleshoot the MCS cluster.
 .IP \(bu 2
+\fB\fCinstall_es\fR: [Beta] Install the specified MDB ES version.
+.IP \(bu 2
 \fB\fChelp\-all\fR: Show help for all commands in man page style.
 .IP \(bu 2
 \fB\fCstatus\fR: Get status information.
@@ -53,11 +55,13 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 .IP \(bu 2
 \fB\fCcluster\fR: MariaDB Columnstore cluster management...
 .IP \(bu 2
-\fB\fCcmapi\fR: CMAPI itself related commands.
+\fB\fCcmapi\fR: Commands related to CMAPI itself.
+.IP \(bu 2
+\fB\fCsentry\fR: Manage Sentry DSN configuration for error...
 .RE
 .SH \fB\fCmcs backup\fR
 .PP
-Backup Columnstore and/or MariDB data.
+Backup Columnstore and/or MariaDB data.
 .PP
 \fBUsage\fP:
 .PP
@@ -362,13 +366,13 @@ $ mcs review [OPTIONS]
 .IP \(bu 2
 \fB\fC\-\-logs\fR: Create a compressed archive of logs for MariaDB Support Ticket
 .IP \(bu 2
-\fB\fC\-\-path\fR: Define the path for where to save files/tarballs and outputs of this script.
+\fB\fC\-\-path TEXT\fR: Define the path for where to save files/tarballs and outputs of this script.
 .IP \(bu 2
 \fB\fC\-\-backupdbrm\fR: Takes a compressed backup of extent map files in dbrm directory.
 .IP \(bu 2
 \fB\fC\-\-testschema\fR: Creates a test schema, tables, imports, queries, drops schema.
 .IP \(bu 2
-\fB\fC\-\-testschemakeep\fR: creates a test schema, tables, imports, queries, does not drop.
+\fB\fC\-\-testschemakeep\fR: Creates a test schema, tables, imports, queries, does not drop.
 .IP \(bu 2
 \fB\fC\-\-ldlischema\fR: Using ldli, creates test schema, tables, imports, queries, drops schema.
 .IP \(bu 2
@@ -400,6 +404,32 @@ $ mcs review [OPTIONS]
 .IP \(bu 2
 \fB\fC\-\-help\fR: Show this message and exit.
 .RE
+.SH \fB\fCmcs install_es\fR
+.PP
+[Beta]
+Install the specified MDB ES version.
+If the version is \[aq]latest\[aq], it will upgrade to the latest tested version
+available.
+.PP
+\fBUsage\fP:
+.PP
+.RS
+.nf
+$ mcs install_es [OPTIONS]
+.fi
+.RE
+.PP
+\fBOptions\fP:
+.RS
+.IP \(bu 2
+\fB\fC\-\-token TEXT\fR: ES API Token to use for the upgrade.  [required]
+.IP \(bu 2
+\fB\fC\-v, \-\-version TEXT\fR: ES version to upgdate.
+.IP \(bu 2
+\fB\fC\-\-ignore\-mismatch\fR: Proceed even if nodes report different installed package versions (use majority as baseline).
+.IP \(bu 2
+\fB\fC\-\-help\fR: Show this message and exit.
+.RE
 .SH \fB\fCmcs help\-all\fR
 .PP
 Show help for all commands in man page style.
@@ -525,6 +555,8 @@ $ mcs node add [OPTIONS]
 .IP \(bu 2
 \fB\fC\-\-node TEXT\fR: node IP, name or FQDN. Can be used multiple times to add several nodes at a time.  [required]
 .IP \(bu 2
+\fB\fC\-\-read\-replica\fR: Add node (or nodes, if more than one is passed) as read replicas.
+.IP \(bu 2
 \fB\fC\-\-help\fR: Show this message and exit.
 .RE
 .SS \fB\fCmcs node remove\fR
@@ -781,6 +813,8 @@ $ mcs cluster node add [OPTIONS]
 .IP \(bu 2
 \fB\fC\-\-node TEXT\fR: node IP, name or FQDN. Can be used multiple times to add several nodes at a time.  [required]
 .IP \(bu 2
+\fB\fC\-\-read\-replica\fR: Add node (or nodes, if more than one is passed) as read replicas.
+.IP \(bu 2
 \fB\fC\-\-help\fR: Show this message and exit.
 .RE
 .SS \fB\fCmcs cluster node remove\fR
@@ -892,7 +926,7 @@ $ mcs cluster set log\-level [OPTIONS]
 .RE
 .SH \fB\fCmcs cmapi\fR
 .PP
-CMAPI itself related commands.
+Commands related to CMAPI itself.
 .PP
 \fBUsage\fP:
 .PP
@@ -932,3 +966,89 @@ $ mcs cmapi is\-ready [OPTIONS]
 .IP \(bu 2
 \fB\fC\-\-help\fR: Show this message and exit.
 .RE
+.SH \fB\fCmcs sentry\fR
+.PP
+Manage Sentry DSN configuration for error tracking.
+.PP
+\fBUsage\fP:
+.PP
+.RS
+.nf
+$ mcs sentry [OPTIONS] COMMAND [ARGS]...
+.fi
+.RE
+.PP
+\fBOptions\fP:
+.RS
+.IP \(bu 2
+\fB\fC\-\-help\fR: Show this message and exit.
+.RE
+.PP
+\fBCommands\fP:
+.RS
+.IP \(bu 2
+\fB\fCshow\fR: Show current Sentry DSN configuration.
+.IP \(bu 2
+\fB\fCenable\fR: Enable Sentry error tracking with the...
+.IP \(bu 2
+\fB\fCdisable\fR: Disable Sentry error tracking by removing...
+.RE
+.SS \fB\fCmcs sentry show\fR
+.PP
+Show current Sentry DSN configuration.
+.PP
+\fBUsage\fP:
+.PP
+.RS
+.nf
+$ mcs sentry show [OPTIONS]
+.fi
+.RE
+.PP
+\fBOptions\fP:
+.RS
+.IP \(bu 2
+\fB\fC\-\-help\fR: Show this message and exit.
+.RE
+.SS \fB\fCmcs sentry enable\fR
+.PP
+Enable Sentry error tracking with the provided DSN.
+.PP
+\fBUsage\fP:
+.PP
+.RS
+.nf
+$ mcs sentry enable [OPTIONS] DSN
+.fi
+.RE
+.PP
+\fBArguments\fP:
+.RS
+.IP \(bu 2
+\fB\fCDSN\fR: Sentry DSN URL to enable for error tracking.  [required]
+.RE
+.PP
+\fBOptions\fP:
+.RS
+.IP \(bu 2
+\fB\fC\-e, \-\-environment TEXT\fR: Sentry environment name (default: development).  [default: development]
+.IP \(bu 2
+\fB\fC\-\-help\fR: Show this message and exit.
+.RE
+.SS \fB\fCmcs sentry disable\fR
+.PP
+Disable Sentry error tracking by removing the configuration.
+.PP
+\fBUsage\fP:
+.PP
+.RS
+.nf
+$ mcs sentry disable [OPTIONS]
+.fi
+.RE
+.PP
+\fBOptions\fP:
+.RS
+.IP \(bu 2
+\fB\fC\-\-help\fR: Show this message and exit.
+.RE

cmapi/mcs_cluster_tool/mcs_cli_log.conf

@@ -9,6 +9,7 @@
     },
     "handlers": {
         "file": {
+            "level": "DEBUG",
             "class" : "logging.handlers.RotatingFileHandler",
             "formatter": "default",
             "filename": "/var/log/mariadb/columnstore/mcs_cli.log",

cmapi/mcs_cluster_tool/restore_commands.py

@@ -7,7 +7,7 @@ import typer
 from typing_extensions import Annotated
 
 from cmapi_server.process_dispatchers.base import BaseDispatcher
-from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH
+from cmapi_server.constants import MCS_BACKUP_MANAGER_SH
 from mcs_cluster_tool.decorators import handle_output
 from mcs_cluster_tool.helpers import cook_sh_arg
 

cmapi/mcs_cluster_tool/tools_commands.py

@@ -2,24 +2,36 @@ import logging
 import os
 import secrets
 import sys
-from datetime import datetime
+import time
+from datetime import datetime, timedelta
 from typing import Optional
+import ast
+from collections import Counter
 
+import requests
 import typer
 from typing_extensions import Annotated
+from rich.console import Console
+from rich.progress import (
+    BarColumn, Progress, SpinnerColumn, TimeElapsedColumn,
+)
+from rich.table import Table
 
 
 from cmapi_server.constants import (
     MCS_DATA_PATH, MCS_SECRETS_FILENAME, REQUEST_TIMEOUT, TRANSACTION_TIMEOUT,
-    CMAPI_CONF_PATH,
+    CMAPI_CONF_PATH, CMAPI_PORT,
 )
-from cmapi_server.controllers.api_clients import ClusterControllerClient
+from cmapi_server.controllers.api_clients import (
-from cmapi_server.exceptions import CEJError
+    AppControllerClient, ClusterControllerClient, NodeControllerClient
+)
+from cmapi_server.exceptions import CEJError, CMAPIBasicError
 from cmapi_server.handlers.cej import CEJPasswordHandler
-from cmapi_server.helpers import get_config_parser
+from cmapi_server.helpers import get_active_nodes, get_config_parser
 from cmapi_server.managers.transaction import TransactionManager
+from cmapi_server.managers.upgrade.utils import ComparableVersion
 from cmapi_server.process_dispatchers.base import BaseDispatcher
-from mcs_cluster_tool.constants import MCS_COLUMNSTORE_REVIEW_SH
+from mcs_cluster_tool.constants import MCS_COLUMNSTORE_REVIEW_SH, INSTALL_ES_LOG_FILEPATH
 from mcs_cluster_tool.decorators import handle_output
 from mcs_cluster_tool.helpers import cook_sh_arg
 
@@ -142,7 +154,7 @@ def bootstrap_single_node(
     client = ClusterControllerClient(request_timeout=REQUEST_TIMEOUT)
     if not key:
         # Generate API key if not provided
-        key = secrets.token_urlsafe(32)
+        key = secrets.token_urlsafe(32)  #pylint: disable=no-member
     # handle_output decorator will catch, show and log errors
     api_key_set_resp = client.set_api_key(key)
     # if operation takes minutes, then it is better to raise by timeout
@@ -395,25 +407,25 @@ def show():
     try:
         # Read existing config
         cfg_parser = get_config_parser(CMAPI_CONF_PATH)
-        
+
         if not cfg_parser.has_section('Sentry'):
             typer.echo('Sentry is disabled (no configuration found).', color='yellow')
             raise typer.Exit(code=0)
-        
+
         dsn = cfg_parser.get('Sentry', 'dsn', fallback='').strip().strip("'\"")
         environment = cfg_parser.get('Sentry', 'environment', fallback='development').strip().strip("'\"")
-        
+
         if not dsn:
             typer.echo('Sentry is disabled (DSN is empty).', color='yellow')
         else:
             typer.echo('Sentry is enabled:', color='green')
             typer.echo(f'  DSN: {dsn}')
             typer.echo(f'  Environment: {environment}')
-        
+
     except Exception as e:
         typer.echo(f'Error reading configuration: {str(e)}', color='red')
         raise typer.Exit(code=1)
-    
+
     raise typer.Exit(code=0)
 
 
@@ -438,31 +450,31 @@ def enable(
     if not dsn:
         typer.echo('DSN cannot be empty.', color='red')
         raise typer.Exit(code=1)
-    
+
     try:
         # Read existing config
         cfg_parser = get_config_parser(CMAPI_CONF_PATH)
-        
+
         # Add or update Sentry section
         if not cfg_parser.has_section('Sentry'):
             cfg_parser.add_section('Sentry')
-        
+
         cfg_parser.set('Sentry', 'dsn', f"'{dsn}'")
         cfg_parser.set('Sentry', 'environment', f"'{environment}'")
-        
+
         # Write config back to file
         with open(CMAPI_CONF_PATH, 'w') as config_file:
             cfg_parser.write(config_file)
-        
+
         typer.echo('Sentry error tracking enabled successfully.', color='green')
         typer.echo(f'DSN: {dsn}', color='green')
         typer.echo(f'Environment: {environment}', color='green')
         typer.echo('Note: Restart cmapi service for changes to take effect.', color='yellow')
-        
+
     except Exception as e:
         typer.echo(f'Error updating configuration: {str(e)}', color='red')
         raise typer.Exit(code=1)
-    
+
     raise typer.Exit(code=0)
 
 
@@ -473,23 +485,489 @@ def disable():
     try:
         # Read existing config
         cfg_parser = get_config_parser(CMAPI_CONF_PATH)
-        
+
         if not cfg_parser.has_section('Sentry'):
             typer.echo('Sentry is already disabled (no configuration found).', color='yellow')
             raise typer.Exit(code=0)
-        
+
         # Remove the entire Sentry section
         cfg_parser.remove_section('Sentry')
-        
+
         # Write config back to file
         with open(CMAPI_CONF_PATH, 'w') as config_file:
             cfg_parser.write(config_file)
-        
+
         typer.echo('Sentry error tracking disabled successfully.', color='green')
         typer.echo('Note: Restart cmapi service for changes to take effect.', color='yellow')
-        
+
     except Exception as e:
         typer.echo(f'Error updating configuration: {str(e)}', color='red')
         raise typer.Exit(code=1)
-    
+
-    raise typer.Exit(code=0)
+    raise typer.Exit(code=0)
+
+
+@handle_output
+def healthcheck():
+    """Check the health of the MCS cluster."""
+    with TransactionManager(
+        timeout=timedelta(minutes=5).total_seconds(), handle_signals=True,
+    ):
+        client = ClusterControllerClient(request_timeout=REQUEST_TIMEOUT)
+        result = client.get_health({'in_transaction': True})
+        # TODO: just a placeholder for now
+        #       need to implement result in a table format with color
+        typer.echo(
+            'Cluster health check completed successfully.',
+            color='green'
+        )
+    raise typer.Exit(code=0)
+
+
+@handle_output
+def install_es(
+    token: Annotated[
+        str,
+        typer.Option(
+            '--token',
+            help='ES API Token to use for the upgrade.',
+            show_default=False
+        )
+    ],
+    target_version: Annotated[
+        str,
+        typer.Option(
+            '-v', '--version',
+            help='ES version to upgdate.',
+            show_default=False
+        )
+    ] = 'latest',
+    ignore_mismatch: Annotated[
+        bool,
+        typer.Option(
+            '--ignore-mismatch',
+            help='Proceed even if nodes report different installed package versions (use majority as baseline).',
+            show_default=False
+        )
+    ] = False,
+):
+    """
+    [Beta]
+    Install the specified MDB ES version.
+    If the version is 'latest', it will upgrade to the latest tested version
+    available.
+    """
+    new_handler = logging.FileHandler(INSTALL_ES_LOG_FILEPATH, mode='w')
+    new_handler.setLevel(logging.DEBUG)
+    new_handler.setFormatter(logging.getLogger('mcs_cli').handlers[0].formatter)
+    for logger_name in ("", "mcs_cli"):
+        current_logger = logging.getLogger(logger_name)
+        current_logger.addHandler(new_handler)
+    console = Console()
+    console.clear()
+    console.rule('[bold green][Beta] MariaDB ES Installer')
+
+    console.print('This utility is now in Beta.', style='yellow underline')
+    console.print(
+        (
+            'Downgrades are supported up to MariaDB 10.6.9-5 and Columnstore 22.08.4.'
+            'Make sure you have a backup of your data before proceeding. '
+            'If you encounter any issues, please report them to MariaDB Support.'
+        ),
+        style='underline'
+    )
+
+    # Collect output (tables/messages) to render AFTER the progress bar finishes
+    post_output: list = []  # items can be strings with rich markup or Rich renderables
+    exit_code: int = 0
+    def post_print(msg: str, color: Optional[str] = None):
+        if color:
+            post_output.append(f'[{color}]{msg}[/{color}]')
+        else:
+            post_output.append(msg)
+
+    active_nodes = get_active_nodes()
+
+    node_api_client = NodeControllerClient()
+    cluster_api_client = ClusterControllerClient()
+    app_api_client = AppControllerClient()
+
+
+    node_api_client.validate_es_token(token)
+    if target_version == 'latest':
+        response = node_api_client.get_latest_mdb_version()
+        target_version = response['latest_mdb_version']
+    else:
+        try:
+            node_api_client.validate_mdb_version(token, target_version, throw_real_exp=True)
+        except requests.exceptions.HTTPError as exc:
+            resp = exc.response
+            error_msg = str(exc)
+            if resp.status_code == 422:
+                try:
+                    resp_json = resp.json()
+                    error_msg = resp_json.get('error', resp_json)
+                except requests.exceptions.JSONDecodeError:
+                    error_msg = resp.text
+            console.print('ERROR:', style='red')
+            console.print(error_msg, style='underline')
+            console.rule()
+            raise typer.Exit(code=1)
+
+    # Retrieve current versions; if nodes are mismatched, prettify the server error.
+    # If --ignore-mismatch is passed we will continue, choosing the majority version
+    # of each package as the baseline "current" version.
+    try:
+        versions = cluster_api_client.get_versions()
+    except CMAPIBasicError as exc:  # custom API client error
+        msg = exc.message
+        mismatch_marker = 'Packages versions:'
+        if mismatch_marker in msg:
+            try:
+                dict_part = msg.split(mismatch_marker, 1)[1].strip()
+                packages_versions = ast.literal_eval(dict_part)
+            except Exception:  # pragma: no cover - defensive
+                # Could not parse, fall back to original behavior
+                console.print(f"[red]{msg}[/red]")
+                raise typer.Exit(code=1)
+
+            console.print('Detected package version mismatch across nodes:', style='yellow')
+            mismatch_table = Table('Node', 'Server', 'Columnstore', 'CMAPI')
+
+            server_vals = [v.get('server_version') for v in packages_versions.values()]
+            cs_vals = [v.get('columnstore_version') for v in packages_versions.values()]
+            cmapi_vals = [v.get('cmapi_version') for v in packages_versions.values()]
+            server_common = Counter(server_vals).most_common(1)[0][0] if server_vals else None
+            cs_common = Counter(cs_vals).most_common(1)[0][0] if cs_vals else None
+            cmapi_common = Counter(cmapi_vals).most_common(1)[0][0] if cmapi_vals else None
+
+            def style(val, common):
+                if val is None:
+                    return '[red]-[/red]'
+                return f'[green]{val}[/green]' if val == common else f'[red]{val}[/red]'
+
+            for node, vers in sorted(packages_versions.items()):
+                mismatch_table.add_row(
+                    node,
+                    style(vers.get('server_version'), server_common),
+                    style(vers.get('columnstore_version'), cs_common),
+                    style(vers.get('cmapi_version'), cmapi_common),
+                )
+            # Print after progress unless we're going to exit early
+            if not ignore_mismatch:
+                # No progress has started yet; render now and exit
+                console.print(mismatch_table)
+                console.print('[yellow]All nodes must have identical package versions before running install-es. '
+                               'Please align versions (upgrade/downgrade individual nodes) and retry, '
+                               'or rerun with --ignore-mismatch to force.[/yellow]')
+                raise typer.Exit(code=1)
+
+            console.print(mismatch_table)
+            # Forced continuation path
+            console.print(
+                (
+                    'Proceeding despite mismatch ( --ignore-mismatch ). Using majority versions '
+                    'as baseline.'
+                ),
+                style='yellow'
+            )
+            versions = {
+                'server_version': server_common or server_vals[0],
+                'columnstore_version': cs_common or cs_vals[0],
+                'cmapi_version': cmapi_common or cmapi_vals[0],
+            }
+        else:
+            # Not a mismatch we recognize; rethrow for decorator to handle
+            raise
+    mdb_curr_ver = versions['server_version']
+    mcs_curr_ver = versions['columnstore_version']
+    cmapi_curr_ver = versions['cmapi_version']
+    mdb_curr_ver_comp = ComparableVersion(mdb_curr_ver)
+    mdb_target_ver_comp = ComparableVersion(target_version)
+
+    console.print('Currently installed vesions:', style='green')
+    table = Table('ES version', 'Columnstore version', 'CMAPI version')
+    table.add_row(mdb_curr_ver, mcs_curr_ver, cmapi_curr_ver)
+    console.print(table)
+    is_downgrade = False
+    if mdb_curr_ver_comp == mdb_target_ver_comp:
+        console.print('[green]The target MariaDB ES version is already installed.[/green]')
+        raise typer.Exit(code=0)
+    elif mdb_curr_ver_comp > mdb_target_ver_comp:
+        downgrade = typer.confirm(
+            'Target version is older than currently installed. '
+            'Are you sure you really want to downgrade?\n'
+            'WARNING: Could cause data loss and/or broken cluster.',
+            prompt_suffix=' '
+        )
+        if not downgrade:
+            raise typer.Exit(code=1)
+        is_downgrade = True
+    elif mdb_curr_ver_comp < mdb_target_ver_comp:
+        upgrade = typer.confirm(
+            f'Are you sure you really want to upgrade to {target_version}?',
+            prompt_suffix=' '
+        )
+        if not upgrade:
+            raise typer.Exit(code=1)
+
+    if not active_nodes:
+        post_print('No active nodes found, using localhost only.', 'yellow')
+        active_nodes.append('localhost')
+
+    with Progress(
+        SpinnerColumn(),
+        '[progress.description]{task.description}',
+        BarColumn(),
+        TimeElapsedColumn(),
+        console=console,
+    ) as progress:
+        step1_stop_cluster = progress.add_task('Stopping MCS cluster...', total=None)
+        with TransactionManager(
+            timeout=timedelta(days=1).total_seconds(), handle_signals=True
+        ):
+            cluster_stop_resp = cluster_api_client.shutdown_cluster(
+                {'in_transaction': True}
+            )
+        progress.update(
+            step1_stop_cluster, description='[green]MCS Cluster stopped ✓', total=100,
+            completed=True
+        )
+        progress.stop_task(step1_stop_cluster)
+
+        step2_stop_mariadb = progress.add_task('Stopping MariaDB server...', total=None)
+        # TODO: put MaxScale into maintainance mode
+        mariadb_stop_resp = cluster_api_client.stop_mariadb(
+            {'in_transaction': True}
+        )
+        progress.update(
+            step2_stop_mariadb, description='[green]MariaDB server stopped ✓', total=100,
+            completed=True
+        )
+        progress.stop_task(step2_stop_mariadb)
+
+        step3_install_es_repo = progress.add_task(
+            'Installing MariaDB ES repository...', total=None
+        )
+        inst_repo_response = cluster_api_client.install_repo(
+            token=token, mariadb_version=target_version
+        )
+        progress.update(
+            step3_install_es_repo, description='[green]Repository installed ✓', total=100,
+            completed=True
+        )
+        progress.stop_task(step3_install_es_repo)
+
+        if target_version == 'latest':
+            # PackageManager accepts latest versions so no need to get numeric
+            mdb_target_ver = mcs_target_ver = cmapi_target_ver = 'latest'
+        else:
+            step3_5_get_available_versions = progress.add_task(
+                'Getting available versions of packages...', total=None
+            )
+            available_versions_resp = node_api_client.repo_pkg_versions()
+            mdb_target_ver = available_versions_resp['server_version']
+            mcs_target_ver = available_versions_resp['columnstore_version']
+            cmapi_target_ver = available_versions_resp['cmapi_version']
+            progress.update(
+                step3_5_get_available_versions,
+                description=(
+                    f'[green]Available versions: ES {mdb_target_ver}, '
+                    f'Columnstore {mcs_target_ver}, CMAPI {cmapi_target_ver} ✓'
+                ),
+                total=100, completed=True
+            )
+            progress.stop_task(step3_5_get_available_versions)
+
+        step4_preupgrade_backup = progress.add_task(
+            'Starting pre-upgrade backup DBRM and configs on each node...', total=None
+        )
+        backup_response = cluster_api_client.preupgrade_backup()
+        progress.update(
+            step4_preupgrade_backup, description='[green]PreUpgrade Backup completed ✓',
+            total=100, completed=True
+        )
+        progress.stop_task(step4_preupgrade_backup)
+
+        step5_upgrade_mdb_mcs = progress.add_task(
+            'Upgrading MariaDB and Columnstore on each node...', total=None
+        )
+        mdb_mcs_upgrade_response = cluster_api_client.upgrade_mdb_mcs(
+            mariadb_version=mdb_target_ver, columnstore_version=mcs_target_ver
+        )
+        progress.update(
+            step5_upgrade_mdb_mcs,
+            description=f'[green]Upgraded to MariaDB {mdb_target_ver} and Columnstore {mcs_target_ver} ✓',
+            total=100, completed=True
+        )
+        progress.stop_task(step5_upgrade_mdb_mcs)
+
+        step6_install_cmapi = progress.add_task('Upgrading CMAPI on each node...', total=None)
+        try:
+            cmapi_upgrade_response = cluster_api_client.upgrade_cmapi(version=cmapi_target_ver)
+            # cmapi_updater service has 5 s timeout to give CMAPI time to handle response,
+            # we need to wait when API become unreachable after CMAPI stop.
+            time.sleep(6)
+        except requests.exceptions.ConnectionError:
+            # during upgrade the connection drop is expected
+            pass
+
+        # Prepare per-node readiness tracking
+        progress.update(
+            step6_install_cmapi, description='Waiting CMAPI to be ready on each node...',
+            completed=None
+        )
+        start_time = datetime.now()
+        timeout_seconds = 300
+        # status per node: {'status': 'PENDING'|'READY'|'ERROR'|'TIMEOUT', 'details': str}
+        node_states = {
+            node: {'status': 'PENDING', 'details': ''} for node in active_nodes
+        }
+
+        # Build a dedicated client per node (localhost already covered)
+        per_node_clients: dict[str, AppControllerClient] = {}
+        for node in active_nodes:
+            if node in ('localhost', '127.0.0.1'):
+                per_node_clients[node] = AppControllerClient()
+            else:
+                per_node_clients[node] = AppControllerClient(
+                    base_url=f'https://{node}:{CMAPI_PORT}'
+                )
+
+        ready_count_prev = -1
+        while (datetime.now() - start_time) < timedelta(seconds=timeout_seconds):
+            ready_count = 0
+            for node, client_obj in per_node_clients.items():
+                # Skip nodes that already finalized (READY or ERROR)
+                if node_states[node]['status'] in ('READY', 'ERROR'):
+                    if node_states[node]['status'] == 'READY':
+                        ready_count += 1
+                    continue
+                try:
+                    node_response = client_obj.get_ready()
+                    if node_response.get('started') is True:
+                        node_states[node]['status'] = 'READY'
+                        node_states[node]['details'] = 'Service started'
+                        ready_count += 1
+                except requests.exceptions.HTTPError as err:
+                    # 503 means not ready yet, anything else mark as ERROR
+                    if err.response.status_code == 503:
+                        node_states[node]['details'] = 'Starting...'
+                    else:
+                        node_states[node]['status'] = 'ERROR'
+                        node_states[node]['details'] = f'HTTP {err.response.status_code}'
+                except requests.exceptions.ConnectionError:
+                    # still restarting
+                    node_states[node]['details'] = 'Connection refused'
+                except FileNotFoundError as fnf_err:  # pragma: no cover - defensive
+                    # Transient race: config file not yet created; do not fail immediately
+                    missing_path = str(fnf_err).split(":")[-1].strip()
+                    node_states[node]['details'] = f'Config pending ({missing_path})'
+                except Exception as err:  # pragma: no cover - defensive
+                    node_states[node]['status'] = 'ERROR'
+                    node_states[node]['details'] = f'Unexpected: {err}'
+
+            # Update progress description only when count changes to reduce flicker
+            if ready_count != ready_count_prev:
+                progress.update(
+                    step6_install_cmapi,
+                    description=(
+                        f'Waiting CMAPI to be ready on each node... '
+                        f'({ready_count}/{len(active_nodes)} ready)'
+                    ),
+                    completed=None
+                )
+                ready_count_prev = ready_count
+
+            if ready_count == len(active_nodes):
+                break
+            time.sleep(1)
+
+        # Mark TIMEOUT for nodes still pending
+        for node, state in node_states.items():
+            if state['status'] == 'PENDING':
+                state['status'] = 'TIMEOUT'
+                state['details'] = f'Not ready after {timeout_seconds}s'
+
+        # Display per-node table after progress ends
+        status_table = Table('Node', 'Status', 'Details')
+        failures = False
+        for node, state in sorted(node_states.items()):
+            status = state['status']
+            details = state['details']
+            color_map = {
+                'READY': 'green',
+                'PENDING': 'yellow',
+                'TIMEOUT': 'red',
+                'ERROR': 'red',
+            }
+            style = color_map.get(status, 'white')
+            status_table.add_row(node, f'[{style}]{status}[/{style}]', details)
+            if status in ('TIMEOUT', 'ERROR'):
+                failures = True
+        # Defer table rendering
+        post_output.append(status_table)
+
+        if failures:
+            progress.update(
+                step6_install_cmapi,
+                description='[red]CMAPI did not start successfully on all nodes ✗',
+                total=100,
+                completed=True
+            )
+            progress.stop_task(step6_install_cmapi)
+            exit_code = 1
+            post_print('CMAPI did not start successfully on all nodes.', 'red')
+        else:
+            progress.update(
+                step6_install_cmapi,
+                description='[green]CMAPI is ready on all nodes ✓',
+                total=100,
+                completed=True
+            )
+            progress.stop_task(step6_install_cmapi)
+
+        if failures:
+            # skip any automatic restarts on failure
+            pass
+        elif is_downgrade:
+            note_panel = Table('Action', 'Status')
+            note_panel.add_row('Automatic restart (MariaDB, Cluster, Health)', '[yellow]SKIPPED (downgrade)')
+            post_output.append(note_panel)
+            post_print(
+                'Downgrade detected: automatic service restarts were skipped. '
+                'Please manually start MariaDB and the ColumnStore cluster, and verify health.',
+                'yellow'
+            )
+            post_print('Suggested manual sequence:', 'yellow')
+            post_print('  1) systemctl start mariadb', 'yellow')
+            post_print('  2) Use mcs-cluster tool to start cluster if needed', 'yellow')
+            exit_code = 0
+        else:
+            step7_start_mariadb = progress.add_task('Starting MariaDB server...', total=None)
+            # TODO: put MaxScale from maintainance into working mode
+            mariadb_start_resp = cluster_api_client.start_mariadb({'in_transaction': True})
+            progress.update(
+                step7_start_mariadb, description='[green]MariaDB server started ✓', completed=True
+            )
+            progress.stop_task(step7_start_mariadb)
+
+            step8_start_cluster = progress.add_task('Starting MCS cluster...', total=None)
+            cluster_start_resp = cluster_api_client.start_cluster(
+                {'in_transaction': True}
+            )
+            with TransactionManager(
+                timeout=timedelta(days=1).total_seconds(), handle_signals=True
+            ):
+                cluster_start_resp = cluster_api_client.start_cluster({'in_transaction': True})
+            progress.update(
+                step8_start_cluster, description='[green]MCS Cluster started ✓', completed=True
+            )
+            progress.stop_task(step8_start_cluster)
+            post_print('Upgrade completed and services restarted successfully.', 'green')
+
+    # Render any deferred output now that the progress bar is complete
+    for item in post_output:
+        console.print(item)
+
+    raise typer.Exit(code=exit_code)

cmapi/requirements.in

@@ -5,6 +5,7 @@ aiohttp==3.11.16
 awscli==1.38.28
 CherryPy==18.10.0
 cryptography==43.0.3
+distro==1.9.0
 furl==2.1.4
 gsutil==5.33
 lxml==5.3.2
@@ -12,7 +13,7 @@ psutil==7.0.0
 pyotp==2.9.0
 requests==2.32.3
 # required for CherryPy RoutesDispatcher,
-# but CherryPy itself has no such dependency
+# but CherryPy itself has no such a dependency
 Routes==2.5.1
 typer==0.15.2
 sentry-sdk==2.34.1

cmapi/requirements.txt

@@ -4,6 +4,23 @@
 #
 #    dev_tools/piptools.sh compile-all
 #
+aiohttp==3.11.16
+awscli==1.38.28
+CherryPy==18.10.0
+cryptography==43.0.3
+distro==1.9.0
+furl==2.1.4
+gsutil==5.33
+lxml==5.3.2
+psutil==7.0.0
+pyotp==2.9.0
+requests==2.32.3
+# required for CherryPy RoutesDispatcher,
+# but CherryPy itself has no such a dependency
+Routes==2.5.1
+typer==0.15.2
+
+# indirect dependencies
 aiohappyeyeballs==2.6.1
     # via aiohttp
 aiohttp==3.11.16
@@ -54,6 +71,8 @@ cryptography==43.0.3
     # via
     #   -r requirements.in
     #   pyopenssl
+distro==1.9.0
+    # via -r requirements.in
 docutils==0.16
     # via awscli
 fasteners==0.19

cmapi/updater/cmapi_updater.service.template

@@ -0,0 +1,9 @@
+[Unit]
+Description=One-shot service to run CMAPI self-updater.
+After=network.target
+
+[Service]
+Type=oneshot
+EnvironmentFile=/tmp/cmapi_updater.conf
+User=${CMAPI_USER}
+ExecStart=${BIN_DIR}/cmapi_updater.sh

cmapi/updater/cmapi_updater.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+VERSION="${CMAPI_VERSION}"
+if [[ -z "$VERSION" ]]; then
+    echo "[Updater] Error: CMAPI_VERSION is not set"
+    exit 1
+fi
+
+# Wait for a few seconds to allow endpoint to respond
+sleep 5s
+echo "[CMAPI Updater] Stopping CMAPI service..."
+systemctl stop mariadb-columnstore-cmapi
+
+echo "[CMAPI Updater] Removing existing package..."
+if command -v apt >/dev/null; then
+    apt remove -y mariadb-columnstore-cmapi
+    apt install -y mariadb-columnstore-cmapi=${VERSION}
+elif command -v yum >/dev/null; then
+    yum remove -y MariaDB-columnstore-cmapi
+    yum install -y MariaDB-columnstore-cmapi-${VERSION}
+else
+    echo "Unsupported package manager"
+    exit 1
+fi
+
+echo "[CMAPI Updater] Restarting CMAPI service..."
+systemctl start mariadb-columnstore-cmapi
+
+echo "[CMAPI Updater] Done."
+exit 0