You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-11-03 17:13:17 +03:00
a76e153a1d
Implements the initial upgrade capability across CMAPI and the CLI, including
repository setup, package operations, environment prechecks, and coordinated
cluster steps with progress reporting.
Details:
- CMAPI upgrade manager:
- Add `cmapi/cmapi_server/managers/upgrade/` modules:
- `repo.py`, `packages.py`, `preinstall.py`, `upgrade.py`, `utils.py` and `__init__.py`
- Extend endpoints and routing to expose upgrade operations and status:
- `cmapi_server/controllers/{endpoints.py, dispatcher.py, api_clients.py}`
- `cmapi_server/managers/{application.py, process.py}`
- Add improved constants and helpers for upgrade flow
- Backup/restore and safety:
- Add `cmapi_server/managers/backup_restore.py`
- Fix pre-upgrade backup regressions (due to `mcs_backup_manager.sh 3.17 changes`)
- Improve cluster version validation; add `ignore_missmatch` override
- CLI enhancements:
- Progress UI and richer feedback (`mcs_cluster_tool/tools_commands.py`, `README.md`, `mcs.1`)
- Add steps to start MDB and start MCS during/after upgrade
- Improved error surfacing for version validation
- Platform and packaging:
- Ubuntu and Rocky Linux support
- RHEL/DNF dry-run support
- Distro detection and platform-dependent logic hardened
- Logging improvements
- Updater service:
- Add `cmapi/updater/cmapi_updater.service.template` and `cmapi_updater.sh` to make CMAPI update itself
- Docs:
- Update mcs cli README and mcs.1 man file
- Add `cmapi/updater/README.md`
179 lines
6.5 KiB
Python
179 lines
6.5 KiB
Python
"""Module contains constants values for cmapi, failover and other .py files.
|
|
|
|
TODO: move main constant paths here and replace in files in next releases.
|
|
"""
|
|
import os
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
from typing import NamedTuple
|
|
|
|
# default MARIADB ColumnStore config path
|
|
MCS_ETC_PATH = '/etc/columnstore'
|
|
DEFAULT_MCS_CONF_PATH = os.path.join(MCS_ETC_PATH, 'Columnstore.xml')
|
|
|
|
# default Storage Manager config path
|
|
DEFAULT_SM_CONF_PATH = os.path.join(MCS_ETC_PATH, 'storagemanager.cnf')
|
|
|
|
# MCSDATADIR (in mcs engine code) and related paths
|
|
MCS_DATA_PATH = '/var/lib/columnstore'
|
|
MCS_MODULE_FILE_PATH = os.path.join(MCS_DATA_PATH, 'local/module')
|
|
EM_PATH_SUFFIX = 'data1/systemFiles/dbrm'
|
|
MCS_EM_PATH = os.path.join(MCS_DATA_PATH, EM_PATH_SUFFIX)
|
|
MCS_BRM_CURRENT_PATH = os.path.join(MCS_EM_PATH, 'BRM_saves_current')
|
|
S3_BRM_CURRENT_PATH = os.path.join(EM_PATH_SUFFIX, 'BRM_saves_current')
|
|
|
|
# keys file for CEJ password encryption\decryption
|
|
# (CrossEngineSupport section in Columnstore.xml)
|
|
MCS_SECRETS_FILENAME = '.secrets'
|
|
MCS_SECRETS_FILE_PATH = os.path.join(MCS_DATA_PATH, MCS_SECRETS_FILENAME)
|
|
|
|
# CMAPI SERVER
|
|
CMAPI_CONFIG_FILENAME = 'cmapi_server.conf'
|
|
CMAPI_ROOT_PATH = os.path.dirname(__file__)
|
|
PROJECT_PATH = os.path.dirname(CMAPI_ROOT_PATH)
|
|
# path to VERSION file
|
|
VERSION_PATH = os.path.join(PROJECT_PATH, 'VERSION')
|
|
CMAPI_LOG_CONF_PATH = os.path.join(CMAPI_ROOT_PATH, 'cmapi_logger.conf')
|
|
# path to CMAPI default config
|
|
CMAPI_DEFAULT_CONF_PATH = os.path.join(CMAPI_ROOT_PATH, CMAPI_CONFIG_FILENAME)
|
|
# CMAPI config path
|
|
CMAPI_CONF_PATH = os.path.join(MCS_ETC_PATH, CMAPI_CONFIG_FILENAME)
|
|
|
|
# TOTP secret key
|
|
SECRET_KEY = 'MCSIsTheBestEver' # not just a random string! (base32)
|
|
|
|
|
|
# network constants
|
|
# according to https://www.ibm.com/docs/en/storage-sentinel/1.1.2?topic=installation-map-your-local-host-loopback-address
|
|
|
|
LOCALHOST_IPS = {
|
|
'127.0.0.1',
|
|
'::1',
|
|
}
|
|
LOCALHOST_HOSTNAMES = {
|
|
'localhost', 'localhost.localdomain',
|
|
'localhost4', 'localhost4.localdomain4',
|
|
'localhost6', 'localhost6.localdomain6',
|
|
}
|
|
LOCALHOSTS = LOCALHOST_IPS.union(LOCALHOST_HOSTNAMES)
|
|
|
|
CMAPI_INSTALL_PATH = '/usr/share/columnstore/cmapi/'
|
|
CMAPI_PYTHON_BIN = os.path.join(CMAPI_INSTALL_PATH, "python/bin/python3")
|
|
CMAPI_PYTHON_DEPS_PATH = os.path.join(CMAPI_INSTALL_PATH, "deps")
|
|
CMAPI_PYTHON_BINARY_DEPS_PATH = os.path.join(CMAPI_PYTHON_DEPS_PATH, "bin")
|
|
CMAPI_SINGLE_NODE_XML = os.path.join(
|
|
CMAPI_INSTALL_PATH, 'cmapi_server/SingleNode.xml'
|
|
)
|
|
|
|
class MCSProgs(str, Enum):
|
|
STORAGE_MANAGER = 'StorageManager'
|
|
WORKER_NODE = 'workernode'
|
|
CONTROLLER_NODE = 'controllernode'
|
|
PRIM_PROC = 'PrimProc'
|
|
WRITE_ENGINE_SERVER = 'WriteEngineServer'
|
|
DML_PROC = 'DMLProc'
|
|
DDL_PROC = 'DDLProc'
|
|
|
|
# constants for dispatchers
|
|
class ProgInfo(NamedTuple):
|
|
"""NamedTuple for some additional info about handling mcs processes."""
|
|
stop_priority: int # priority for building stop sequence
|
|
service_name: str # systemd service name
|
|
subcommand: str # subcommand for process run in docker container
|
|
only_primary: bool # use this process only on primary
|
|
delay: int = 0 # delay after process start in docker container
|
|
|
|
# mcs-loadbrm and mcs-savebrm are dependencies for workernode and resolved
|
|
# on top level of process handling
|
|
# mcs-storagemanager starts conditionally inside mcs-loadbrm, but should be
|
|
# stopped using cmapi
|
|
ALL_MCS_PROGS: dict[MCSProgs, ProgInfo] = {
|
|
# workernode starts on primary and non primary node with 1 or 2 added
|
|
# to subcommand (DBRM_Worker1 - on primary, DBRM_Worker2 - non primary)
|
|
MCSProgs.STORAGE_MANAGER: ProgInfo(15, 'mcs-storagemanager', '', False, 1),
|
|
MCSProgs.WORKER_NODE: ProgInfo(13, 'mcs-workernode', 'DBRM_Worker{}', False, 1),
|
|
MCSProgs.CONTROLLER_NODE: ProgInfo(11, 'mcs-controllernode', 'fg', True),
|
|
MCSProgs.PRIM_PROC: ProgInfo(5, 'mcs-primproc', '', False, 1),
|
|
MCSProgs.WRITE_ENGINE_SERVER: ProgInfo(7, 'mcs-writeengineserver', '', False, 3),
|
|
MCSProgs.DML_PROC: ProgInfo(3, 'mcs-dmlproc', '', False),
|
|
MCSProgs.DDL_PROC: ProgInfo(1, 'mcs-ddlproc', '', False),
|
|
}
|
|
|
|
# constants for docker container dispatcher
|
|
MCS_INSTALL_BIN = '/usr/bin'
|
|
IFLAG = os.path.join(MCS_ETC_PATH, 'container-initialized')
|
|
LIBJEMALLOC_DEFAULT_PATH = os.path.join(MCS_DATA_PATH, 'libjemalloc.so.2')
|
|
MCS_LOG_PATH = '/var/log/mariadb/columnstore'
|
|
|
|
# BRM shmem lock inspection/reset tool
|
|
SHMEM_LOCKS_PATH = os.path.join(MCS_INSTALL_BIN, 'mcs-shmem-locks')
|
|
# mcs and cmapi constanst shared
|
|
MCS_BACKUP_MANAGER_SH = os.path.join(MCS_INSTALL_BIN, 'mcs_backup_manager.sh')
|
|
|
|
# client constants
|
|
CMAPI_PORT = 8640 #TODO: use it in all places
|
|
CURRENT_NODE_CMAPI_URL = f'https://localhost:{CMAPI_PORT}'
|
|
REQUEST_TIMEOUT: float = 30.0
|
|
TRANSACTION_TIMEOUT: float = 300.0 # 5 minutes
|
|
|
|
# API version
|
|
_version = '0.4.0'
|
|
|
|
# constants for packages and repositories
|
|
SUPPORTED_DISTROS = (
|
|
'ubuntu',
|
|
'debian',
|
|
'centos',
|
|
'rhel',
|
|
'rocky',
|
|
)
|
|
SUPPORTED_ARCHITECTURES = ('x86_64', 'amd64', 'aarch64', 'arm64')
|
|
|
|
@dataclass(frozen=True)
|
|
class MultiDistroNamer:
|
|
rhel: str
|
|
deb: str
|
|
|
|
MDB_SERVER_PACKAGE_NAME = MultiDistroNamer(
|
|
rhel='MariaDB-server',
|
|
deb='mariadb-server'
|
|
)
|
|
MDB_CS_PACKAGE_NAME = MultiDistroNamer(
|
|
rhel='MariaDB-columnstore-engine',
|
|
deb='mariadb-plugin-columnstore'
|
|
)
|
|
CMAPI_PACKAGE_NAME = MultiDistroNamer(
|
|
rhel='MariaDB-columnstore-cmapi',
|
|
deb='mariadb-columnstore-cmapi'
|
|
)
|
|
ES_REPO = MultiDistroNamer(
|
|
rhel=(
|
|
'''[mariadb-es-main]
|
|
name = MariaDB Enterprise Server
|
|
baseurl = https://dlm.mariadb.com/repo/{token}/mariadb-enterprise-server/{mdb_version}/rpm/rhel/{os_major_version}/{arch}
|
|
gpgkey = {gpg_key_url}
|
|
gpgcheck = 1
|
|
enabled = 1
|
|
module_hotfixes = 1
|
|
'''
|
|
),
|
|
deb='deb [arch=amd64,arm64] https://dlm.mariadb.com/repo/{token}/mariadb-enterprise-server/{mdb_version}/deb {os_version} main'
|
|
)
|
|
ES_REPO_PRIORITY_PREFS = '''
|
|
Package: *
|
|
Pin: origin dlm.mariadb.com
|
|
Pin-Priority: 1700
|
|
'''
|
|
ES_VERIFY_URL = MultiDistroNamer(
|
|
rhel='https://dlm.mariadb.com/repo/{token}/mariadb-enterprise-server/{mdb_version}/rpm/rhel/{os_major_version}/{arch}/repodata/repomd.xml',
|
|
deb='https://dlm.mariadb.com/repo/{token}/mariadb-enterprise-server/{mdb_version}/deb/dists/{os_version}/Release'
|
|
)
|
|
MDB_GPG_KEY_URL = 'https://supplychain.mariadb.com/MariaDB-Enterprise-GPG-KEY'
|
|
ES_TOKEN_VERIFY_URL = 'https://dlm.mariadb.com/browse/{token}/mariadb_enterprise_server/'
|
|
MDB_LATEST_TESTED_MAJOR = '10.6'
|
|
MDB_LATEST_RELEASES_URL = 'https://dlm.mariadb.com/rest/releases/mariadb_enterprise_server/'
|
|
PKG_GET_VER_CMD = MultiDistroNamer(
|
|
rhel="rpm -q --queryformat '%{{VERSION}}' {package_name}",
|
|
deb="dpkg-query -f '${{Version}}' -W {package_name}"
|
|
)
|