You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-30 19:23:07 +03:00
MCOL-5618-related: Duplicate to stable 23.10 (#3223)
* feat(package_manager): improved cs_package_manager functionality for better UX - add apt support for community and enterprise install - add enterprise_staging install - filter check results down to os/arch - add --token flag for enterprise (#3002) * Update cs package manager dev ubuntu (#3060) * cs_package_manager.sh - fix ubuntu dev installs * cs_package_manager.sh - remove some useless comments * debian/ubuntu dev install to use drone repo + remove wget dependancy * + exits if repo files missing * auto install aws cli for dev builds * removed commented section from cs_package_manager * feat(tooling): add rhel support in cs_package_manager.sh (#3105) * added dbrm_restore for LocalStorage & S3 * small comment cleanup * resolved PR comments for cs_package_manager.sh & mcs_backup_manager.sh * added dbrm_restore for LocalStorage & S3 * Added backup retention flag -r and incremental auto_most_recent option to mcs_backup_manager.sh * resolved PR comments for cs_package_manager.sh & mcs_backup_manager.sh * add comment to auto_select_most_recent_backup_for_incremental in mcs_backup_manager.sh * feat(cmapi,cli): MCOL-5618: Add backup, restore, backup-dbrm, restore-dbrm commands for mcs cli tool. (#3130) * MCOL-5618: Add backup, restore, backup-dbrm, restore-dbrm commands for mcs cli tool. [add] mcs_cluster_tool/helpers.py file with cook_sh_arg function inside [add] MCS_BACKUP_MANAGER_SH to mcs_cluster_tool/constants.py [add] backup and restore commands to "mcs" Typer APP * MCOL-5618: Move mcs_backup_manager.sh to cmapi/scripts. * MCOL-5618: Install mcs_backup_manager.sh with CMAPI package. * MCOL-5618: Testing fix + script minor fixes. - [fix] wrong cooking of .sh invoke string - [fix] restore_dbrm -> dbrm_restore, backup_dbrm -> dbrm_backup to use same naming in both python wrapper and mcs_backup_manager.sh - [fix] mathod of checking cmapi online or not in mcs_backup_manager.sh * MCOL-5618: Minor improvements to sync with PR 3144. [add] -ns option for dbrm_restore. * MCOL-5618: Minor fixes to sync after #3147 [add] -r option for backup command * MCOL-5618: Minor fixes to sync after PR #1398. [add] -ssm argument for dbrm backup [add] -sdbk -ssm arguments for dbrm restore [add] silenece annoying pylint warning unused argument for bachup_commands.py and restore_commnads.py * update to the latest mcs_backup_manager + cs_package_manager * Intermediate commit to move changes from old filepath to the new one. * remove old file * MCOL-5618: Fix packaging. [mv] extra/cs_package_manager.sh -> cmapi/scripts/cs_package_manager.sh to pack and deliver the script with cmapi package [add] cs_package_manager.sh to cmapi package [fix] download paths for mcs_backup_manager.sh and cs_package_manager.sh * MCOL-5618: Fix after testing [add] -P to restore help in mcs_backup_restore.sh [fix] -f argument default value in both backup and restore wrapper commands [fix] -i argument behaviour and type + default value [fix] -P argument position and help for both backup and restore wrapper commands [fix] disable some pylint warning for both backup and restore wrapper files --------- Co-authored-by: Allen Herrera <82840027+mariadb-AllenHerrera@users.noreply.github.com> Co-authored-by: Allen Herrera <allen.herrera@mariadb.com> Co-authored-by: Allen Herrera <allen@Allens-Black-MacBook-Pro.local> Co-authored-by: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com>
This commit is contained in:
Alan Mologorsky
GitHub
@ -84,6 +84,9 @@ INSTALL(FILES mcs_aws
|
||||
INSTALL(FILES mcs_gsutil
|
||||
PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
|
||||
DESTINATION ${BIN_DIR})
|
||||
INSTALL(FILES scripts/mcs_backup_manager.sh scripts/cs_package_manager.sh
|
||||
PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
|
||||
DESTINATION ${BIN_DIR})
|
||||
|
||||
OPTION(RPM "Build an RPM" OFF)
|
||||
IF(RPM)
|
||||
|
||||
@ -4,7 +4,9 @@ import sys
|
||||
import typer
|
||||
|
||||
from cmapi_server.logging_management import dict_config, add_logging_level
|
||||
from mcs_cluster_tool import cluster_app, cmapi_app
|
||||
from mcs_cluster_tool import (
|
||||
cluster_app, cmapi_app, backup_commands, restore_commands
|
||||
)
|
||||
from mcs_cluster_tool.constants import MCS_CLI_LOG_CONF_PATH
|
||||
|
||||
|
||||
@ -16,11 +18,15 @@ app = typer.Typer(
|
||||
'MCS services'
|
||||
),
|
||||
)
|
||||
app.add_typer(cluster_app.app, name="cluster")
|
||||
app.add_typer(cmapi_app.app, name="cmapi")
|
||||
app.add_typer(cluster_app.app, name='cluster')
|
||||
app.add_typer(cmapi_app.app, name='cmapi')
|
||||
app.command()(backup_commands.backup)
|
||||
app.command('backup-dbrm')(backup_commands.dbrm_backup)
|
||||
app.command()(restore_commands.restore)
|
||||
app.command('restore-dbrm')(restore_commands.dbrm_restore)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if __name__ == '__main__':
|
||||
add_logging_level('TRACE', 5) #TODO: remove when stadalone mode added.
|
||||
dict_config(MCS_CLI_LOG_CONF_PATH)
|
||||
logger = logging.getLogger('mcs_cli')
|
||||
|
||||
353
cmapi/mcs_cluster_tool/backup_commands.py
Normal file
353
cmapi/mcs_cluster_tool/backup_commands.py
Normal file
@ -0,0 +1,353 @@
|
||||
"""Typer application for backup Columnstore data."""
|
||||
import logging
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing_extensions import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from cmapi_server.process_dispatchers.base import BaseDispatcher
|
||||
from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH
|
||||
from mcs_cluster_tool.decorators import handle_output
|
||||
from mcs_cluster_tool.helpers import cook_sh_arg
|
||||
|
||||
|
||||
logger = logging.getLogger('mcs_cli')
|
||||
# pylint: disable=unused-argument, too-many-arguments, too-many-locals
|
||||
# pylint: disable=invalid-name, line-too-long
|
||||
|
||||
|
||||
@handle_output
|
||||
def backup(
|
||||
bl: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bl', '--backup-location',
|
||||
help=(
|
||||
'What directory to store the backups on this machine or the target machine.\n'
|
||||
'Consider write permissions of the scp user and the user running this script.\n'
|
||||
'Mariadb-backup will use this location as a tmp dir for S3 and remote backups temporarily.\n'
|
||||
'Example: /mnt/backups/'
|
||||
)
|
||||
)
|
||||
] = '/tmp/backups/',
|
||||
bd: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bd', '--backup-destination',
|
||||
help=(
|
||||
'Are the backups going to be stored on the same machine this '
|
||||
'script is running on or another server - if Remote you need '
|
||||
'to setup scp='
|
||||
'Options: "Local" or "Remote"'
|
||||
)
|
||||
)
|
||||
] = 'Local',
|
||||
scp: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-scp',
|
||||
help=(
|
||||
'Used only if --backup-destination="Remote".\n'
|
||||
'The user/credentials that will be used to scp the backup '
|
||||
'files\n'
|
||||
'Example: "centos@10.14.51.62"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
bb: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bb', '--backup-bucket',
|
||||
help=(
|
||||
'Only used if --storage=S3\n'
|
||||
'Name of the bucket to store the columnstore backups.\n'
|
||||
'Example: "s3://my-cs-backups"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
url: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-url', '--endpoint-url',
|
||||
help=(
|
||||
'Used by on premise S3 vendors.\n'
|
||||
'Example: "http://127.0.0.1:8000"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
nv_ssl: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl',
|
||||
help='Skips verifying ssl certs, useful for onpremise s3 storage.'
|
||||
)
|
||||
] = False,
|
||||
s: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-s', '--storage',
|
||||
help=(
|
||||
'What storage topogoly is being used by Columnstore - found '
|
||||
'in /etc/columnstore/storagemanager.cnf.\n'
|
||||
'Options: "LocalStorage" or "S3"'
|
||||
)
|
||||
)
|
||||
] = 'LocalStorage',
|
||||
i: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-i', '--incremental',
|
||||
help=(
|
||||
'Adds columnstore deltas to an existing full backup. '
|
||||
'Backup folder to apply increment could be a value or '
|
||||
'"auto_most_recent" - the incremental backup applies to '
|
||||
'last full backup.'
|
||||
),
|
||||
show_default=False
|
||||
)
|
||||
] = '',
|
||||
ha: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-ha/-no-ha', '--highavilability/--no-highavilability',
|
||||
help=(
|
||||
'Hint wether shared storage is attached @ below on all nodes '
|
||||
'to see all data\n'
|
||||
' HA LocalStorage ( /var/lib/columnstore/dataX/ )\n'
|
||||
' HA S3 ( /var/lib/columnstore/storagemanager/ )'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
f: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-f', '--config-file',
|
||||
help='Path to backup configuration file to load variables from.',
|
||||
show_default=False
|
||||
)
|
||||
] = '',
|
||||
sbrm: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-sbrm/-no-sbrm', '--skip-save-brm/--no-skip-save-brm',
|
||||
help=(
|
||||
'Skip saving brm prior to running a backup - '
|
||||
'ideal for dirty backups.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
spoll: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-spoll/-no-spoll', '--skip-polls/--no-skip-polls',
|
||||
help='Skip sql checks confirming no write/cpimports running.'
|
||||
)
|
||||
] = False,
|
||||
slock: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-slock/-no-slock', '--skip-locks/--no-skip-locks',
|
||||
help='Skip issuing write locks - ideal for dirty backups.'
|
||||
)
|
||||
] = False,
|
||||
smdb: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-smdb/-no-smdb', '--skip-mariadb-backup/--no-skip-mariadb-backup',
|
||||
help=(
|
||||
'Skip running a mariadb-backup for innodb data - ideal for '
|
||||
'incremental dirty backups.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
sb: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-sb/-no-sb', '--skip-bucket-data/--no-skip-bucket-data',
|
||||
help='Skip taking a copy of the columnstore data in the bucket.'
|
||||
)
|
||||
] = False,
|
||||
pi: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-pi', '--poll-interval',
|
||||
help=(
|
||||
'Number of seconds between poll checks for active writes & '
|
||||
'cpimports.'
|
||||
)
|
||||
)
|
||||
] = 5,
|
||||
pmw: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-pmw', '--poll-max-wait',
|
||||
help=(
|
||||
'Max number of minutes for polling checks for writes to wait '
|
||||
'before exiting as a failed backup attempt.'
|
||||
)
|
||||
)
|
||||
] = 60,
|
||||
q: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-q/-no-q', '--quiet/--no-quiet',
|
||||
help='Silence verbose copy command outputs.'
|
||||
)
|
||||
] = False,
|
||||
c: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-c', '--compress',
|
||||
help='Compress backup in X format - Options: [ pigz ].',
|
||||
show_default=False
|
||||
)
|
||||
] = '',
|
||||
P: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-P', '--parallel',
|
||||
help=(
|
||||
'Determines if columnstore data directories will have '
|
||||
'multiple rsync running at the same time for different '
|
||||
'subfolders to parallelize writes. '
|
||||
'Ignored if "-c/--compress" argument not set.'
|
||||
)
|
||||
)
|
||||
] = 4,
|
||||
nb: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-nb', '--name-backup',
|
||||
help='Define the name of the backup - default: $(date +%m-%d-%Y)'
|
||||
)
|
||||
] = datetime.now().strftime('%m-%d-%Y'),
|
||||
m: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-m', '--mode',
|
||||
help=(
|
||||
'Modes ["direct","indirect"] - direct backups run on the '
|
||||
'columnstore nodes themselves. indirect run on another '
|
||||
'machine that has read-only mounts associated with '
|
||||
'columnstore/mariadb\n'
|
||||
),
|
||||
hidden=True
|
||||
)
|
||||
] = 'direct',
|
||||
r: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-r', '--retention-days',
|
||||
help=(
|
||||
'Retain backups created within the last X days, '
|
||||
'default 0 == keep all backups.'
|
||||
)
|
||||
)
|
||||
] = 0,
|
||||
):
|
||||
"""Backup Columnstore and/or MariDB data."""
|
||||
|
||||
# Local Storage Examples:
|
||||
# ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage
|
||||
# ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage -P 8
|
||||
# ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage --incremental 02-18-2022
|
||||
# ./$0 backup -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -s LocalStorage
|
||||
|
||||
# S3 Examples:
|
||||
# ./$0 backup -bb s3://my-cs-backups -s S3
|
||||
# ./$0 backup -bb s3://my-cs-backups -c pigz --quiet -sb
|
||||
# ./$0 backup -bb gs://my-cs-backups -s S3 --incremental 02-18-2022
|
||||
# ./$0 backup -bb s3://my-onpremise-bucket -s S3 -url http://127.0.0.1:8000
|
||||
|
||||
# Cron Example:
|
||||
# */60 */24 * * * root bash /root/$0 -bb s3://my-cs-backups -s S3 >> /root/csBackup.log 2>&1
|
||||
|
||||
arguments = []
|
||||
for arg_name, value in locals().items():
|
||||
sh_arg = cook_sh_arg(arg_name, value)
|
||||
if sh_arg is None:
|
||||
continue
|
||||
arguments.append(sh_arg)
|
||||
cmd = f'{MCS_BACKUP_MANAGER_SH} backup {" ".join(arguments)}'
|
||||
success, _ = BaseDispatcher.exec_command(cmd, stdout=sys.stdout)
|
||||
return {'success': success}
|
||||
|
||||
|
||||
@handle_output
|
||||
def dbrm_backup(
|
||||
m: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-m', '--mode',
|
||||
help=(
|
||||
'"loop" or "once" ; Determines if this script runs in a '
|
||||
'forever loop sleeping -i minutes or just once.'
|
||||
),
|
||||
)
|
||||
] = 'once',
|
||||
i: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-i', '--interval',
|
||||
help='Number of minutes to sleep when --mode=loop.'
|
||||
)
|
||||
] = 90,
|
||||
r: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-r', '--retention-days',
|
||||
help=(
|
||||
'Retain dbrm backups created within the last X days, '
|
||||
'the rest are deleted'
|
||||
)
|
||||
)
|
||||
] = 7,
|
||||
p: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-p', '--path',
|
||||
help='Path of where to save the dbrm backups on disk.'
|
||||
)
|
||||
] = '/tmp/dbrm_backups',
|
||||
nb: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-nb', '--name-backup',
|
||||
help='Custom name to prefex dbrm backups with.'
|
||||
)
|
||||
] = 'dbrm_backup',
|
||||
q: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-q/-no-q', '--quiet/--no-quiet',
|
||||
help='Silence verbose copy command outputs.'
|
||||
)
|
||||
] = False,
|
||||
ssm: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager',
|
||||
help='Skip backing up storagemanager directory.'
|
||||
)
|
||||
] = False,
|
||||
):
|
||||
"""Columnstore DBRM Backup."""
|
||||
|
||||
# Default: ./$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups
|
||||
|
||||
# Examples:
|
||||
# ./$0 dbrm_backup --mode loop --interval 90 --retention-days 7 --path /mnt/dbrm_backups
|
||||
# ./$0 dbrm_backup --mode once --retention-days 7 --path /mnt/dbrm_backups -nb my-one-off-backup
|
||||
|
||||
# Cron Example:
|
||||
# */60 */3 * * * root bash /root/$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups >> /tmp/dbrm_backups/cs_backup.log 2>&1
|
||||
arguments = []
|
||||
for arg_name, value in locals().items():
|
||||
sh_arg = cook_sh_arg(arg_name, value)
|
||||
if sh_arg is None:
|
||||
continue
|
||||
arguments.append(sh_arg)
|
||||
cmd = f'{MCS_BACKUP_MANAGER_SH} dbrm_backup {" ".join(arguments)}'
|
||||
success, _ = BaseDispatcher.exec_command(cmd, stdout=sys.stdout)
|
||||
return {'success': success}
|
||||
@ -1,4 +1,9 @@
|
||||
import os
|
||||
|
||||
from cmapi_server.constants import MCS_INSTALL_BIN
|
||||
|
||||
|
||||
MCS_CLI_ROOT_PATH = os.path.dirname(__file__)
|
||||
MCS_CLI_LOG_CONF_PATH = os.path.join(MCS_CLI_ROOT_PATH, 'mcs_cli_log.conf')
|
||||
|
||||
MCS_BACKUP_MANAGER_SH = os.path.join(MCS_INSTALL_BIN, 'mcs_backup_manager.sh')
|
||||
|
||||
29
cmapi/mcs_cluster_tool/helpers.py
Normal file
29
cmapi/mcs_cluster_tool/helpers.py
Normal file
@ -0,0 +1,29 @@
|
||||
"""Module with helper functions for mcs cli tool."""
|
||||
from typing import Union
|
||||
|
||||
|
||||
def cook_sh_arg(arg_name: str, value:Union[str, int, bool]) -> str:
|
||||
"""Convert argument and and value from function locals to bash argument.
|
||||
|
||||
:param arg_name: function argument name
|
||||
:type arg_name: str
|
||||
:param value: function argument value
|
||||
:type value: Union[str, int, bool]
|
||||
:return: bash argument string
|
||||
:rtype: str
|
||||
"""
|
||||
# skip "arguments" list and Typer ctx variables from local scope
|
||||
if arg_name in ('arguments', 'ctx'):
|
||||
return None
|
||||
# skip args that have empty string as value
|
||||
if value == '':
|
||||
return None
|
||||
if '_' in arg_name:
|
||||
arg_name = arg_name.replace('_', '-')
|
||||
# skip boolean args that have False value
|
||||
if isinstance(value, bool):
|
||||
if not value:
|
||||
return None
|
||||
# if True value presented just pass only arg name without value
|
||||
value = ''
|
||||
return f'-{arg_name} {value}' if value else f'-{arg_name}'
|
||||
324
cmapi/mcs_cluster_tool/restore_commands.py
Normal file
324
cmapi/mcs_cluster_tool/restore_commands.py
Normal file
@ -0,0 +1,324 @@
|
||||
"""Typer application for restore Columnstore data."""
|
||||
import logging
|
||||
import sys
|
||||
from typing_extensions import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from cmapi_server.process_dispatchers.base import BaseDispatcher
|
||||
from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH
|
||||
from mcs_cluster_tool.decorators import handle_output
|
||||
from mcs_cluster_tool.helpers import cook_sh_arg
|
||||
|
||||
|
||||
logger = logging.getLogger('mcs_cli')
|
||||
# pylint: disable=unused-argument, too-many-arguments, too-many-locals
|
||||
# pylint: disable=invalid-name, line-too-long
|
||||
|
||||
|
||||
@handle_output
|
||||
def restore(
|
||||
l: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-l', '--load',
|
||||
help='What date folder to load from the backup_location.'
|
||||
)
|
||||
] = '',
|
||||
bl: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bl', '--backup-location',
|
||||
help=(
|
||||
'Where the backup to load is found.\n'
|
||||
'Example: /mnt/backups/'
|
||||
)
|
||||
)
|
||||
] = '/tmp/backups/',
|
||||
bd: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bd', '--backup_destination',
|
||||
help=(
|
||||
'Is this backup on the same or remote server compared to '
|
||||
'where this script is running.\n'
|
||||
'Options: "Local" or "Remote"'
|
||||
)
|
||||
)
|
||||
] = 'Local',
|
||||
scp: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-scp', '--secure-copy-protocol',
|
||||
help=(
|
||||
'Used only if --backup-destination=Remote'
|
||||
'The user/credentials that will be used to scp the backup files.'
|
||||
'Example: "centos@10.14.51.62"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
bb: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-bb', '--backup-bucket',
|
||||
help=(
|
||||
'Only used if --storage=S3\n'
|
||||
'Name of the bucket to store the columnstore backups.\n'
|
||||
'Example: "s3://my-cs-backups"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
url: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-url', '--endpoint-url',
|
||||
help=(
|
||||
'Used by on premise S3 vendors.\n'
|
||||
'Example: "http://127.0.0.1:8000"'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
s: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-s', '--storage',
|
||||
help=(
|
||||
'What storage topogoly is being used by Columnstore - found '
|
||||
'in /etc/columnstore/storagemanager.cnf.\n'
|
||||
'Options: "LocalStorage" or "S3"'
|
||||
)
|
||||
)
|
||||
] = 'LocalStorage',
|
||||
dbs: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-dbs', '--dbroots',
|
||||
help='Number of database roots in the backup.'
|
||||
)
|
||||
] = 1,
|
||||
pm: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-pm', '--nodeid',
|
||||
help=(
|
||||
'Forces the handling of the restore as this node as opposed '
|
||||
'to whats detected on disk.'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
nb: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-nb', '--new-bucket',
|
||||
help=(
|
||||
'Defines the new bucket to copy the s3 data to from the '
|
||||
'backup bucket. Use -nb if the new restored cluster should '
|
||||
'use a different bucket than the backup bucket itself.'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
nr: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-nr', '--new-region',
|
||||
help=(
|
||||
'Defines the region of the new bucket to copy the s3 data to '
|
||||
'from the backup bucket.'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
nk: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-nk', '--new-key',
|
||||
help='Defines the aws key to connect to the new_bucket.'
|
||||
)
|
||||
] = '',
|
||||
ns: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-ns', '--new-secret',
|
||||
help=(
|
||||
'Defines the aws secret of the aws key to connect to the '
|
||||
'new_bucket.'
|
||||
)
|
||||
)
|
||||
] = '',
|
||||
ha: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-ha/-no-ha', '--highavilability/--no-highavilability',
|
||||
help=(
|
||||
'Flag for high available systems (meaning shared storage '
|
||||
'exists supporting the topology so that each node sees '
|
||||
'all data)'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
cont: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-cont/-no-cont', '--continue/--no-continue',
|
||||
help=(
|
||||
'This acknowledges data in your --new_bucket is ok to delete '
|
||||
'when restoring S3. When set to true skips the enforcement '
|
||||
'that new_bucket should be empty prior to starting a restore.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
f: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-f', '--config-file',
|
||||
help='Path to backup configuration file to load variables from.',
|
||||
show_default=False
|
||||
)
|
||||
] = '',
|
||||
smdb: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-smdb/-no-smdb', '--skip-mariadb-backup/--no-skip-mariadb-backup',
|
||||
help=(
|
||||
'Skip restoring mariadb server via mariadb-backup - ideal for '
|
||||
'only restoring columnstore.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
sb: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-sb/-no-sb', '--skip-bucket-data/--no-skip-bucket-data',
|
||||
help=(
|
||||
'Skip restoring columnstore data in the bucket - ideal if '
|
||||
'looking to only restore mariadb server.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
m: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-m', '--mode',
|
||||
help=(
|
||||
'Modes ["direct","indirect"] - direct backups run on the '
|
||||
'columnstore nodes themselves. indirect run on another '
|
||||
'machine that has read-only mounts associated with '
|
||||
'columnstore/mariadb\n'
|
||||
),
|
||||
hidden=True
|
||||
)
|
||||
] = 'direct',
|
||||
c: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-c', '--compress',
|
||||
help=(
|
||||
'Hint that the backup is compressed in X format. '
|
||||
'Options: [ pigz ].'
|
||||
),
|
||||
show_default=False
|
||||
)
|
||||
] = '',
|
||||
P: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
'-P', '--parallel',
|
||||
help=(
|
||||
'Determines number of decompression and mdbstream threads. '
|
||||
'Ignored if "-c/--compress" argument not set.'
|
||||
)
|
||||
)
|
||||
] = 4,
|
||||
q: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-q/-no-q', '--quiet/--no-quiet',
|
||||
help='Silence verbose copy command outputs.'
|
||||
)
|
||||
] = False,
|
||||
nv_ssl: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl',
|
||||
help='Skips verifying ssl certs, useful for onpremise s3 storage.'
|
||||
)
|
||||
] = False,
|
||||
):
|
||||
"""Restore Columnstore (and/or MariaDB) data."""
|
||||
|
||||
# Local Storage Examples:
|
||||
# ./$0 restore -s LocalStorage -bl /tmp/backups/ -bd Local -l 12-29-2021
|
||||
# ./$0 restore -s LocalStorage -bl /tmp/backups/ -bd Remote -scp root@172.31.6.163 -l 12-29-2021
|
||||
|
||||
# S3 Storage Examples:
|
||||
# ./$0 restore -s S3 -bb s3://my-cs-backups -l 12-29-2021
|
||||
# ./$0 restore -s S3 -bb gs://on-premise-bucket -l 12-29-2021 -url http://127.0.0.1:8000
|
||||
# ./$0 restore -s S3 -bb s3://my-cs-backups -l 08-16-2022 -nb s3://new-data-bucket -nr us-east-1 -nk AKIAxxxxxxx3FHCADF -ns GGGuxxxxxxxxxxnqa72csk5 -ha
|
||||
arguments = []
|
||||
for arg_name, value in locals().items():
|
||||
sh_arg = cook_sh_arg(arg_name, value)
|
||||
if sh_arg is None:
|
||||
continue
|
||||
arguments.append(sh_arg)
|
||||
cmd = f'{MCS_BACKUP_MANAGER_SH} restore {" ".join(arguments)}'
|
||||
success, _ = BaseDispatcher.exec_command(cmd, stdout=sys.stdout)
|
||||
return {'success': success}
|
||||
|
||||
|
||||
@handle_output
|
||||
def dbrm_restore(
|
||||
p: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-p', '--path',
|
||||
help='Path of where dbrm backups stored on disk.'
|
||||
)
|
||||
] = '/tmp/dbrm_backups',
|
||||
d: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
'-d', '--directory',
|
||||
help='Date or directory chose to restore from.'
|
||||
)
|
||||
] = '',
|
||||
ns: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-ns', '--no-start',
|
||||
help=(
|
||||
'Do not attempt columnstore startup post dbrm_restore.'
|
||||
)
|
||||
)
|
||||
] = False,
|
||||
sdbk: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-sdbk/-no-sdbk', '--skip-dbrm-backup/--no-skip-dbrm-backup',
|
||||
help=(
|
||||
'Skip backing up dbrms before restoring.'
|
||||
)
|
||||
)
|
||||
] = True,
|
||||
ssm: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
'-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager',
|
||||
help='Skip backing up storagemanager directory.'
|
||||
)
|
||||
] = True,
|
||||
):
|
||||
"""Restore Columnstore DBRM data."""
|
||||
|
||||
# Default: ./$0 dbrm_restore --path /tmp/dbrm_backups
|
||||
|
||||
# Examples:
|
||||
# ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842
|
||||
# ./$0 dbrm_restore --path /tmp/dbrm_backups --directory dbrm_backup_20240318_172842 --no-start
|
||||
arguments = []
|
||||
for arg_name, value in locals().items():
|
||||
sh_arg = cook_sh_arg(arg_name, value)
|
||||
if sh_arg is None:
|
||||
continue
|
||||
arguments.append(sh_arg)
|
||||
cmd = f'{MCS_BACKUP_MANAGER_SH} dbrm_restore {" ".join(arguments)}'
|
||||
success, _ = BaseDispatcher.exec_command(cmd, stdout=sys.stdout)
|
||||
return {'success': success}
|
||||
2095
cmapi/scripts/cs_package_manager.sh
Normal file
2095
cmapi/scripts/cs_package_manager.sh
Normal file
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,839 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Documentation
|
||||
# ./ cs_package_manager.sh help
|
||||
|
||||
# Variables
|
||||
enterprise_token=""
|
||||
dev_drone_key=""
|
||||
|
||||
if [ ! -f /var/lib/columnstore/local/module ]; then pm="pm1"; else pm=$(cat /var/lib/columnstore/local/module); fi;
|
||||
pm_number=$(echo "$pm" | tr -dc '0-9')
|
||||
action=$1
|
||||
|
||||
print_help_text() {
|
||||
echo "Version 1.0
|
||||
|
||||
Example Remove:
|
||||
bash cs_package_manager.sh remove
|
||||
bash cs_package_manager.sh remove all
|
||||
|
||||
Example Install:
|
||||
bash cs_package_manager.sh install [enterprise|community|dev] [version|branch] [build num]
|
||||
bash cs_package_manager.sh install enterprise 10.6.12-8
|
||||
bash cs_package_manager.sh install community 11.1
|
||||
bash cs_package_manager.sh install dev develop cron/8629
|
||||
bash cs_package_manager.sh install dev develop-23.02 pull_request/7256
|
||||
|
||||
Example Check:
|
||||
bash cs_package_manager.sh check community
|
||||
bash cs_package_manager.sh check enterprise
|
||||
"
|
||||
}
|
||||
|
||||
wait_cs_down() {
|
||||
retries=$1;
|
||||
|
||||
if [ $retries -gt 6 ]; then
|
||||
printf "\n[!!!] Columnstore is still online ... exiting \n\n"
|
||||
exit 2;
|
||||
fi;
|
||||
|
||||
# if columnstore still online stop
|
||||
if [ -z $(pidof PrimProc) ]; then
|
||||
printf "[+] Confirmation: columnstore OFFLINE \n";
|
||||
mcs_offine=true
|
||||
return 1;
|
||||
else
|
||||
printf "\n[+] Columnstore is ONLINE - waiting 5s to retry, attempt: $retries...\n";
|
||||
sleep 5
|
||||
((retries++))
|
||||
wait_cs_down $retries
|
||||
fi;
|
||||
}
|
||||
|
||||
print_and_delete() {
|
||||
printf " - %-25s ..." "$1"
|
||||
rm -rf $1
|
||||
printf " Done\n"
|
||||
}
|
||||
|
||||
init_cs_down() {
|
||||
mcs_offine=false
|
||||
if [ "$pm_number" == "1" ]; then
|
||||
if [ -z $(pidof PrimProc) ]; then
|
||||
# printf "\n[+] Columnstore offline already";
|
||||
mcs_offine=true
|
||||
else
|
||||
|
||||
# Adjust for package manager
|
||||
cmapi_installed_command=""
|
||||
case $package_manager in
|
||||
yum )
|
||||
cmapi_installed_command="yum list installed MariaDB-columnstore-cmapi &> /dev/null;";
|
||||
;;
|
||||
apt )
|
||||
cmapi_installed_command="dpkg-query -s mariadb-columnstore-cmapi &> /dev/null;";
|
||||
;;
|
||||
*) # unknown option
|
||||
echo "\npackage manager not implemented: $package_manager\n"
|
||||
exit 2;
|
||||
esac
|
||||
|
||||
# Check cmapi installed
|
||||
if eval $cmapi_installed_command ; then
|
||||
|
||||
# Check for edge case
|
||||
if [ "$(mcs cluster status | jq -r '.num_nodes')" == "0" ]; then
|
||||
printf "[!!] Noticed cmapi installed but no nodes configured...\n"
|
||||
add_primary_node_cmapi
|
||||
sleep 1;
|
||||
fi
|
||||
|
||||
# Stop columnstore
|
||||
printf "\n[+] Stopping columnstore ... \n";
|
||||
if command -v mcs &> /dev/null; then
|
||||
if ! mcs cluster stop; then
|
||||
echo "[!] Failed stopping via mcs ... trying cmapi curl"
|
||||
stop_cs_cmapi_via_curl
|
||||
fi
|
||||
else
|
||||
stop_cs_cmapi_via_curl
|
||||
fi
|
||||
else
|
||||
stop_cs_via_systemctl
|
||||
fi
|
||||
fi
|
||||
fi;
|
||||
}
|
||||
|
||||
do_yum_remove() {
|
||||
|
||||
if ! command -v yum &> /dev/null ; then
|
||||
printf "[!!] Cant access yum\n"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
init_cs_down
|
||||
|
||||
wait_cs_down 0
|
||||
|
||||
printf "[+] Removing packages - $(date) ... \n"
|
||||
|
||||
if yum list installed MariaDB-server &>/dev/null; then
|
||||
systemctl stop mariadb
|
||||
fi
|
||||
|
||||
if yum list installed MariaDB-columnstore-cmapi &>/dev/null; then
|
||||
systemctl stop mariadb-columnstore-cmapi
|
||||
fi
|
||||
|
||||
# remove any mdb rpms on disk
|
||||
if ls MariaDB-*.rpm &>/dev/null; then
|
||||
print_and_delete "MariaDB-*.rpm"
|
||||
fi
|
||||
|
||||
# remove all current MDB packages
|
||||
if yum list installed MariaDB-* &>/dev/null; then
|
||||
yum remove MariaDB-* -y
|
||||
fi
|
||||
|
||||
# remove offical & custom yum repos
|
||||
print_and_delete "/etc/yum.repos.d/mariadb.repo"
|
||||
print_and_delete "/etc/yum.repos.d/drone.repo"
|
||||
}
|
||||
|
||||
do_apt_remove() {
|
||||
|
||||
if ! command -v apt &> /dev/null ; then
|
||||
printf "[!!] Cant access apt\n"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
if ! command -v dpkg-query &> /dev/null ; then
|
||||
printf "[!!] Cant access dpkg-query\n"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
init_cs_down
|
||||
|
||||
wait_cs_down 0
|
||||
|
||||
printf "[+] Removing packages - $(date) ... \n"
|
||||
|
||||
if dpkg-query -s mariadb-server &>/dev/null; then
|
||||
systemctl stop mariadb
|
||||
fi
|
||||
|
||||
if dpkg-query -s mariadb-columnstore-cmapi &>/dev/null; then
|
||||
systemctl stop mariadb-columnstore-cmapi
|
||||
fi
|
||||
|
||||
# remove any mdb rpms on disk
|
||||
if ls mariadb*.deb &>/dev/null; then
|
||||
print_and_delete "mariadb*.deb"
|
||||
fi
|
||||
|
||||
# remove all current MDB packages
|
||||
if [ "$(apt list --installed mariadb-* 2>/dev/null | wc -l)" -gt 1 ]; then
|
||||
apt remove mariadb-* -y
|
||||
fi
|
||||
|
||||
# remove offical & custom yum repos
|
||||
print_and_delete "/lib/systemd/system/mariadb.service"
|
||||
print_and_delete "/lib/systemd/system/mariadb.service.d"
|
||||
print_and_delete "/etc/apt/sources.list.d/mariadb.list"
|
||||
print_and_delete "/etc/apt/sources.list.d/drone.list"
|
||||
systemctl daemon-reload
|
||||
}
|
||||
|
||||
do_remove() {
|
||||
|
||||
check_operating_system
|
||||
check_package_managers
|
||||
|
||||
case $distro_info in
|
||||
centos | rocky )
|
||||
do_yum_remove
|
||||
;;
|
||||
# debian )
|
||||
|
||||
# ;;
|
||||
ubuntu )
|
||||
do_apt_remove
|
||||
;;
|
||||
*) # unknown option
|
||||
echo "\nos & version not implemented: $distro_info\n"
|
||||
exit 2;
|
||||
esac
|
||||
|
||||
if [ "$2" == "all" ]; then
|
||||
printf "\n[+] Removing all columnstore files & dirs\n"
|
||||
print_and_delete "/var/lib/mysql/"
|
||||
print_and_delete "/var/lib/columnstore/"
|
||||
print_and_delete "/etc/my.cnf.d/*"
|
||||
print_and_delete "/etc/columnstore/*"
|
||||
fi;
|
||||
|
||||
printf "\n[+] Done\n\n"
|
||||
}
|
||||
|
||||
check_package_managers() {
|
||||
|
||||
package_manager='';
|
||||
if command -v apt &> /dev/null ; then
|
||||
if ! command -v dpkg-query &> /dev/null ; then
|
||||
printf "[!!] Cant access dpkg-query\n"
|
||||
exit 1;
|
||||
fi
|
||||
package_manager="apt";
|
||||
fi
|
||||
|
||||
if command -v yum &> /dev/null ; then
|
||||
package_manager="yum";
|
||||
fi
|
||||
|
||||
if [ $package_manager == '' ]; then
|
||||
echo "[!!] No package manager found: yum or apt must be installed"
|
||||
exit 1;
|
||||
fi;
|
||||
}
|
||||
|
||||
check_operating_system() {
|
||||
|
||||
distro_info=$(awk -F= '/^ID=/{gsub(/"/, "", $2); print $2}' /etc/os-release)
|
||||
version_id=$(grep 'VERSION_ID=' /etc/os-release | awk -F= '{gsub(/"/, "", $2); print $2}' | awk -F. '{print $1}')
|
||||
|
||||
echo "Distro: $distro_info"
|
||||
echo "Version: $version_id"
|
||||
|
||||
# distros=(centos7 debian11 debian12 rockylinux8 rockylinux9 ubuntu20.04 ubuntu22.04)
|
||||
case $distro_info in
|
||||
centos )
|
||||
distro="${distro_info}${version_id}"
|
||||
;;
|
||||
debian )
|
||||
distro="${distro_info}${version_id}"
|
||||
;;
|
||||
rocky )
|
||||
distro="rockylinux${version_id}"
|
||||
;;
|
||||
ubuntu )
|
||||
distro="${distro_info}${version_id}"
|
||||
;;
|
||||
*) # unknown option
|
||||
printf "\ncheck_operating_system: unknown os & version: $distro_info\n"
|
||||
exit 2;
|
||||
esac
|
||||
}
|
||||
|
||||
check_cpu_architecture() {
|
||||
|
||||
architecture=$(uname -m)
|
||||
echo "CPU: $architecture"
|
||||
|
||||
# arch=(amd64 arm64)
|
||||
case $architecture in
|
||||
x86_64 )
|
||||
arch="amd64"
|
||||
;;
|
||||
aarch64 )
|
||||
arch="arm64"
|
||||
;;
|
||||
*) # unknown option
|
||||
echo "Error: Unsupported architecture ($architecture)"
|
||||
esac
|
||||
}
|
||||
|
||||
check_no_mdb_installed() {
|
||||
|
||||
packages=""
|
||||
case $distro_info in
|
||||
centos )
|
||||
packages=$(yum list installed | grep -i mariadb)
|
||||
;;
|
||||
# debian )
|
||||
|
||||
# ;;
|
||||
rocky )
|
||||
packages=$(yum list installed | grep -i mariadb)
|
||||
;;
|
||||
ubuntu )
|
||||
packages=$(apt list --installed mariadb-* 2>/dev/null | grep -i mariadb);
|
||||
;;
|
||||
*) # unknown option
|
||||
printf "\ncheck_no_mdb_installed: os & version not implemented: $distro_info\n"
|
||||
exit 2;
|
||||
esac
|
||||
|
||||
if [ -n "$packages" ]; then
|
||||
printf "\nMariaDB packages are installed. Please uninstall them before continuing.\n"
|
||||
echo $packages;
|
||||
printf "Example: bash $0 remove\n\n"
|
||||
exit 2;
|
||||
fi;
|
||||
}
|
||||
|
||||
check_aws_cli_installed() {
|
||||
|
||||
if ! command -v aws &> /dev/null ; then
|
||||
echo "[!] aws cli - binary could not be found"
|
||||
echo "[+] Installing aws cli ..."
|
||||
rm -rf aws awscliv2.zip
|
||||
yum install unzip -y;
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip";
|
||||
unzip awscliv2.zip;
|
||||
sudo ./aws/install;
|
||||
mv /usr/local/bin/aws /usr/bin/aws;
|
||||
aws configure set default.s3.max_concurrent_requests 700
|
||||
fi
|
||||
}
|
||||
|
||||
check_dev_build_exists() {
|
||||
|
||||
if ! aws s3 ls $s3_path --no-sign-request &> /dev/null; then
|
||||
printf "Defined dev build doesnt exist in aws\n\n"
|
||||
exit 2;
|
||||
fi;
|
||||
}
|
||||
|
||||
do_enterprise_apt_install() {
|
||||
|
||||
# Install MariaDB
|
||||
apt-get clean
|
||||
apt install mariadb-server -y
|
||||
sleep 2
|
||||
systemctl daemon-reload
|
||||
systemctl enable mariadb
|
||||
systemctl start mariadb
|
||||
|
||||
# Install Columnstore
|
||||
apt install mariadb-plugin-columnstore mariadb-columnstore-cmapi jq -y
|
||||
systemctl daemon-reload
|
||||
systemctl enable mariadb-columnstore-cmapi
|
||||
systemctl start mariadb-columnstore-cmapi
|
||||
mariadb -e "show status like '%Columnstore%';"
|
||||
sleep 2
|
||||
|
||||
# Startup cmapi - if defined
|
||||
add_primary_node_cmapi
|
||||
}
|
||||
|
||||
do_enterprise_yum_install() {
|
||||
|
||||
# Install MariaDB
|
||||
yum clean all
|
||||
yum install MariaDB-server -y
|
||||
sleep 2
|
||||
systemctl enable mariadb
|
||||
systemctl start mariadb
|
||||
|
||||
# Install Columnstore
|
||||
yum install MariaDB-columnstore-engine MariaDB-columnstore-cmapi -y
|
||||
systemctl enable mariadb-columnstore-cmapi
|
||||
systemctl start mariadb-columnstore-cmapi
|
||||
mariadb -e "show status like '%Columnstore%';"
|
||||
sleep 1;
|
||||
|
||||
# Startup cmapi - if defined
|
||||
add_primary_node_cmapi
|
||||
}
|
||||
|
||||
enterprise_install() {
|
||||
|
||||
version=$3
|
||||
if [ -z $enterprise_token ]; then
|
||||
printf "Enterprise token empty: $enterprise_token\n"
|
||||
printf "edit $0 to add token\n\n"
|
||||
|
||||
exit 1;
|
||||
fi;
|
||||
|
||||
if [ -z $version ]; then
|
||||
printf "Version empty: $version\n"
|
||||
exit 1;
|
||||
fi;
|
||||
|
||||
echo "Token: $enterprise_token"
|
||||
echo "MariaDB Version: $version"
|
||||
|
||||
# Download Repo setup
|
||||
rm -rf mariadb_es_repo_setup
|
||||
wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup ;chmod +x mariadb_es_repo_setup;
|
||||
if ! bash mariadb_es_repo_setup --token="$enterprise_token" --apply --mariadb-server-version="$version"; then
|
||||
echo "exiting ..."
|
||||
exit 2;
|
||||
fi;
|
||||
|
||||
case $distro_info in
|
||||
centos | rocky )
|
||||
do_enterprise_yum_install "$@"
|
||||
;;
|
||||
# debian )
|
||||
|
||||
# ;;
|
||||
ubuntu )
|
||||
do_enterprise_apt_install "$@"
|
||||
;;
|
||||
*) # unknown option
|
||||
printf "\nos & version not implemented: $distro_info\n"
|
||||
exit 2;
|
||||
esac
|
||||
}
|
||||
|
||||
community_install() {
|
||||
|
||||
version=$3
|
||||
if [ -z $version ]; then
|
||||
printf "Version empty: $version\n"
|
||||
|
||||
exit 1;
|
||||
fi;
|
||||
|
||||
echo "MariaDB Community Version: $version"
|
||||
|
||||
# Download Repo setup
|
||||
rm -rf mariadb_repo_setup
|
||||
|
||||
if ! curl -sS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | bash -s -- --mariadb-server-version=mariadb-$version ; then
|
||||
echo "version bad. exiting ..."
|
||||
exit 2;
|
||||
fi;
|
||||
yum clean all
|
||||
|
||||
# Install MariaDB then Columnstore
|
||||
yum install MariaDB-server -y
|
||||
sleep 2;
|
||||
systemctl enable mariadb
|
||||
systemctl start mariadb
|
||||
yum install MariaDB-columnstore-engine MariaDB-columnstore-cmapi -y
|
||||
systemctl enable mariadb-columnstore-cmapi
|
||||
systemctl start mariadb-columnstore-cmapi
|
||||
mariadb -e "show status like '%Columnstore%';"
|
||||
|
||||
add_primary_node_cmapi
|
||||
}
|
||||
|
||||
get_set_cmapi_key() {
|
||||
|
||||
CMAPI_CNF="/etc/columnstore/cmapi_server.conf"
|
||||
|
||||
if [ ! -f $CMAPI_CNF ]; then
|
||||
echo "[!!] No cmapi config file found"
|
||||
exit 1;
|
||||
fi;
|
||||
|
||||
# Add API Key if missing
|
||||
if [ -z "$(grep ^x-api-key $CMAPI_CNF)" ]; then
|
||||
|
||||
if ! command -v openssl &> /dev/null ; then
|
||||
api_key="19bb89d77cb8edfe0864e05228318e3dfa58e8f45435fbd9bd12c462a522a1e9"
|
||||
else
|
||||
api_key=$(openssl rand -hex 32)
|
||||
fi
|
||||
# mcs cluster set api-key --key $primary_ip
|
||||
echo "[+] Setting API Key:"
|
||||
if curl -s https://127.0.0.1:8640/cmapi/0.4.0/cluster/status \
|
||||
--header 'Content-Type:application/json' \
|
||||
--header "x-api-key:$api_key" -k ; then
|
||||
sleep 2;
|
||||
else
|
||||
echo "Failed to set API key"
|
||||
exit 1;
|
||||
fi
|
||||
else
|
||||
api_key=$(grep ^x-api-key $CMAPI_CNF | cut -d "=" -f 2 | tr -d " ")
|
||||
fi
|
||||
}
|
||||
|
||||
add_node_cmapi_via_curl() {
|
||||
|
||||
node_ip=$1
|
||||
if [ -z $api_key ]; then get_set_cmapi_key; fi;
|
||||
|
||||
# Add Node
|
||||
printf "\n[+] Adding primary node: \n"
|
||||
if curl -k -s -X PUT https://127.0.0.1:8640/cmapi/0.4.0/cluster/node \
|
||||
--header 'Content-Type:application/json' \
|
||||
--header "x-api-key:$api_key" \
|
||||
--data "{\"timeout\": 120, \"node\": \"$node_ip\"}"; then
|
||||
printf "\n[+] Success adding $node_ip\n"
|
||||
else
|
||||
echo "Failed adding node"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
start_cs_via_systemctl() {
|
||||
if systemctl start mariadb-columnstore ; then
|
||||
echo "[+] Started Columnstore"
|
||||
else
|
||||
echo "[!!] Failed to start columnstore"
|
||||
exit 1;
|
||||
fi;
|
||||
}
|
||||
|
||||
start_cs_cmapi_via_curl() {
|
||||
|
||||
if [ -z $api_key ]; then get_set_cmapi_key; fi;
|
||||
|
||||
if curl -k -s -X PUT https://127.0.0.1:8640/cmapi/0.4.0/cluster/start \
|
||||
--header 'Content-Type:application/json' \
|
||||
--header "x-api-key:$api_key" \
|
||||
--data '{"timeout":20}'; then
|
||||
echo "[+] Started Columnstore"
|
||||
else
|
||||
echo "[!] Failed to start columnstore"
|
||||
echo "Trying via systemctl ..."
|
||||
start_cs_via_systemctl
|
||||
fi;
|
||||
}
|
||||
|
||||
stop_cs_via_systemctl() {
|
||||
if systemctl stop mariadb-columnstore ; then
|
||||
echo "[+] Stopped Columnstore"
|
||||
else
|
||||
echo "[!!] Failed to stop columnstore"
|
||||
exit 1;
|
||||
fi;
|
||||
}
|
||||
|
||||
stop_cs_cmapi_via_curl() {
|
||||
|
||||
if [ -z $api_key ]; then get_set_cmapi_key; fi;
|
||||
|
||||
if curl -k -s -X PUT https://127.0.0.1:8640/cmapi/0.4.0/cluster/shutdown \
|
||||
--header 'Content-Type:application/json' \
|
||||
--header "x-api-key:$api_key" \
|
||||
--data '{"timeout":20}'; then
|
||||
echo "[+] Stopped Columnstore"
|
||||
else
|
||||
echo "[!] Failed to stop columnstore via cmapi"
|
||||
echo "Trying via systemctl ..."
|
||||
stop_cs_via_systemctl
|
||||
fi;
|
||||
}
|
||||
|
||||
add_primary_node_cmapi() {
|
||||
|
||||
primary_ip="127.0.0.1"
|
||||
get_set_cmapi_key
|
||||
|
||||
if ! command -v mcs &> /dev/null ; then
|
||||
echo "mcs - binary could not be found"
|
||||
add_node_cmapi_via_curl $primary_ip
|
||||
echo "[+] Starting Columnstore ..."
|
||||
start_cs_cmapi_via_curl
|
||||
|
||||
else
|
||||
if [ "$(mcs cluster status | jq -r '.num_nodes')" == "0" ]; then
|
||||
|
||||
echo "[+] Adding primary node ..."
|
||||
if timeout 30s mcs cluster node add --node $primary_ip; then
|
||||
echo "[+] Success adding $primary_ip"
|
||||
else
|
||||
echo "[!] Failed ... trying cmapi curl"
|
||||
add_node_cmapi_via_curl $primary_ip
|
||||
fi;
|
||||
fi;
|
||||
echo "[+] Starting Columnstore ..."
|
||||
mcs cluster start
|
||||
fi
|
||||
}
|
||||
|
||||
dev_install() {
|
||||
|
||||
if [ -z $dev_drone_key ]; then printf "Missing dev_drone_key: \n"; exit; fi;
|
||||
check_aws_cli_installed
|
||||
|
||||
echo "Branch: $3"
|
||||
echo "Build: $4"
|
||||
dronePath="s3://$dev_drone_key"
|
||||
branch="$3"
|
||||
build="$4"
|
||||
product="10.6-enterprise"
|
||||
|
||||
# Construct URLs
|
||||
s3_path="$dronePath/$branch/$build/$product/$arch/$distro"
|
||||
replace="https://$dev_drone_key.s3.amazonaws.com/"
|
||||
# Use sed to replace the s3 path to create https path
|
||||
yum_http=$(echo "$s3_path" | sed "s|s3://$dev_drone_key/|$replace|")
|
||||
echo "Locations:"
|
||||
echo "RPM: $s3_path"
|
||||
echo "Yum: $yum_http"
|
||||
echo "###################################"
|
||||
|
||||
check_dev_build_exists
|
||||
|
||||
echo "[drone]
|
||||
name=Drone Repository
|
||||
baseurl="$yum_http"
|
||||
gpgcheck=0
|
||||
enabled=1
|
||||
" > /etc/yum.repos.d/drone.repo
|
||||
yum clean all
|
||||
# yum makecache
|
||||
# yum list --disablerepo="*" --enablerepo="drone"
|
||||
|
||||
# ALL RPMS: aws s3 cp $s3_path/ . --recursive --exclude "debuginfo" --include "*.rpm"
|
||||
aws s3 cp $s3_path/ . --recursive --exclude "*" --include "MariaDB-server*" --exclude "*debug*" --no-sign-request
|
||||
|
||||
# Confirm Downloaded server rpm
|
||||
if ! ls MariaDB-server-*.rpm 1> /dev/null 2>&1; then
|
||||
echo "Error: No MariaDB-server RPMs were found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
yum install MariaDB-server-*.rpm -y
|
||||
yum install MariaDB-columnstore-engine MariaDB-columnstore-cmapi -y
|
||||
systemctl start mariadb
|
||||
systemctl start mariadb-columnstore-cmapi
|
||||
mariadb -e "show status like '%Columnstore%';"
|
||||
|
||||
add_primary_node_cmapi
|
||||
}
|
||||
|
||||
do_install() {
|
||||
|
||||
check_operating_system
|
||||
check_cpu_architecture
|
||||
check_no_mdb_installed
|
||||
|
||||
repo=$2
|
||||
echo "Repository: $repo"
|
||||
case $repo in
|
||||
enterprise )
|
||||
# pull from public enterprise repo
|
||||
enterprise_install "$@" ;
|
||||
;;
|
||||
community )
|
||||
# pull from public community repo
|
||||
community_install "$@" ;
|
||||
;;
|
||||
dev )
|
||||
dev_install "$@" ;
|
||||
;;
|
||||
*) # unknown option
|
||||
echo "Unknown repo: $repo\n"
|
||||
exit 2;
|
||||
esac
|
||||
|
||||
printf "\nDone - $(date)\n"
|
||||
}
|
||||
|
||||
do_check() {
|
||||
|
||||
check_operating_system
|
||||
check_cpu_architecture
|
||||
|
||||
repo=$2
|
||||
echo "Repository: $repo"
|
||||
case $repo in
|
||||
enterprise )
|
||||
|
||||
if [ -z $enterprise_token ]; then
|
||||
printf "Enterprise token empty: $enterprise_token\n"
|
||||
printf "edit $0 to add token\n\n"
|
||||
exit 1;
|
||||
fi;
|
||||
|
||||
url_base="https://dlm.mariadb.com"
|
||||
url_page="/browse/$enterprise_token/mariadb_enterprise_server/"
|
||||
ignore="/login"
|
||||
curl -s "$url_base$url_page" > mdb-tmp.html
|
||||
major_version_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep $url_page | grep -v $ignore )
|
||||
#echo $major_version_links
|
||||
for major_link in ${major_version_links[@]}
|
||||
do
|
||||
#echo "Major: $major_link"
|
||||
curl -s "$url_base$major_link" > mdb-tmp.html
|
||||
minor_version_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep $url_page | grep -v $ignore )
|
||||
for minor_link in ${minor_version_links[@]}
|
||||
do
|
||||
if [ "$minor_link" != "$url_page" ]; then
|
||||
#echo " Minor: $minor_link"
|
||||
case $distro_info in
|
||||
centos | rocky )
|
||||
path="rpm/rhel/$version_id/$architecture/rpms/"
|
||||
curl -s "$url_base$minor_link$path" > mdb-tmp.html
|
||||
package_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep "$path" | grep "columnstore-engine" | grep -v debug | tail -1 )
|
||||
if [ ! -z "$package_links" ]; then
|
||||
#echo "----------"
|
||||
#echo "$package_links"
|
||||
|
||||
mariadb_version="${package_links#*mariadb-enterprise-server/}"
|
||||
columnstore_version="${mariadb_version#*columnstore-engine-}"
|
||||
mariadb_version="$( echo $mariadb_version | awk -F/ '{print $1}' )"
|
||||
|
||||
# unqiue to enterprise
|
||||
standard_mariadb_version="${mariadb_version//-/_}"
|
||||
columnstore_version="$( echo $columnstore_version | awk -F"${standard_mariadb_version}_" '{print $2}' | awk -F".el" '{print $1}' )"
|
||||
printf "%-8s %-12s %-12s %-12s\n" "MariaDB:" "$mariadb_version" "Columnstore:" "$columnstore_version";
|
||||
fi;
|
||||
;;
|
||||
ubuntu )
|
||||
path="deb/pool/main/m/"
|
||||
curl -s "$url_base$minor_link$path" > mdb-tmp.html
|
||||
|
||||
# unqiue - this link/path can change
|
||||
mariadb_version_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep -v $ignore | grep -v cmapi | grep ^mariadb )
|
||||
#echo "$url_base$minor_link$path"
|
||||
for mariadb_link in ${mariadb_version_links[@]}
|
||||
do
|
||||
#echo $mariadb_link
|
||||
path="deb/pool/main/m/$mariadb_link"
|
||||
curl -s "$url_base$minor_link$path" > mdb-tmp.html
|
||||
package_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep "$path" | grep "columnstore_" | grep -v debug | tail -1 )
|
||||
if [ ! -z "$package_links" ]; then
|
||||
# echo "$package_links"
|
||||
# echo "----------"
|
||||
mariadb_version="${package_links#*mariadb-enterprise-server/}"
|
||||
columnstore_version="${mariadb_version#*columnstore-engine-}"
|
||||
mariadb_version="$( echo $mariadb_version | awk -F/ '{print $1}' )"
|
||||
columnstore_version="$( echo $columnstore_version | awk -F"columnstore_" '{print $2}' | awk -F"-" '{print $2}' | awk -F"+maria" '{print $1}' )"
|
||||
# echo "MariaDB: $mariadb_version Columnstore: $columnstore_version"
|
||||
printf "%-8s %-12s %-12s %-12s\n" "MariaDB:" "$mariadb_version" "Columnstore:" "$columnstore_version";
|
||||
fi;
|
||||
done
|
||||
|
||||
|
||||
;;
|
||||
*) # unknown option
|
||||
printf "Not implemented for: $distro_info\n"
|
||||
exit 2;
|
||||
esac
|
||||
|
||||
|
||||
fi;
|
||||
done
|
||||
done
|
||||
;;
|
||||
community )
|
||||
|
||||
# pull from public community repo
|
||||
url_base="https://dlm.mariadb.com"
|
||||
url_page="/browse/mariadb_server/"
|
||||
ignore="/login"
|
||||
curl -s "$url_base$url_page" > mdb-tmp.html
|
||||
major_version_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep $url_page | grep -v $ignore )
|
||||
|
||||
for major_link in ${major_version_links[@]}
|
||||
do
|
||||
#echo "Major: $major_link"
|
||||
curl -s "$url_base$major_link" > mdb-tmp.html
|
||||
minor_version_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep $url_page | grep -v $ignore )
|
||||
for minor_link in ${minor_version_links[@]}
|
||||
do
|
||||
if [ "$minor_link" != "$url_page" ]; then
|
||||
#echo " Minor: $minor_link"
|
||||
case $distro_info in
|
||||
centos | rocky )
|
||||
path="yum/centos/$version_id/$architecture/rpms/"
|
||||
curl -s "$url_base$minor_link$path" > mdb-tmp.html
|
||||
package_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep "$path" | grep "columnstore-engine" | grep -v debug | tail -1 )
|
||||
if [ ! -z "$package_links" ]; then
|
||||
# echo "$package_links"
|
||||
# echo "----------"
|
||||
mariadb_version="${package_links#*mariadb-}"
|
||||
columnstore_version="${mariadb_version#*columnstore-engine-}"
|
||||
mariadb_version="$( echo $mariadb_version | awk -F/ '{print $1}' )"
|
||||
columnstore_version="$( echo $columnstore_version | awk -F_ '{print $2}' | awk -F".el" '{print $1}' )"
|
||||
# echo "MariaDB: $mariadb_version Columnstore: $columnstore_version"
|
||||
printf "%-8s %-12s %-12s %-12s\n" "MariaDB:" "$mariadb_version" "Columnstore:" "$columnstore_version";
|
||||
fi;
|
||||
;;
|
||||
ubuntu )
|
||||
path="repo/$distro_info/pool/main/m/mariadb/"
|
||||
curl -s "$url_base$minor_link$path" > mdb-tmp.html
|
||||
package_links=$(grep -oP 'href="\K[^"]+' mdb-tmp.html | grep "$path" | grep "columnstore_" | grep -v debug | tail -1 )
|
||||
if [ ! -z "$package_links" ]; then
|
||||
# echo "$package_links"
|
||||
# echo "----------"
|
||||
mariadb_version="${package_links#*mariadb-}"
|
||||
columnstore_version="${mariadb_version#*columnstore-engine-}"
|
||||
mariadb_version="$( echo $mariadb_version | awk -F/ '{print $1}' )"
|
||||
columnstore_version="$( echo $columnstore_version | awk -F"columnstore_" '{print $2}' | awk -F"-" '{print $2}' | awk -F"+maria" '{print $1}' )"
|
||||
# echo "MariaDB: $mariadb_version Columnstore: $columnstore_version"
|
||||
printf "%-8s %-12s %-12s %-12s\n" "MariaDB:" "$mariadb_version" "Columnstore:" "$columnstore_version";
|
||||
fi;
|
||||
;;
|
||||
*) # unknown option
|
||||
printf "Not implemented for: $distro_info\n"
|
||||
exit 2;
|
||||
esac
|
||||
|
||||
|
||||
fi;
|
||||
done
|
||||
done
|
||||
;;
|
||||
dev )
|
||||
printf "Not implemented for: $repo\n"
|
||||
exit 1;
|
||||
;;
|
||||
*) # unknown option
|
||||
printf "Unknown repo: $repo\n"
|
||||
exit 2;
|
||||
esac
|
||||
}
|
||||
|
||||
|
||||
case $action in
|
||||
remove )
|
||||
do_remove "$@" ;
|
||||
;;
|
||||
install )
|
||||
do_install "$@";
|
||||
;;
|
||||
check )
|
||||
do_check "$@"
|
||||
;;
|
||||
help | -h | --help | -help)
|
||||
print_help_text;
|
||||
exit 1;
|
||||
;;
|
||||
*) # unknown option
|
||||
printf "Unknown Action: $1\n"
|
||||
print_help_text
|
||||
exit 2;
|
||||
esac
|
||||
Reference in New Issue
Block a user