diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index f2b17c328..f6491d57d 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -47,6 +47,7 @@ install(PROGRAMS columnstore-post-install mcs_module_installer.sh mcs-stop-controllernode.sh mcs-loadbrm.py + mcs-savebrm.py mariadb-columnstore-start.sh mariadb-columnstore-stop.sh DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 05104ea37..565201eab 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -261,16 +261,13 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then if [ -z "$MCS_S3_BUCKET" ]; then echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_BUCKET." fi - if [ -z "$MCS_S3_ENDPOINT" ]; then - echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_ENDPOINT." - fi if [ -z "$MCS_S3_ACCESS_KEY_ID" ]; then echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_ACCESS_KEY_ID." fi if [ -z "$MCS_S3_SECRET_ACCESS_KEY" ]; then echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_SECRET_ACCESS_KEY." fi - if [ -z "$MCS_S3_BUCKET" ] || [ -z "$MCS_S3_ENDPOINT" ] || [ -z "$MCS_S3_ACCESS_KEY_ID" ] || [ -z "$MCS_S3_SECRET_ACCESS_KEY" ]; then + if [ -z "$MCS_S3_BUCKET" ] || [ -z "$MCS_S3_ACCESS_KEY_ID" ] || [ -z "$MCS_S3_SECRET_ACCESS_KEY" ]; then echo "Using local storage." else mcsSetConfig -d Installation DBRootStorageType "storagemanager" @@ -291,7 +288,7 @@ systemctl cat mariadb-columnstore.service > /dev/null 2>&1 if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then echo "Populating the engine initial system catalog." systemctl start mariadb-columnstore - sleep 1 + sleep 5 dbbuilder 7 > $tmpDir/dbbuilder.log 2>&1 fi diff --git a/oam/install_scripts/mcs-controllernode.service.in b/oam/install_scripts/mcs-controllernode.service.in index 34b23b0ca..3931d890a 100644 --- a/oam/install_scripts/mcs-controllernode.service.in +++ b/oam/install_scripts/mcs-controllernode.service.in @@ -5,6 +5,7 @@ After=mcs-workernode.service [Service] Type=simple +ExecStartPre=/bin/sleep 7 ExecStart=@ENGINE_BINDIR@/controllernode fg Restart=on-failure ExecStop=@ENGINE_BINDIR@/mcs-stop-controllernode.sh $MAINPID diff --git a/oam/install_scripts/mcs-loadbrm.py.in b/oam/install_scripts/mcs-loadbrm.py.in index 82e0c7fa9..fdf9ae7ba 100644 --- a/oam/install_scripts/mcs-loadbrm.py.in +++ b/oam/install_scripts/mcs-loadbrm.py.in @@ -3,20 +3,17 @@ import subprocess import sys import xml.etree.ElementTree as ET from pathlib import Path - +import time import configparser -api_config_file = '/etc/columnstore/cmapi_server.conf' +API_CONFIG_PATH = '/etc/columnstore/cmapi_server.conf' +BYPASS_SM_PATH = '/tmp/columnstore_tmp_files/rdwrscratch/BRM_saves' def get_key(): cmapi_config = configparser.ConfigParser() - try: - cmapi_config.read(api_config_file) - except FileNotFoundError: - return '' - + cmapi_config.read(API_CONFIG_PATH) if 'Authentication' not in cmapi_config.sections(): return '' return cmapi_config['Authentication'].get('x-api-key', '') @@ -45,12 +42,13 @@ if __name__ == '__main__': bucket = 'some_bucket' dbrmroot = config_root.find('./SystemConfig/DBRMRoot').text + pmCount = int(config_root.find('./SystemModuleConfig/ModuleCount3').text) loadbrm = '/usr/bin/load_brm' brm_saves_current = '' if storage.lower() == 's3' and not bucket.lower() == 'some_bucket': - # load s3 + # start SM cmd = 'systemctl start mcs-storagemanager' retcode = subprocess.call(cmd, shell=True) if retcode < 0: @@ -67,18 +65,20 @@ if __name__ == '__main__': config_root.find('./SystemConfig/DataFilePlugin').text = "libcloudio.so" cs_config.write('/etc/columnstore/Columnstore.xml') + # Delay to allow SM to start up + time.sleep(15) + # Single-node on S3 + if storage.lower() == 's3' and not bucket.lower() == 'some_bucket' and pmCount == 1: try: brm_saves_current = subprocess.check_output(['smcat', brm]) except subprocess.CalledProcessError as e: # will happen when brm file does not exist print('{} does not exist.'.format(brm), file=sys.stderr) else: - pmCount = int(config_root.find('./SystemModuleConfig/ModuleCount3').text) brm = '{}_current'.format(dbrmroot) - + # Multi-node if pmCount > 1: - # load multinode dbrm try: import requests requests.packages.urllib3.disable_warnings() @@ -105,7 +105,12 @@ api_port, api_version, e) if (r.status_code != 200): raise RuntimeError("Error requesting {} from the primary \ node.".format(e)) + + # To avoid SM storing BRM files + if storage.lower() == 's3' and bucket.lower() != 'some_bucket': + dbrmroot = BYPASS_SM_PATH current_name = '{}_{}'.format(dbrmroot, e) + print ("Saving {} to {}".format(e, current_name)) path = Path(current_name) path.write_bytes(r.content) diff --git a/oam/install_scripts/mcs-savebrm.py b/oam/install_scripts/mcs-savebrm.py new file mode 100755 index 000000000..48478fee4 --- /dev/null +++ b/oam/install_scripts/mcs-savebrm.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import subprocess +import sys +import xml.etree.ElementTree as ET +import configparser + +XML_CONFIG_PATH = '/etc/columnstore/Columnstore.xml' +SM_CONFIG_PATH = '/etc/columnstore/storagemanager.cnf' +REST_REQUEST_TO = 10 + + +def get_version(): + return '0.4.0' + + +def get_port(): + return '8640' + + +if __name__ == '__main__': + master_addr = '' + pm_count = 0 + try: + cs_config = ET.parse(XML_CONFIG_PATH) + config_root = cs_config.getroot() + master_addr = config_root.find('./DBRM_Controller/IPAddr').text + pm_count = int(config_root.find('./SystemModuleConfig/ModuleCount3').text) + except (FileNotFoundError, AttributeError, ValueError) as e: + print("Exception had been raised. Continue anyway") + print(str(e)) + + storage = 'LocalStorage' + sm_config = configparser.ConfigParser() + files_read = len(sm_config.read(SM_CONFIG_PATH)) + if files_read == 1: + storage = sm_config.get('ObjectStorage', 'service') + + default_addr = '127.0.0.1' + savebrm = 'save_brm' + is_primary = False + + # For multi-node with local storage or default installations + if (storage.lower() != 's3' and master_addr != default_addr) or \ +master_addr == default_addr: + is_primary = True + print('Multi-node with local-storage detected.') + else: + has_requests = False + try: + import requests + requests.packages.urllib3.disable_warnings() + has_requests = True + except ImportError as e: + print('requests Python module does not exist. \ + Please install CMAPI first.') + if has_requests is True: + try: + print('Requesting for the primary node status.') + api_version = get_version() + api_port = get_port() + url = "https://{}:{}/cmapi/{}/node/primary".format(default_addr, \ + api_port, api_version) + resp = requests.get(url, + verify=False, + timeout=REST_REQUEST_TO) + if (resp.status_code != 200): + print("Error sending GET /node/primary.") + else: + is_primary = resp.json()['is_primary'] == 'True' + except: + print('Failed to request.') + print(str(e)) + + if is_primary is True: + try: + retcode = subprocess.call(savebrm, shell=True) + if retcode < 0: + print('{} exits with {}.'.format(cmd, retcode)) + sys.exit(0) + except OSError as e: + print(str(e)) + sys.exit(0) + + sys.exit(0) diff --git a/oam/install_scripts/mcs-workernode.service.in b/oam/install_scripts/mcs-workernode.service.in index 9e849a35b..4376ccd6a 100644 --- a/oam/install_scripts/mcs-workernode.service.in +++ b/oam/install_scripts/mcs-workernode.service.in @@ -7,6 +7,6 @@ Type=simple ExecStart=@ENGINE_BINDIR@/workernode DBRM_Worker1 fg Restart=on-failure ExecStop=/usr/bin/env bash -c "kill -15 $MAINPID" -ExecStopPost=-@ENGINE_BINDIR@/save_brm +ExecStopPost=-@ENGINE_BINDIR@/mcs-savebrm.py ExecStopPost=/usr/bin/env bash -c "clearShm > /dev/null 2>&1" TimeoutStopSec=120