Introduced an artificial delay b/w SM and controllernode to allow SM to start

Introduced mcs-savebrm.py to avoid saving BRM on non-primary nodes
in multi-node S3 clusters

Refactored mcs-loadbrm.py to download BRM files from the primary in
in multi-node S3 clusters

oam/install_scripts/CMakeLists.txt

@@ -47,6 +47,7 @@ install(PROGRAMS columnstore-post-install
                 mcs_module_installer.sh
                 mcs-stop-controllernode.sh
                 mcs-loadbrm.py
+                mcs-savebrm.py
                 mariadb-columnstore-start.sh
                 mariadb-columnstore-stop.sh
                 DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

oam/install_scripts/columnstore-post-install.in

@@ -261,16 +261,13 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then
   if [ -z "$MCS_S3_BUCKET" ]; then
     echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_BUCKET."
   fi  
-  if [ -z "$MCS_S3_ENDPOINT" ]; then
-    echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_ENDPOINT."
-  fi  
   if [ -z "$MCS_S3_ACCESS_KEY_ID" ]; then
     echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_ACCESS_KEY_ID."
   fi  
   if [ -z "$MCS_S3_SECRET_ACCESS_KEY" ]; then
     echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_SECRET_ACCESS_KEY."
   fi  
-  if [ -z "$MCS_S3_BUCKET" ] || [ -z "$MCS_S3_ENDPOINT" ] || [ -z "$MCS_S3_ACCESS_KEY_ID" ] || [ -z "$MCS_S3_SECRET_ACCESS_KEY" ]; then 
+  if [ -z "$MCS_S3_BUCKET" ] || [ -z "$MCS_S3_ACCESS_KEY_ID" ] || [ -z "$MCS_S3_SECRET_ACCESS_KEY" ]; then 
     echo "Using local storage."
   else
       mcsSetConfig -d Installation DBRootStorageType "storagemanager"
@@ -291,7 +288,7 @@ systemctl cat mariadb-columnstore.service > /dev/null 2>&1
 if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then
     echo "Populating the engine initial system catalog."
     systemctl start mariadb-columnstore
-    sleep 1
+    sleep 5
     dbbuilder 7 > $tmpDir/dbbuilder.log 2>&1
 fi
  

oam/install_scripts/mcs-controllernode.service.in

@@ -5,6 +5,7 @@ After=mcs-workernode.service
 
 [Service]
 Type=simple
+ExecStartPre=/bin/sleep 7
 ExecStart=@ENGINE_BINDIR@/controllernode fg
 Restart=on-failure
 ExecStop=@ENGINE_BINDIR@/mcs-stop-controllernode.sh $MAINPID

oam/install_scripts/mcs-loadbrm.py.in

@@ -3,20 +3,17 @@ import subprocess
 import sys
 import xml.etree.ElementTree as ET
 from pathlib import Path
-
+import time
 import configparser
 
 
-api_config_file = '/etc/columnstore/cmapi_server.conf'
+API_CONFIG_PATH = '/etc/columnstore/cmapi_server.conf'
+BYPASS_SM_PATH = '/tmp/columnstore_tmp_files/rdwrscratch/BRM_saves'
 
 
 def get_key():
     cmapi_config = configparser.ConfigParser()
-    try:
+    cmapi_config.read(API_CONFIG_PATH)
-        cmapi_config.read(api_config_file)
-    except FileNotFoundError:
-        return ''
-
     if 'Authentication' not in cmapi_config.sections():
         return ''
     return cmapi_config['Authentication'].get('x-api-key', '')
@@ -45,12 +42,13 @@ if __name__ == '__main__':
         bucket = 'some_bucket'
 
     dbrmroot = config_root.find('./SystemConfig/DBRMRoot').text
+    pmCount = int(config_root.find('./SystemModuleConfig/ModuleCount3').text)
     loadbrm = '/usr/bin/load_brm'
 
     brm_saves_current = ''
 
     if storage.lower() == 's3' and not bucket.lower() == 'some_bucket':
-        # load s3
+        # start SM
         cmd = 'systemctl start mcs-storagemanager'
         retcode = subprocess.call(cmd, shell=True)
         if retcode < 0:
@@ -67,18 +65,20 @@ if __name__ == '__main__':
 
         config_root.find('./SystemConfig/DataFilePlugin').text = "libcloudio.so"
         cs_config.write('/etc/columnstore/Columnstore.xml')
+        # Delay to allow SM to start up
+        time.sleep(15)
 
+    # Single-node on S3
+    if storage.lower() == 's3' and not bucket.lower() == 'some_bucket' and pmCount == 1:
         try:
             brm_saves_current = subprocess.check_output(['smcat', brm])
         except subprocess.CalledProcessError as e:
             # will happen when brm file does not exist
             print('{} does not exist.'.format(brm), file=sys.stderr)
     else:
-        pmCount = int(config_root.find('./SystemModuleConfig/ModuleCount3').text)
         brm = '{}_current'.format(dbrmroot)
-
+        # Multi-node
         if pmCount > 1:
-            # load multinode dbrm
             try:
                 import requests
                 requests.packages.urllib3.disable_warnings()
@@ -105,7 +105,12 @@ api_port, api_version, e)
                     if (r.status_code != 200):
                         raise RuntimeError("Error requesting {} from the primary \
 node.".format(e))
+
+                    # To avoid SM storing BRM files
+                    if storage.lower() == 's3' and bucket.lower() != 'some_bucket':
+                        dbrmroot = BYPASS_SM_PATH
                     current_name = '{}_{}'.format(dbrmroot, e)
+
                     print ("Saving {} to {}".format(e, current_name))
                     path = Path(current_name)
                     path.write_bytes(r.content)

oam/install_scripts/mcs-savebrm.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+import configparser
+
+XML_CONFIG_PATH = '/etc/columnstore/Columnstore.xml'
+SM_CONFIG_PATH = '/etc/columnstore/storagemanager.cnf'
+REST_REQUEST_TO = 10
+
+
+def get_version():
+    return '0.4.0'
+
+
+def get_port():
+    return '8640'
+
+
+if __name__ == '__main__':
+    master_addr = ''
+    pm_count = 0
+    try:
+        cs_config = ET.parse(XML_CONFIG_PATH)
+        config_root = cs_config.getroot()
+        master_addr = config_root.find('./DBRM_Controller/IPAddr').text
+        pm_count = int(config_root.find('./SystemModuleConfig/ModuleCount3').text)
+    except (FileNotFoundError, AttributeError, ValueError) as e:
+        print("Exception had been raised. Continue anyway")
+        print(str(e))
+
+    storage = 'LocalStorage'
+    sm_config = configparser.ConfigParser()
+    files_read = len(sm_config.read(SM_CONFIG_PATH))
+    if files_read == 1:
+        storage = sm_config.get('ObjectStorage', 'service')
+
+    default_addr = '127.0.0.1'   
+    savebrm = 'save_brm'
+    is_primary = False
+
+    # For multi-node with local storage or default installations
+    if (storage.lower() != 's3' and master_addr != default_addr) or \
+master_addr == default_addr:
+        is_primary = True
+        print('Multi-node with local-storage detected.')
+    else:
+        has_requests = False
+        try:
+            import requests
+            requests.packages.urllib3.disable_warnings()
+            has_requests = True
+        except ImportError as e:
+            print('requests Python module does not exist. \
+    Please install CMAPI first.')
+        if has_requests is True:
+            try:
+                print('Requesting for the primary node status.')
+                api_version = get_version()
+                api_port = get_port()
+                url = "https://{}:{}/cmapi/{}/node/primary".format(default_addr, \
+    api_port, api_version)
+                resp = requests.get(url, 
+                                 verify=False,
+                                 timeout=REST_REQUEST_TO)
+                if (resp.status_code != 200):
+                    print("Error sending GET /node/primary.")
+                else:
+                    is_primary = resp.json()['is_primary'] == 'True'
+            except:
+                print('Failed to request.')
+                print(str(e))
+
+    if is_primary is True:
+        try:
+            retcode = subprocess.call(savebrm, shell=True)
+            if retcode < 0:
+                print('{} exits with {}.'.format(cmd, retcode))
+                sys.exit(0)
+        except OSError as e:
+                print(str(e))
+                sys.exit(0)
+
+    sys.exit(0)

oam/install_scripts/mcs-workernode.service.in

@@ -7,6 +7,6 @@ Type=simple
 ExecStart=@ENGINE_BINDIR@/workernode DBRM_Worker1 fg
 Restart=on-failure
 ExecStop=/usr/bin/env bash -c "kill -15 $MAINPID"
-ExecStopPost=-@ENGINE_BINDIR@/save_brm
+ExecStopPost=-@ENGINE_BINDIR@/mcs-savebrm.py
 ExecStopPost=/usr/bin/env bash -c "clearShm > /dev/null 2>&1"
 TimeoutStopSec=120