MCOL-4989: Fix error handling for misconfigured StorageManager startup. (#2385)

oam/install_scripts/mcs-loadbrm.py.in

@@ -98,7 +98,7 @@ if __name__ == '__main__':
         if use_systemd is True:
             cmd = 'systemctl start mcs-storagemanager'
             retcode = subprocess.call(cmd, shell=True)
-            if retcode < 0:
+            if retcode != 0:
                 print('Failed to start storagemanager. \
 {} exits with {}.'.format(cmd, retcode))
                 sys.exit(1)
@@ -219,7 +219,7 @@ brm_saves_current.decode("utf-8").replace("BRM_saves", ""), USER)
 brm_saves_current.decode("utf-8").replace("BRM_saves", ""))
         try:
             retcode = subprocess.call(cmd, shell=True)
-            if retcode < 0:
+            if retcode != 0:
                 print('{} exits with {}.'.format(cmd, retcode))
                 sys.exit(1)
             # systemd services by default works using mysql privileges.
@@ -230,3 +230,9 @@ brm_saves_current.decode("utf-8").replace("BRM_saves", ""))
                     shutil.chown(shm_file, USER, GROUP)
         except OSError as e:
             sys.exit(1)
+    else:
+        if s3_enabled:
+            print('brm_saves_currenty returned empty string from read_from_sm_with_retry')
+        else:
+            print('brm_saves_currenty returned empty string from read_from_disk')
+        sys.exit(1)

storage-manager/src/CloudStorage.cpp

@@ -56,7 +56,18 @@ CloudStorage* CloudStorage::get()
   if (inst)
     return inst;
   if (type == "s3")
-    inst = new S3Storage();
+  {
+    try
+    {
+      inst = new S3Storage();
+    }
+    catch (exception& e)
+    {
+      cout << "S3 Storage Manager Configuration Error:" << endl;
+      cout << e.what() << endl;
+      throw runtime_error("S3Storage: Failed");
+    }
+  }
   else if (type == "local" || type == "localstorage")
     inst = new LocalStorage();
   else

storage-manager/src/S3Storage.cpp

@@ -52,7 +52,7 @@ static size_t WriteCallback(void* contents, size_t size, size_t nmemb, void* use
 inline bool retryable_error(uint8_t s3err)
 {
   return (s3err == MS3_ERR_RESPONSE_PARSE || s3err == MS3_ERR_REQUEST_ERROR || s3err == MS3_ERR_OOM ||
-          s3err == MS3_ERR_IMPOSSIBLE || s3err == MS3_ERR_AUTH || s3err == MS3_ERR_SERVER ||
+          s3err == MS3_ERR_IMPOSSIBLE || s3err == MS3_ERR_SERVER ||
           s3err == MS3_ERR_AUTH_ROLE);
 }
 
@@ -294,6 +294,12 @@ void S3Storage::testConnectivityAndPerms()
   err = deleteObject(testObjKey);
   if (err)
     FAIL(DELETE)
+  err = exists(testObjKey, &_exists);
+  if (err)
+  {
+    logger->log(LOG_CRIT, "S3Storage::exists() failed on nonexistent object. Check 'ListBucket' permissions.");
+    FAIL(HEAD)
+  }
   logger->log(LOG_INFO, "S3Storage: S3 connectivity & permissions are OK");
 }
 

storage-manager/src/smcat.cpp

@@ -87,6 +87,7 @@ void catFileOffline(const char* filename, int prefixlen)
   catch (exception& e)
   {
     cerr << "smcat catFileOffline FAIL: " << e.what() << endl;
+    exit(1);
   }
 }