PROJQUAY-1376 - Handle non 200 api response from executors (#619)

* Handle non 200 api response from executors

* Allows the CA cert to be specified in the config for server verification

Allow the CA cert used for server verification to be specified in the
config even if client certificate authentication is not used.
Handles non-200 responses from executors when trying to get worker count.

buildman/manager/ephemeral.py

@@ -583,13 +583,17 @@ class EphemeralBuilderManager(BuildStateInterface):
         logger.debug("Scheduling build %s", build_id)
 
         allowed_worker_count = self._manager_config.get("ALLOWED_WORKER_COUNT", 1)
-        if self._running_workers() >= allowed_worker_count:
-            logger.warning(
-                "Could not schedule build %s. Number of workers at capacity: %s.",
-                build_id,
-                self._running_workers(),
-            )
+        try:
+            if self._running_workers() >= allowed_worker_count:
+                logger.warning(
+                    "Could not schedule build %s. Number of workers at capacity: %s.",
+                    build_id,
+                    self._running_workers(),
+                )
             return False, TOO_MANY_WORKERS_SLEEP_DURATION
+        except Exception as exe:
+            logger.warning("Failed to get worker count from executors: %s", exe)
+            return False, EPHEMERAL_API_TIMEOUT
 
         job_id = self._job_key(build_id)
         try:

buildman/manager/executor.py

@@ -215,10 +215,14 @@ class EC2Executor(BuilderExecutor):
 
     @property
     def running_builders_count(self):
-        ec2_conn = self._get_conn()
-        resp = ec2_conn.describe_instances(
-            Filters=[{"Name": "tag:Name", "Values": ["Quay Ephemeral Builder"]}]
-        )
+        try:
+            ec2_conn = self._get_conn()
+            resp = ec2_conn.describe_instances(
+                Filters=[{"Name": "tag:Name", "Values": ["Quay Ephemeral Builder"]}]
+            )
+        except Exception as ec2e:
+            logger.error("EC2 executor error: %s", ec2e)
+            raise ExecutorException(ec2e)
 
         count = 0
         for reservation in resp["Reservations"]:
@@ -402,6 +406,19 @@ class KubernetesExecutor(BuilderExecutor):
     def running_builders_count(self):
         q = {"labelSelector": "build,time,manager,quay-sha"}
         jobs_list = self._request("GET", self._jobs_path(), params=q)
+        if jobs_list.status_code != 200:
+            logger.error(
+                "Kubernetes executor request error: %s %s - %s",
+                "GET",
+                jobs_list.url,
+                jobs_list.status_code,
+            )
+            raise ExecutorException(
+                "Failed to get runnning builder count from executor %s: %s %s",
+                self.name,
+                jobs_list.status_code,
+                jobs_list.reason,
+            )
         return len(jobs_list.json()["items"])
 
     def _request(self, method, path, **kwargs):
@@ -412,6 +429,9 @@ class KubernetesExecutor(BuilderExecutor):
         tls_ca = self.executor_config.get("K8S_API_TLS_CA")
         service_account_token = self.executor_config.get("SERVICE_ACCOUNT_TOKEN")
 
+        if tls_ca:
+            request_options["verify"] = tls_ca
+
         if "timeout" not in request_options:
             request_options["timeout"] = self.executor_config.get("K8S_API_TIMEOUT", 20)
 
@@ -423,8 +443,6 @@ class KubernetesExecutor(BuilderExecutor):
             scheme = "https"
             request_options["cert"] = (tls_cert, tls_key)
             logger.debug("Using tls certificate and key for Kubernetes authentication")
-            if tls_ca:
-                request_options["verify"] = tls_ca
         else:
             scheme = "http"