fix!(cmapi): MCOL-5454: Self-signed certificate autorenew. (#3213)

[add] managers/certificate.py with CertificateManger class
[mv] creating self-signed certificate logic into CertificateManger class
[add] renew and days_before_expire methods to CertificateManger class
[mv] several certificate dependent constants to managers/certificate.py
[add] CherryPy BackgroundTask to invoke certificate check hourly (3600 secs)
[fix] tests
[fix] bug with txn timer clean (clean_txn_by_timeout, worker and invoking of it)

cmapi/cmapi_server/__main__.py

@@ -8,15 +8,10 @@ import logging
 import os
 import threading
 import time
-from datetime import datetime, timedelta
+from datetime import datetime
 
 import cherrypy
 from cherrypy.process import plugins
-from cryptography import x509
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization, hashes
-from cryptography.hazmat.primitives.asymmetric import rsa
-from cryptography.x509.oid import NameOID
 
 # TODO: fix dispatcher choose logic because code executing in endpoints.py
 #       while import process, this cause module logger misconfiguration
@@ -27,28 +22,26 @@ from cmapi_server import helpers
 from cmapi_server.constants import DEFAULT_MCS_CONF_PATH, CMAPI_CONF_PATH
 from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error
 from cmapi_server.failover_agent import FailoverAgent
-from cmapi_server.managers.process import MCSProcessManager
 from cmapi_server.managers.application import AppManager
+from cmapi_server.managers.process import MCSProcessManager
+from cmapi_server.managers.certificate import CertificateManager
 from failover.node_monitor import NodeMonitor
 from mcs_node_control.models.dbrm_socket import SOCK_TIMEOUT, DBRMSocketHandler
 from mcs_node_control.models.node_config import NodeConfig
 
 
-cert_filename = './cmapi_server/self-signed.crt'
+def worker(app):
-
-
-def worker():
     """Background Timer that runs clean_txn_by_timeout() every 5 seconds
     TODO: this needs to be fixed/optimized. I don't like creating the thread
     repeatedly.
     """
     while True:
-        t = threading.Timer(5.0, clean_txn_by_timeout)
+        t = threading.Timer(5.0, clean_txn_by_timeout, app)
         t.start()
         t.join()
 
 
-def clean_txn_by_timeout():
+def clean_txn_by_timeout(app):
     txn_section = app.config.get('txn', None)
     timeout_timestamp = txn_section.get('timeout') if txn_section is not None else None
     current_timestamp = int(datetime.now().timestamp())
@@ -82,7 +75,9 @@ class TxnBackgroundThread(plugins.SimplePlugin):
     def start(self):
         """Plugin entrypoint"""
 
-        self.t = threading.Thread(target=worker, name='TxnBackgroundThread')
+        self.t = threading.Thread(
+            target=worker, name='TxnBackgroundThread', args=(self.app)
+        )
         self.t.daemon = True
         self.t.start()
 
@@ -139,65 +134,14 @@ class FailoverBackgroundThread(plugins.SimplePlugin):
         self._stop()
 
 
-def create_self_signed_certificate():
-    key_filename = './cmapi_server/self-signed.key'
-
-    key = rsa.generate_private_key(
-        public_exponent=65537,
-        key_size=2048,
-        backend=default_backend()
-    )
-
-    with open(key_filename, "wb") as f:
-        f.write(key.private_bytes(
-            encoding=serialization.Encoding.PEM,
-            format=serialization.PrivateFormat.TraditionalOpenSSL,
-            encryption_algorithm=serialization.NoEncryption()),
-        )
-
-    subject = issuer = x509.Name([
-        x509.NameAttribute(NameOID.COUNTRY_NAME, 'US'),
-        x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'California'),
-        x509.NameAttribute(NameOID.LOCALITY_NAME, 'Redwood City'),
-        x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'MariaDB'),
-        x509.NameAttribute(NameOID.COMMON_NAME, 'mariadb.com'),
-    ])
-
-    basic_contraints = x509.BasicConstraints(ca=True, path_length=0)
-
-    cert = x509.CertificateBuilder(
-    ).subject_name(
-        subject
-    ).issuer_name(
-        issuer
-    ).public_key(
-        key.public_key()
-    ).serial_number(
-        x509.random_serial_number()
-    ).not_valid_before(
-        datetime.utcnow()
-    ).not_valid_after(
-        datetime.utcnow() + timedelta(days=365)
-    ).add_extension(
-        basic_contraints,
-        False
-    ).add_extension(
-        x509.SubjectAlternativeName([x509.DNSName('localhost')]),
-        critical=False
-    ).sign(key, hashes.SHA256(), default_backend())
-
-    with open(cert_filename, 'wb') as f:
-        f.write(cert.public_bytes(serialization.Encoding.PEM))
-
-
 if __name__ == '__main__':
     logging.info(f'CMAPI Version: {AppManager.get_version()}')
 
     # TODO: read cmapi config filepath as an argument
     helpers.cmapi_config_check()
 
-    if not os.path.exists(cert_filename):
+    CertificateManager.create_self_signed_certificate_if_not_exist()
-        create_self_signed_certificate()
+    CertificateManager.renew_certificate()
 
     app = cherrypy.tree.mount(root=None, config=CMAPI_CONF_PATH)
     app.config.update({
@@ -224,6 +168,10 @@ if __name__ == '__main__':
     # subscribe FailoverBackgroundThread plugin code to bus channels
     # code below not starting "real" failover background thread
     FailoverBackgroundThread(cherrypy.engine, turn_on_failover).subscribe()
+    cherrypy.engine.certificate_monitor = plugins.BackgroundTask(
+        3600, CertificateManager.renew_certificate
+    )
+    cherrypy.engine.certificate_monitor.start()
     cherrypy.engine.start()
     cherrypy.engine.wait(cherrypy.engine.states.STARTED)
 

cmapi/cmapi_server/managers/certificate.py

@@ -0,0 +1,107 @@
+"""Module related to CMAPI self-signed certificate management logic."""
+import os
+import logging
+from datetime import datetime, timedelta, timezone
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization, hashes
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+
+
+CERT_FILENAME = './cmapi_server/self-signed.crt'
+KEY_FILENAME = './cmapi_server/self-signed.key'
+CERT_DAYS = 365
+
+
+class CertificateManager():
+    """Class with methods to manage self-signed certificate."""
+
+    @staticmethod
+    def create_self_signed_certificate() -> None:
+        """Create self-signed certificate."""
+        key = rsa.generate_private_key(
+            public_exponent=65537,
+            key_size=2048,
+            backend=default_backend()
+        )
+
+        with open(KEY_FILENAME, "wb") as f:
+            f.write(key.private_bytes(
+                encoding=serialization.Encoding.PEM,
+                format=serialization.PrivateFormat.TraditionalOpenSSL,
+                encryption_algorithm=serialization.NoEncryption()),
+            )
+
+        subject = issuer = x509.Name([
+            x509.NameAttribute(NameOID.COUNTRY_NAME, 'US'),
+            x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'California'),
+            x509.NameAttribute(NameOID.LOCALITY_NAME, 'Redwood City'),
+            x509.NameAttribute(
+                NameOID.ORGANIZATION_NAME, 'MariaDB Corporation'
+            ),
+            x509.NameAttribute(NameOID.COMMON_NAME, 'mariadb.com'),
+        ])
+
+        basic_contraints = x509.BasicConstraints(ca=True, path_length=0)
+
+        cert = x509.CertificateBuilder(
+        ).subject_name(
+            subject
+        ).issuer_name(
+            issuer
+        ).public_key(
+            key.public_key()
+        ).serial_number(
+            x509.random_serial_number()
+        ).not_valid_before(
+            datetime.now(timezone.utc)
+        ).not_valid_after(
+            datetime.now(timezone.utc) + timedelta(days=CERT_DAYS)
+        ).add_extension(
+            basic_contraints,
+            False
+        ).add_extension(
+            x509.SubjectAlternativeName([x509.DNSName('localhost')]),
+            critical=False
+        ).sign(key, hashes.SHA256(), default_backend())
+
+        with open(CERT_FILENAME, 'wb') as f:
+            f.write(cert.public_bytes(serialization.Encoding.PEM))
+        logging.info('Created self signed sertificate for CMAPI API access.')
+    
+    @staticmethod
+    def create_self_signed_certificate_if_not_exist() -> None:
+        """Create self-signed certificate if not exist."""
+        if not os.path.exists(CERT_FILENAME):
+            CertificateManager.create_self_signed_certificate()
+
+    @staticmethod
+    def days_before_expire() -> int:
+        """Calculates how many days before expiration left.
+
+        Creates self-signed cetificate if certificate doesn't exist.
+
+        :return: days left
+        :rtype: int
+        """
+        CertificateManager.create_self_signed_certificate_if_not_exist()
+        with open(CERT_FILENAME, 'rb') as cert_file:
+            cert_data = cert_file.read()
+        cert = x509.load_pem_x509_certificate(cert_data)
+        days_before_expire = (cert.not_valid_after - datetime.now()).days
+        return days_before_expire
+
+    @staticmethod
+    def renew_certificate() -> None:
+        """Creates new self signed certificate.
+        
+        Creates self-signed cetificate if certificate doesn't exist or
+        expires in a 1 day or less.
+        """
+        if CertificateManager.days_before_expire() <= 0:
+            logging.warning(
+                'Self signed certificate nearly to expire. Renewing.'
+            )
+            CertificateManager.create_self_signed_certificate()

cmapi/cmapi_server/test/test_em_endpoints.py

@@ -16,17 +16,16 @@ from cmapi_server.constants import (
 from cmapi_server.controllers.dispatcher import (
     dispatcher, jsonify_error,_version
 )
+from cmapi_server.managers.certificate import CertificateManager
 from cmapi_server.test.unittest_global import (
-    create_self_signed_certificate, cert_filename, cmapi_config_filename,
+    cmapi_config_filename, tmp_cmapi_config_filename
-    tmp_cmapi_config_filename
 )
 from mcs_node_control.models.node_config import NodeConfig
 
 
 @contextmanager
 def run_server():
-    if not path.exists(cert_filename):
+    CertificateManager.create_self_signed_certificate_if_not_exist()
-        create_self_signed_certificate()
     cherrypy.engine.start()
     cherrypy.engine.wait(cherrypy.engine.states.STARTED)
     yield

cmapi/cmapi_server/test/test_txns.py

@@ -8,15 +8,16 @@ from contextlib import contextmanager
 from cmapi_server import helpers, node_manipulation
 from mcs_node_control.models.node_config import NodeConfig
 from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error
-from cmapi_server.test.unittest_global import create_self_signed_certificate, \
+from cmapi_server.managers.certificate import CertificateManager
-        cert_filename, mcs_config_filename, cmapi_config_filename, \
+from cmapi_server.test.unittest_global import (
-        tmp_mcs_config_filename, tmp_cmapi_config_filename
+    mcs_config_filename, cmapi_config_filename, tmp_mcs_config_filename, 
+    tmp_cmapi_config_filename,
+)
 
 
 @contextmanager
 def start_server():
-    if not os.path.exists(cert_filename):
+    CertificateManager.create_self_signed_certificate_if_not_exist()
-        create_self_signed_certificate()
 
     app = cherrypy.tree.mount(root = None, config = cmapi_config_filename)
     app.config.update({

cmapi/cmapi_server/test/unittest_global.py

@@ -18,10 +18,10 @@ from cmapi_server import helpers
 from cmapi_server.constants import CMAPI_CONF_PATH
 from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error
 from cmapi_server.managers.process import MCSProcessManager
+from cmapi_server.managers.certificate import CertificateManager
 
 
 TEST_API_KEY = 'somekey123'
-cert_filename = './cmapi_server/self-signed.crt'
 MCS_CONFIG_FILEPATH = '/etc/columnstore/Columnstore.xml'
 COPY_MCS_CONFIG_FILEPATH = './cmapi_server/test/original_Columnstore.xml'
 TEST_MCS_CONFIG_FILEPATH = './cmapi_server/test/CS-config-test.xml'
@@ -42,57 +42,6 @@ SYSTEMCTL = 'sudo systemctl'
 logging.basicConfig(level=logging.DEBUG)
 
 
-def create_self_signed_certificate():
-    key_filename = './cmapi_server/self-signed.key'
-
-    key = rsa.generate_private_key(
-        public_exponent=65537,
-        key_size=2048,
-        backend=default_backend()
-    )
-
-    with open(key_filename, "wb") as f:
-        f.write(key.private_bytes(
-            encoding=serialization.Encoding.PEM,
-            format=serialization.PrivateFormat.TraditionalOpenSSL,
-            encryption_algorithm=serialization.NoEncryption()),
-        )
-
-    subject = issuer = x509.Name([
-        x509.NameAttribute(NameOID.COUNTRY_NAME, u"US"),
-        x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u"California"),
-        x509.NameAttribute(NameOID.LOCALITY_NAME, u"Redwood City"),
-        x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"MariaDB"),
-        x509.NameAttribute(NameOID.COMMON_NAME, u"mariadb.com"),
-    ])
-
-    basic_contraints = x509.BasicConstraints(ca=True, path_length=0)
-
-    cert = x509.CertificateBuilder(
-    ).subject_name(
-        subject
-    ).issuer_name(
-        issuer
-    ).public_key(
-        key.public_key()
-    ).serial_number(
-        x509.random_serial_number()
-    ).not_valid_before(
-        datetime.utcnow()
-    ).not_valid_after(
-        datetime.utcnow() + timedelta(days=365)
-    ).add_extension(
-        basic_contraints,
-        False
-    ).add_extension(
-        x509.SubjectAlternativeName([x509.DNSName(u"localhost")]),
-        critical=False
-    ).sign(key, hashes.SHA256(), default_backend())
-
-    with open(cert_filename, "wb") as f:
-        f.write(cert.public_bytes(serialization.Encoding.PEM))
-
-
 def run_detect_processes():
     cfg_parser = helpers.get_config_parser(CMAPI_CONF_PATH)
     d_name, d_path = helpers.get_dispatcher_name_and_path(cfg_parser)
@@ -101,8 +50,7 @@ def run_detect_processes():
 
 @contextmanager
 def run_server():
-    if not os.path.exists(cert_filename):
+    CertificateManager.create_self_signed_certificate_if_not_exist()
-        create_self_signed_certificate()
 
     cherrypy.engine.start()
     cherrypy.engine.wait(cherrypy.engine.states.STARTED)

cmapi/failover/test/test_agent_comm.py

@@ -6,18 +6,19 @@ import cherrypy
 import os.path
 from contextlib import contextmanager
 from ..agent_comm import AgentComm
+from cmapi_server import helpers, node_manipulation
 from cmapi_server.failover_agent import FailoverAgent
 from mcs_node_control.models.node_config import NodeConfig
 from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error
-from cmapi_server.test.unittest_global import create_self_signed_certificate, cert_filename
+from cmapi_server.managers.certificate import CertificateManager
-from cmapi_server import helpers, node_manipulation
+
 
 config_filename = './cmapi_server/cmapi_server.conf'
 
+
 @contextmanager
 def start_server():
-    if not os.path.exists(cert_filename):
+    CertificateManager.create_self_signed_certificate_if_not_exist()
-        create_self_signed_certificate()
 
     app = cherrypy.tree.mount(root = None, config = config_filename)
     app.config.update({