From a00711cc163eff7f073fef9983684a27c7fc0800 Mon Sep 17 00:00:00 2001 From: Alan Mologorsky <89034356+mariadb-AlanMologorsky@users.noreply.github.com> Date: Sun, 21 Jul 2024 07:25:32 +0300 Subject: [PATCH] fix!(cmapi): MCOL-5454: Self-signed certificate autorenew. (#3213) [add] managers/certificate.py with CertificateManger class [mv] creating self-signed certificate logic into CertificateManger class [add] renew and days_before_expire methods to CertificateManger class [mv] several certificate dependent constants to managers/certificate.py [add] CherryPy BackgroundTask to invoke certificate check hourly (3600 secs) [fix] tests [fix] bug with txn timer clean (clean_txn_by_timeout, worker and invoking of it) --- cmapi/cmapi_server/__main__.py | 82 +++----------- cmapi/cmapi_server/managers/certificate.py | 107 +++++++++++++++++++ cmapi/cmapi_server/test/test_em_endpoints.py | 7 +- cmapi/cmapi_server/test/test_txns.py | 11 +- cmapi/cmapi_server/test/unittest_global.py | 56 +--------- cmapi/failover/test/test_agent_comm.py | 9 +- 6 files changed, 138 insertions(+), 134 deletions(-) create mode 100644 cmapi/cmapi_server/managers/certificate.py diff --git a/cmapi/cmapi_server/__main__.py b/cmapi/cmapi_server/__main__.py index e5bf5b753..70877c208 100644 --- a/cmapi/cmapi_server/__main__.py +++ b/cmapi/cmapi_server/__main__.py @@ -8,15 +8,10 @@ import logging import os import threading import time -from datetime import datetime, timedelta +from datetime import datetime import cherrypy from cherrypy.process import plugins -from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization, hashes -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import NameOID # TODO: fix dispatcher choose logic because code executing in endpoints.py # while import process, this cause module logger misconfiguration @@ -27,28 +22,26 @@ from cmapi_server import helpers from cmapi_server.constants import DEFAULT_MCS_CONF_PATH, CMAPI_CONF_PATH from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error from cmapi_server.failover_agent import FailoverAgent -from cmapi_server.managers.process import MCSProcessManager from cmapi_server.managers.application import AppManager +from cmapi_server.managers.process import MCSProcessManager +from cmapi_server.managers.certificate import CertificateManager from failover.node_monitor import NodeMonitor from mcs_node_control.models.dbrm_socket import SOCK_TIMEOUT, DBRMSocketHandler from mcs_node_control.models.node_config import NodeConfig -cert_filename = './cmapi_server/self-signed.crt' - - -def worker(): +def worker(app): """Background Timer that runs clean_txn_by_timeout() every 5 seconds TODO: this needs to be fixed/optimized. I don't like creating the thread repeatedly. """ while True: - t = threading.Timer(5.0, clean_txn_by_timeout) + t = threading.Timer(5.0, clean_txn_by_timeout, app) t.start() t.join() -def clean_txn_by_timeout(): +def clean_txn_by_timeout(app): txn_section = app.config.get('txn', None) timeout_timestamp = txn_section.get('timeout') if txn_section is not None else None current_timestamp = int(datetime.now().timestamp()) @@ -82,7 +75,9 @@ class TxnBackgroundThread(plugins.SimplePlugin): def start(self): """Plugin entrypoint""" - self.t = threading.Thread(target=worker, name='TxnBackgroundThread') + self.t = threading.Thread( + target=worker, name='TxnBackgroundThread', args=(self.app) + ) self.t.daemon = True self.t.start() @@ -139,65 +134,14 @@ class FailoverBackgroundThread(plugins.SimplePlugin): self._stop() -def create_self_signed_certificate(): - key_filename = './cmapi_server/self-signed.key' - - key = rsa.generate_private_key( - public_exponent=65537, - key_size=2048, - backend=default_backend() - ) - - with open(key_filename, "wb") as f: - f.write(key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption()), - ) - - subject = issuer = x509.Name([ - x509.NameAttribute(NameOID.COUNTRY_NAME, 'US'), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'California'), - x509.NameAttribute(NameOID.LOCALITY_NAME, 'Redwood City'), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'MariaDB'), - x509.NameAttribute(NameOID.COMMON_NAME, 'mariadb.com'), - ]) - - basic_contraints = x509.BasicConstraints(ca=True, path_length=0) - - cert = x509.CertificateBuilder( - ).subject_name( - subject - ).issuer_name( - issuer - ).public_key( - key.public_key() - ).serial_number( - x509.random_serial_number() - ).not_valid_before( - datetime.utcnow() - ).not_valid_after( - datetime.utcnow() + timedelta(days=365) - ).add_extension( - basic_contraints, - False - ).add_extension( - x509.SubjectAlternativeName([x509.DNSName('localhost')]), - critical=False - ).sign(key, hashes.SHA256(), default_backend()) - - with open(cert_filename, 'wb') as f: - f.write(cert.public_bytes(serialization.Encoding.PEM)) - - if __name__ == '__main__': logging.info(f'CMAPI Version: {AppManager.get_version()}') # TODO: read cmapi config filepath as an argument helpers.cmapi_config_check() - if not os.path.exists(cert_filename): - create_self_signed_certificate() + CertificateManager.create_self_signed_certificate_if_not_exist() + CertificateManager.renew_certificate() app = cherrypy.tree.mount(root=None, config=CMAPI_CONF_PATH) app.config.update({ @@ -224,6 +168,10 @@ if __name__ == '__main__': # subscribe FailoverBackgroundThread plugin code to bus channels # code below not starting "real" failover background thread FailoverBackgroundThread(cherrypy.engine, turn_on_failover).subscribe() + cherrypy.engine.certificate_monitor = plugins.BackgroundTask( + 3600, CertificateManager.renew_certificate + ) + cherrypy.engine.certificate_monitor.start() cherrypy.engine.start() cherrypy.engine.wait(cherrypy.engine.states.STARTED) diff --git a/cmapi/cmapi_server/managers/certificate.py b/cmapi/cmapi_server/managers/certificate.py new file mode 100644 index 000000000..1665bc935 --- /dev/null +++ b/cmapi/cmapi_server/managers/certificate.py @@ -0,0 +1,107 @@ +"""Module related to CMAPI self-signed certificate management logic.""" +import os +import logging +from datetime import datetime, timedelta, timezone + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization, hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID + + +CERT_FILENAME = './cmapi_server/self-signed.crt' +KEY_FILENAME = './cmapi_server/self-signed.key' +CERT_DAYS = 365 + + +class CertificateManager(): + """Class with methods to manage self-signed certificate.""" + + @staticmethod + def create_self_signed_certificate() -> None: + """Create self-signed certificate.""" + key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend() + ) + + with open(KEY_FILENAME, "wb") as f: + f.write(key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption()), + ) + + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, 'US'), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'California'), + x509.NameAttribute(NameOID.LOCALITY_NAME, 'Redwood City'), + x509.NameAttribute( + NameOID.ORGANIZATION_NAME, 'MariaDB Corporation' + ), + x509.NameAttribute(NameOID.COMMON_NAME, 'mariadb.com'), + ]) + + basic_contraints = x509.BasicConstraints(ca=True, path_length=0) + + cert = x509.CertificateBuilder( + ).subject_name( + subject + ).issuer_name( + issuer + ).public_key( + key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.now(timezone.utc) + ).not_valid_after( + datetime.now(timezone.utc) + timedelta(days=CERT_DAYS) + ).add_extension( + basic_contraints, + False + ).add_extension( + x509.SubjectAlternativeName([x509.DNSName('localhost')]), + critical=False + ).sign(key, hashes.SHA256(), default_backend()) + + with open(CERT_FILENAME, 'wb') as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + logging.info('Created self signed sertificate for CMAPI API access.') + + @staticmethod + def create_self_signed_certificate_if_not_exist() -> None: + """Create self-signed certificate if not exist.""" + if not os.path.exists(CERT_FILENAME): + CertificateManager.create_self_signed_certificate() + + @staticmethod + def days_before_expire() -> int: + """Calculates how many days before expiration left. + + Creates self-signed cetificate if certificate doesn't exist. + + :return: days left + :rtype: int + """ + CertificateManager.create_self_signed_certificate_if_not_exist() + with open(CERT_FILENAME, 'rb') as cert_file: + cert_data = cert_file.read() + cert = x509.load_pem_x509_certificate(cert_data) + days_before_expire = (cert.not_valid_after - datetime.now()).days + return days_before_expire + + @staticmethod + def renew_certificate() -> None: + """Creates new self signed certificate. + + Creates self-signed cetificate if certificate doesn't exist or + expires in a 1 day or less. + """ + if CertificateManager.days_before_expire() <= 0: + logging.warning( + 'Self signed certificate nearly to expire. Renewing.' + ) + CertificateManager.create_self_signed_certificate() diff --git a/cmapi/cmapi_server/test/test_em_endpoints.py b/cmapi/cmapi_server/test/test_em_endpoints.py index d6b92fd60..a166cc6e8 100644 --- a/cmapi/cmapi_server/test/test_em_endpoints.py +++ b/cmapi/cmapi_server/test/test_em_endpoints.py @@ -16,17 +16,16 @@ from cmapi_server.constants import ( from cmapi_server.controllers.dispatcher import ( dispatcher, jsonify_error,_version ) +from cmapi_server.managers.certificate import CertificateManager from cmapi_server.test.unittest_global import ( - create_self_signed_certificate, cert_filename, cmapi_config_filename, - tmp_cmapi_config_filename + cmapi_config_filename, tmp_cmapi_config_filename ) from mcs_node_control.models.node_config import NodeConfig @contextmanager def run_server(): - if not path.exists(cert_filename): - create_self_signed_certificate() + CertificateManager.create_self_signed_certificate_if_not_exist() cherrypy.engine.start() cherrypy.engine.wait(cherrypy.engine.states.STARTED) yield diff --git a/cmapi/cmapi_server/test/test_txns.py b/cmapi/cmapi_server/test/test_txns.py index a2f48f79c..01b2232db 100644 --- a/cmapi/cmapi_server/test/test_txns.py +++ b/cmapi/cmapi_server/test/test_txns.py @@ -8,15 +8,16 @@ from contextlib import contextmanager from cmapi_server import helpers, node_manipulation from mcs_node_control.models.node_config import NodeConfig from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error -from cmapi_server.test.unittest_global import create_self_signed_certificate, \ - cert_filename, mcs_config_filename, cmapi_config_filename, \ - tmp_mcs_config_filename, tmp_cmapi_config_filename +from cmapi_server.managers.certificate import CertificateManager +from cmapi_server.test.unittest_global import ( + mcs_config_filename, cmapi_config_filename, tmp_mcs_config_filename, + tmp_cmapi_config_filename, +) @contextmanager def start_server(): - if not os.path.exists(cert_filename): - create_self_signed_certificate() + CertificateManager.create_self_signed_certificate_if_not_exist() app = cherrypy.tree.mount(root = None, config = cmapi_config_filename) app.config.update({ diff --git a/cmapi/cmapi_server/test/unittest_global.py b/cmapi/cmapi_server/test/unittest_global.py index 8e1fad672..57ec6c749 100644 --- a/cmapi/cmapi_server/test/unittest_global.py +++ b/cmapi/cmapi_server/test/unittest_global.py @@ -18,10 +18,10 @@ from cmapi_server import helpers from cmapi_server.constants import CMAPI_CONF_PATH from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error from cmapi_server.managers.process import MCSProcessManager +from cmapi_server.managers.certificate import CertificateManager TEST_API_KEY = 'somekey123' -cert_filename = './cmapi_server/self-signed.crt' MCS_CONFIG_FILEPATH = '/etc/columnstore/Columnstore.xml' COPY_MCS_CONFIG_FILEPATH = './cmapi_server/test/original_Columnstore.xml' TEST_MCS_CONFIG_FILEPATH = './cmapi_server/test/CS-config-test.xml' @@ -42,57 +42,6 @@ SYSTEMCTL = 'sudo systemctl' logging.basicConfig(level=logging.DEBUG) -def create_self_signed_certificate(): - key_filename = './cmapi_server/self-signed.key' - - key = rsa.generate_private_key( - public_exponent=65537, - key_size=2048, - backend=default_backend() - ) - - with open(key_filename, "wb") as f: - f.write(key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption()), - ) - - subject = issuer = x509.Name([ - x509.NameAttribute(NameOID.COUNTRY_NAME, u"US"), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u"California"), - x509.NameAttribute(NameOID.LOCALITY_NAME, u"Redwood City"), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"MariaDB"), - x509.NameAttribute(NameOID.COMMON_NAME, u"mariadb.com"), - ]) - - basic_contraints = x509.BasicConstraints(ca=True, path_length=0) - - cert = x509.CertificateBuilder( - ).subject_name( - subject - ).issuer_name( - issuer - ).public_key( - key.public_key() - ).serial_number( - x509.random_serial_number() - ).not_valid_before( - datetime.utcnow() - ).not_valid_after( - datetime.utcnow() + timedelta(days=365) - ).add_extension( - basic_contraints, - False - ).add_extension( - x509.SubjectAlternativeName([x509.DNSName(u"localhost")]), - critical=False - ).sign(key, hashes.SHA256(), default_backend()) - - with open(cert_filename, "wb") as f: - f.write(cert.public_bytes(serialization.Encoding.PEM)) - - def run_detect_processes(): cfg_parser = helpers.get_config_parser(CMAPI_CONF_PATH) d_name, d_path = helpers.get_dispatcher_name_and_path(cfg_parser) @@ -101,8 +50,7 @@ def run_detect_processes(): @contextmanager def run_server(): - if not os.path.exists(cert_filename): - create_self_signed_certificate() + CertificateManager.create_self_signed_certificate_if_not_exist() cherrypy.engine.start() cherrypy.engine.wait(cherrypy.engine.states.STARTED) diff --git a/cmapi/failover/test/test_agent_comm.py b/cmapi/failover/test/test_agent_comm.py index b4d21f114..3ec34557e 100644 --- a/cmapi/failover/test/test_agent_comm.py +++ b/cmapi/failover/test/test_agent_comm.py @@ -6,18 +6,19 @@ import cherrypy import os.path from contextlib import contextmanager from ..agent_comm import AgentComm +from cmapi_server import helpers, node_manipulation from cmapi_server.failover_agent import FailoverAgent from mcs_node_control.models.node_config import NodeConfig from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error -from cmapi_server.test.unittest_global import create_self_signed_certificate, cert_filename -from cmapi_server import helpers, node_manipulation +from cmapi_server.managers.certificate import CertificateManager + config_filename = './cmapi_server/cmapi_server.conf' + @contextmanager def start_server(): - if not os.path.exists(cert_filename): - create_self_signed_certificate() + CertificateManager.create_self_signed_certificate_if_not_exist() app = cherrypy.tree.mount(root = None, config = config_filename) app.config.update({