Added support for Sentry in cmapi server

Support distributed request tracing

-Direct dependencies now in requirements[-dev].in, pip-compile generates full requirement[-dev].txt from them

cmapi/README.md

@@ -2,13 +2,13 @@
 [![Build Status](https://ci.columnstore.mariadb.net/api/badges/mariadb-corporation/mariadb-columnstore-cmapi/status.svg)](https://ci.columnstore.mariadb.net/mariadb-corporation/mariadb-columnstore-cmapi)
 
 ## Overview
-This RESTfull server enables multi-node setups for MCS.
+This RESTful server enables multi-node setups for MCS.
 
 ## Requirements
 
 See requirements.txt file.
 
-All the Python packages prerequisits are shipped with a pre-built Python enterpreter.
+All the Python packages prerequisites are shipped with a pre-built Python interpreter.
 
 ## Usage
 

cmapi/cmapi_server/__main__.py

@@ -16,6 +16,7 @@ from cherrypy.process import plugins
 # TODO: fix dispatcher choose logic because code executing in endpoints.py
 #       while import process, this cause module logger misconfiguration
 from cmapi_server.logging_management import config_cmapi_server_logging
+from cmapi_server.sentry import maybe_init_sentry, register_sentry_cherrypy_tool
 config_cmapi_server_logging()
 
 from cmapi_server import helpers
@@ -140,15 +141,24 @@ if __name__ == '__main__':
     # TODO: read cmapi config filepath as an argument
     helpers.cmapi_config_check()
 
+    # Init Sentry if DSN is present
+    sentry_active = maybe_init_sentry()
+    if sentry_active:
+        register_sentry_cherrypy_tool()
+
     CertificateManager.create_self_signed_certificate_if_not_exist()
     CertificateManager.renew_certificate()
 
     app = cherrypy.tree.mount(root=None, config=CMAPI_CONF_PATH)
+    root_config = {
+        "request.dispatch": dispatcher,
+        "error_page.default": jsonify_error,
+    }
+    if sentry_active:
+        root_config["tools.sentry.on"] = True
+
     app.config.update({
-        '/': {
+        '/': root_config,
-            'request.dispatch': dispatcher,
-            'error_page.default': jsonify_error,
-        },
         'config': {
             'path': CMAPI_CONF_PATH,
         },

cmapi/cmapi_server/sentry.py

@@ -0,0 +1,197 @@
+import logging
+import socket
+
+import cherrypy
+import sentry_sdk
+from sentry_sdk.integrations.aiohttp import AioHttpIntegration
+from sentry_sdk.integrations.logging import LoggingIntegration
+
+from cmapi_server import helpers
+from cmapi_server.constants import CMAPI_CONF_PATH
+
+SENTRY_ACTIVE = False
+
+logger = logging.getLogger(__name__)
+
+def maybe_init_sentry() -> bool:
+    """Initialize Sentry from CMAPI configuration.
+
+    Reads config and initializes Sentry only if dsn parameter is present in corresponding section.
+    The initialization enables the following integrations:
+    - LoggingIntegration: capture warning-level logs as Sentry events and use
+      lower-level logs as breadcrumbs.
+    - AioHttpIntegration: propagate trace headers for outbound requests made
+      with `aiohttp`.
+
+    The function is a no-op if the DSN is missing.
+
+    Returns: True if Sentry is initialized, False otherwise.
+    """
+    global SENTRY_ACTIVE
+    try:
+        cfg_parser = helpers.get_config_parser(CMAPI_CONF_PATH)
+        dsn = helpers.dequote(
+            cfg_parser.get('Sentry', 'dsn', fallback='').strip()
+        )
+        if not dsn:
+            return False
+
+        environment = helpers.dequote(
+            cfg_parser.get('Sentry', 'environment', fallback='development').strip()
+        )
+        traces_sample_rate_str = helpers.dequote(
+            cfg_parser.get('Sentry', 'traces_sample_rate', fallback='1.0').strip()
+        )
+    except Exception:
+        logger.exception('Failed to initialize Sentry.')
+        return False
+
+    try:
+        sentry_logging = LoggingIntegration(
+            level=logging.INFO,
+            event_level=logging.WARNING,
+        )
+
+        try:
+            traces_sample_rate = float(traces_sample_rate_str)
+        except ValueError:
+            logger.error('Invalid traces_sample_rate: %s', traces_sample_rate_str)
+            traces_sample_rate = 1.0
+
+        sentry_sdk.init(
+            dsn=dsn,
+            environment=environment,
+            traces_sample_rate=traces_sample_rate,
+            integrations=[sentry_logging, AioHttpIntegration()],
+        )
+        SENTRY_ACTIVE = True
+        logger.info('Sentry initialized for CMAPI via config.')
+    except Exception:
+        logger.exception('Failed to initialize Sentry.')
+        return False
+
+    logger.info('Sentry successfully initialized.')
+    return True
+
+def _sentry_on_start_resource():
+    """Start or continue a Sentry transaction for the current CherryPy request.
+
+    - Continues an incoming distributed trace using Sentry trace headers if
+      present; otherwise starts a new transaction with `op='http.server'`.
+    - Pushes the transaction into the current Sentry scope and attaches useful
+      request metadata as tags and context (HTTP method, path, client IP,
+      hostname, request ID, and a filtered subset of headers).
+    - Stores the transaction on the CherryPy request object for later finishing
+      in `_sentry_on_end_request`.
+    """
+    if not SENTRY_ACTIVE:
+        return
+    try:
+        request = cherrypy.request
+        headers = dict(getattr(request, 'headers', {}) or {})
+        name = f"{request.method} {request.path_info}"
+        transaction = sentry_sdk.start_transaction(
+            op='http.server', name=name, continue_from_headers=headers
+        )
+        sentry_sdk.Hub.current.scope.set_span(transaction)
+
+        # Add request-level context/tags
+        scope = sentry_sdk.Hub.current.scope
+        scope.set_tag('http.method', request.method)
+        scope.set_tag('http.path', request.path_info)
+        scope.set_tag('client.ip', getattr(request.remote, 'ip', ''))
+        scope.set_tag('instance.hostname', socket.gethostname())
+        request_id = getattr(request, 'unique_id', None)
+        if request_id:
+            scope.set_tag('request.id', request_id)
+        # Optionally add headers as context without sensitive values
+        safe_headers = {k: v for k, v in headers.items()
+                        if k.lower() not in {'authorization', 'x-api-key'}}
+        scope.set_context('headers', safe_headers)
+
+        request.sentry_transaction = transaction
+    except Exception:
+        logger.exception('Failed to start Sentry transaction.')
+
+
+def _sentry_before_error_response():
+    """Capture the current exception (if any) to Sentry before error response.
+
+    This hook runs when CherryPy prepares an error response. If an exception is
+    available in the current context, it will be sent to Sentry.
+    """
+    if not SENTRY_ACTIVE:
+        return
+    try:
+        sentry_sdk.capture_exception()
+    except Exception:
+        logger.exception('Failed to capture exception to Sentry.')
+
+
+def _sentry_on_end_request():
+    """Finish the Sentry transaction for the current CherryPy request.
+
+    Attempts to set the HTTP status code on the active transaction and then
+    finishes it. If no transaction was started on this request, the function is
+    a no-op.
+    """
+    if not SENTRY_ACTIVE:
+        return
+    try:
+        request = cherrypy.request
+        transaction = getattr(request, 'sentry_transaction', None)
+        if transaction is None:
+            return
+        status = cherrypy.response.status
+        try:
+            status_code = int(str(status).split()[0])
+        except Exception:
+            status_code = None
+        try:
+            if status_code is not None and hasattr(transaction, 'set_http_status'):
+                transaction.set_http_status(status_code)
+        except Exception:
+            logger.exception('Failed to set HTTP status code on Sentry transaction.')
+        transaction.finish()
+    except Exception:
+        logger.exception('Failed to finish Sentry transaction.')
+
+
+class SentryTool(cherrypy.Tool):
+    """CherryPy Tool that wires Sentry request lifecycle hooks.
+
+    The tool attaches handlers for `on_start_resource`, `before_error_response`,
+    and `on_end_request` in order to manage Sentry transactions and error
+    capture across the request lifecycle.
+    """
+    def __init__(self):
+        cherrypy.Tool.__init__(self, 'on_start_resource', self._tool_callback, priority=50)
+
+    @staticmethod
+    def _tool_callback():
+        """Attach Sentry lifecycle callbacks to the current CherryPy request."""
+        cherrypy.request.hooks.attach(
+            'on_start_resource', _sentry_on_start_resource, priority=50
+        )
+        cherrypy.request.hooks.attach(
+            'before_error_response', _sentry_before_error_response, priority=60
+        )
+        cherrypy.request.hooks.attach(
+            'on_end_request', _sentry_on_end_request, priority=70
+        )
+
+
+def register_sentry_cherrypy_tool() -> None:
+    """Register the Sentry CherryPy tool under `tools.sentry`.
+
+    This function is safe to call multiple times; failures are silently ignored
+    to avoid impacting the application startup.
+    """
+    if not SENTRY_ACTIVE:
+        return
+
+    try:
+        cherrypy.tools.sentry = SentryTool()
+    except Exception:
+        logger.exception('Failed to register Sentry CherryPy tool.')
+

cmapi/dev_tools/piptools.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cmapi_dir="$(realpath "${script_dir}/..")"
+
+export CUSTOM_COMPILE_COMMAND="dev_tools/piptools.sh compile-all"
+
+ensure_piptools() {
+	if ! command -v pip-compile >/dev/null 2>&1; then
+		echo "Installing pip-tools..."
+		python3 -m pip install --upgrade pip
+		python3 -m pip install pip-tools
+	fi
+}
+
+compile_runtime() {
+	ensure_piptools
+	cd "${cmapi_dir}"
+	pip-compile --quiet --resolver=backtracking --output-file=requirements.txt requirements.in
+}
+
+compile_dev() {
+	ensure_piptools
+	cd "${cmapi_dir}"
+	pip-compile --quiet --resolver=backtracking --output-file=requirements-dev.txt requirements-dev.in
+}
+
+compile_all() {
+	compile_runtime
+	compile_dev
+}
+
+sync_runtime() {
+	ensure_piptools
+	cd "${cmapi_dir}"
+	pip-sync requirements.txt
+}
+
+sync_dev() {
+	ensure_piptools
+	cd "${cmapi_dir}"
+	pip-sync requirements.txt requirements-dev.txt
+}
+
+usage() {
+	cat <<EOF
+Usage: dev_tools/piptools.sh <command>
+
+Commands:
+	compile-runtime   Compile requirements.in -> requirements.txt
+	compile-dev       Compile requirements-dev.in -> requirements-dev.txt
+	compile-all       Compile both runtime and dev requirements (default)
+	sync-runtime      pip-sync runtime requirements only
+	sync-dev          pip-sync runtime + dev requirements
+	help              Show this help
+EOF
+}
+
+cmd="${1:-compile-all}"
+case "${cmd}" in
+	compile-runtime) compile_runtime ;;
+	compile-dev)     compile_dev ;;
+	compile-all)     compile_all ;;
+	sync-runtime)    sync_runtime ;;
+	sync-dev)        sync_dev ;;
+	help|--help|-h)  usage ;;
+	*) echo "Unknown command: ${cmd}" >&2; usage; exit 1 ;;
+esac
+

cmapi/requirements-dev.in

@@ -0,0 +1,8 @@
+# Direct, top-level development/testing dependencies
+# Compile with: pip-compile --output-file=requirements-dev.txt requirements-dev.in
+
+pytest==8.3.5
+fabric==3.2.2
+# Tooling
+pip-tools
+

cmapi/requirements-dev.txt

@@ -1,37 +1,69 @@
-# For integration tests
+#
-pytest==8.3.5
+# This file is autogenerated by pip-compile with Python 3.9
-fabric==3.2.2
+# by the following command:
-
+#
-# This frozen part is autogenerated by pip-compile: pip-compile requirements-dev.txt
+#    dev_tools/piptools.sh compile-all
+#
 bcrypt==4.3.0
     # via paramiko
+build==1.3.0
+    # via pip-tools
 cffi==1.17.1
     # via
     #   cryptography
     #   pynacl
+click==8.1.8
+    # via pip-tools
 cryptography==45.0.5
     # via paramiko
 decorator==5.2.1
     # via fabric
 deprecated==1.2.18
     # via fabric
+exceptiongroup==1.3.0
+    # via pytest
 fabric==3.2.2
-    # via -r requirements-dev.txt
+    # via -r requirements-dev.in
+importlib-metadata==8.7.0
+    # via build
 iniconfig==2.1.0
     # via pytest
 invoke==2.2.0
     # via fabric
 packaging==25.0
-    # via pytest
+    # via
+    #   build
+    #   pytest
 paramiko==3.5.1
     # via fabric
+pip-tools==7.5.0
+    # via -r requirements-dev.in
 pluggy==1.6.0
     # via pytest
 pycparser==2.22
     # via cffi
 pynacl==1.5.0
     # via paramiko
+pyproject-hooks==1.2.0
+    # via
+    #   build
+    #   pip-tools
 pytest==8.3.5
-    # via -r requirements-dev.txt
+    # via -r requirements-dev.in
+tomli==2.2.1
+    # via
+    #   build
+    #   pip-tools
+    #   pytest
+typing-extensions==4.14.1
+    # via exceptiongroup
+wheel==0.45.1
+    # via pip-tools
 wrapt==1.17.2
     # via deprecated
+zipp==3.23.0
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools

cmapi/requirements.in

@@ -0,0 +1,19 @@
+# Direct, top-level runtime dependencies for cmapi
+# Compile with: pip-compile --output-file=requirements.txt requirements.in
+
+aiohttp==3.11.16
+awscli==1.38.28
+CherryPy==18.10.0
+cryptography==43.0.3
+furl==2.1.4
+gsutil==5.33
+lxml==5.3.2
+psutil==7.0.0
+pyotp==2.9.0
+requests==2.32.3
+# required for CherryPy RoutesDispatcher,
+# but CherryPy itself has no such dependency
+Routes==2.5.1
+typer==0.15.2
+sentry-sdk==2.34.1
+

cmapi/requirements.txt

@@ -1,78 +1,231 @@
-aiohttp==3.11.16
+#
-awscli==1.38.28
+# This file is autogenerated by pip-compile with Python 3.9
-CherryPy==18.10.0
+# by the following command:
-cryptography==43.0.3
+#
-furl==2.1.4
+#    dev_tools/piptools.sh compile-all
-gsutil==5.33
+#
-lxml==5.3.2
-psutil==7.0.0
-pyotp==2.9.0
-requests==2.32.3
-# required for CherryPy RoutesDispatcher,
-# but CherryPy itself has no such a dependency
-Routes==2.5.1
-typer==0.15.2
-
-# indirect dependencies
 aiohappyeyeballs==2.6.1
+    # via aiohttp
+aiohttp==3.11.16
+    # via
+    #   -r requirements.in
+    #   google-auth
 aiosignal==1.3.2
+    # via aiohttp
 argcomplete==3.6.2
+    # via gsutil
 async-timeout==5.0.1
+    # via aiohttp
 attrs==25.3.0
+    # via aiohttp
 autocommand==2.2.2
-backports.tarfile==1.2.0
+    # via jaraco-text
+awscli==1.38.28
+    # via -r requirements.in
+backports-tarfile==1.2.0
+    # via jaraco-context
 boto==2.49.0
+    # via gcs-oauth2-boto-plugin
 botocore==1.37.28
+    # via
+    #   awscli
+    #   s3transfer
 cachetools==5.5.2
+    # via google-auth
 certifi==2025.1.31
+    # via
+    #   requests
+    #   sentry-sdk
 cffi==1.17.1
+    # via cryptography
 charset-normalizer==3.4.1
+    # via requests
 cheroot==10.0.1
+    # via cherrypy
+cherrypy==18.10.0
+    # via -r requirements.in
 click==8.1.8
+    # via typer
 colorama==0.4.6
+    # via awscli
 crcmod==1.7
+    # via gsutil
+cryptography==43.0.3
+    # via
+    #   -r requirements.in
+    #   pyopenssl
 docutils==0.16
+    # via awscli
 fasteners==0.19
+    # via
+    #   google-apitools
+    #   gsutil
 frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+furl==2.1.4
+    # via -r requirements.in
 gcs-oauth2-boto-plugin==3.2
+    # via gsutil
 google-apitools==0.5.32
-google-auth==2.17.0
+    # via gsutil
+google-auth[aiohttp]==2.17.0
+    # via
+    #   gcs-oauth2-boto-plugin
+    #   google-auth-httplib2
+    #   gsutil
 google-auth-httplib2==0.2.0
+    # via
+    #   gcs-oauth2-boto-plugin
+    #   gsutil
 google-reauth==0.1.1
+    # via
+    #   gcs-oauth2-boto-plugin
+    #   gsutil
+gsutil==5.33
+    # via -r requirements.in
 httplib2==0.20.4
+    # via
+    #   gcs-oauth2-boto-plugin
+    #   google-apitools
+    #   google-auth-httplib2
+    #   gsutil
+    #   oauth2client
 idna==3.10
-jaraco.collections==5.1.0
+    # via
-jaraco.context==6.0.1
+    #   requests
-jaraco.functools==4.1.0
+    #   yarl
-jaraco.text==4.0.0
+jaraco-collections==5.1.0
+    # via cherrypy
+jaraco-context==6.0.1
+    # via jaraco-text
+jaraco-functools==4.1.0
+    # via
+    #   cheroot
+    #   jaraco-text
+    #   tempora
+jaraco-text==4.0.0
+    # via jaraco-collections
 jmespath==1.0.1
+    # via botocore
+lxml==5.3.2
+    # via -r requirements.in
 markdown-it-py==3.0.0
+    # via rich
 mdurl==0.1.2
+    # via markdown-it-py
 monotonic==1.6
+    # via gsutil
 more-itertools==10.6.0
+    # via
+    #   cheroot
+    #   cherrypy
+    #   jaraco-functools
+    #   jaraco-text
 multidict==6.3.2
+    # via
+    #   aiohttp
+    #   yarl
 oauth2client==4.1.3
+    # via
+    #   gcs-oauth2-boto-plugin
+    #   google-apitools
 orderedmultidict==1.0.1
+    # via furl
 portend==3.2.0
+    # via cherrypy
 propcache==0.3.1
+    # via
+    #   aiohttp
+    #   yarl
+psutil==7.0.0
+    # via -r requirements.in
 pyasn1==0.6.1
-pyasn1_modules==0.4.2
+    # via
+    #   oauth2client
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.4.2
+    # via
+    #   google-auth
+    #   oauth2client
 pycparser==2.22
-Pygments==2.19.1
+    # via cffi
-pyOpenSSL==24.2.1
+pygments==2.19.1
+    # via rich
+pyopenssl==24.2.1
+    # via
+    #   gcs-oauth2-boto-plugin
+    #   gsutil
+pyotp==2.9.0
+    # via -r requirements.in
 pyparsing==3.2.3
+    # via httplib2
 python-dateutil==2.9.0.post0
+    # via
+    #   botocore
+    #   tempora
 pyu2f==0.1.5
-PyYAML==6.0.2
+    # via google-reauth
-repoze.lru==0.7
+pyyaml==6.0.2
+    # via awscli
+repoze-lru==0.7
+    # via routes
+requests==2.32.3
+    # via
+    #   -r requirements.in
+    #   google-auth
 retry-decorator==1.1.1
+    # via
+    #   gcs-oauth2-boto-plugin
+    #   gsutil
 rich==14.0.0
+    # via typer
+routes==2.5.1
+    # via -r requirements.in
 rsa==4.7.2
+    # via
+    #   awscli
+    #   gcs-oauth2-boto-plugin
+    #   google-auth
+    #   oauth2client
 s3transfer==0.11.4
+    # via awscli
+sentry-sdk==2.34.1
+    # via -r requirements.in
 shellingham==1.5.4
+    # via typer
 six==1.17.0
+    # via
+    #   furl
+    #   gcs-oauth2-boto-plugin
+    #   google-apitools
+    #   google-auth
+    #   gsutil
+    #   oauth2client
+    #   orderedmultidict
+    #   python-dateutil
+    #   pyu2f
+    #   routes
 tempora==5.8.0
-typing_extensions==4.13.1
+    # via portend
+typer==0.15.2
+    # via -r requirements.in
+typing-extensions==4.13.1
+    # via
+    #   multidict
+    #   rich
+    #   typer
 urllib3==1.26.20
+    # via
+    #   botocore
+    #   requests
+    #   sentry-sdk
 yarl==1.19.0
-zc.lockfile==3.0.post1
+    # via aiohttp
+zc-lockfile==3.0.post1
+    # via cherrypy
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools