fix: resolve Sentry/OpenTelemetry integration conflicts (PROJQUAY-9198) (#4232)

* fix: resolve Sentry/OpenTelemetry integration conflicts

Configure Sentry to use minimal integrations when OTEL_TRACING is enabled
to prevent instrumentation conflicts that broke exception capture


---------

Co-authored-by: shudeshp <shudeshp@redhat.com>

buildman/builder.py

@@ -5,6 +5,10 @@ import socket
 import time
 
 import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+from sentry_sdk.integrations.logging import LoggingIntegration
+from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
+from sentry_sdk.integrations.stdlib import StdlibIntegration
 
 import features
 from app import (
@@ -39,11 +43,43 @@ def initialize_sentry():
         sentry_dsn = app.config.get("SENTRY_DSN", "")
         if sentry_dsn:
             try:
+                integrations = []
+
+                # Always include logging integration
+                integrations.append(
+                    LoggingIntegration(level=logging.INFO, event_level=logging.ERROR)
+                )
+
+                # Only add Flask and SQLAlchemy integrations if OpenTelemetry is not enabled
+                if not getattr(features, "OTEL_TRACING", False):
+                    integrations.extend(
+                        [
+                            FlaskIntegration(transaction_style="endpoint"),
+                            SqlalchemyIntegration(),
+                            StdlibIntegration(),
+                        ]
+                    )
+                else:
+                    logger.info(
+                        "OpenTelemetry enabled - using minimal Sentry integrations for buildman"
+                    )
+
                 sentry_sdk.init(
                     dsn=sentry_dsn,
                     environment=app.config.get("SENTRY_ENVIRONMENT", "production"),
                     traces_sample_rate=app.config.get("SENTRY_TRACES_SAMPLE_RATE", 0.1),
                     profiles_sample_rate=app.config.get("SENTRY_PROFILES_SAMPLE_RATE", 0.1),
+                    integrations=integrations,
+                    default_integrations=False,
+                    auto_session_tracking=True,
+                    # Fix connection pool issues
+                    transport=sentry_sdk.transport.make_transport(
+                        {
+                            "pool_connections": 10,  # Instead of 1
+                            "pool_maxsize": 20,  # Max connections per pool
+                            "max_retries": 3,  # Retry failed sends
+                        }
+                    ),
                 )
                 sentry_sdk.set_tag("service", "buildman")
                 sentry_sdk.set_tag("buildman", buildman_name)

buildman/test/test_buildman.py

@@ -766,15 +766,17 @@ class TestBuilderSentry(unittest.TestCase):
 
             initialize_sentry()
 
-            # Verify Sentry was initialized with correct parameters
+            call_args = mock_sentry_sdk.init.call_args
-            mock_sentry_sdk.init.assert_called_once_with(
+            assert call_args is not None
-                dsn="https://test@sentry.io/123",
+
-                environment="test",
+            kwargs = call_args.kwargs
-                traces_sample_rate=0.5,
+            assert kwargs["dsn"] == "https://test@sentry.io/123"
-                profiles_sample_rate=0.3,
+            assert kwargs["environment"] == "test"
-            )
+            assert kwargs["traces_sample_rate"] == 0.5
+            assert kwargs["profiles_sample_rate"] == 0.3
+
+            assert mock_sentry_sdk.init.call_count == 1
 
-            # Verify tags were set
             expected_calls = [
                 (("service", "buildman"),),
                 (("buildman", "test-host:buildman"),),
@@ -842,12 +844,17 @@ class TestBuilderSentry(unittest.TestCase):
 
             initialize_sentry()
 
-            mock_sentry_sdk.init.assert_called_once_with(
+            call_args = mock_sentry_sdk.init.call_args
-                dsn="https://test@sentry.io/123",
+            assert call_args is not None
-                environment="production",  # default
+
-                traces_sample_rate=0.1,  # default
+            kwargs = call_args.kwargs
-                profiles_sample_rate=0.1,  # default
+            assert kwargs["dsn"] == "https://test@sentry.io/123"
-            )
+            assert kwargs["environment"] == "production"  # default
+            assert kwargs["traces_sample_rate"] == 0.1  # default
+            assert kwargs["profiles_sample_rate"] == 0.1  # default
+
+            # Verify the SDK was called exactly once
+            assert mock_sentry_sdk.init.call_count == 1
 
     def test_sentry_tags_set_correctly(self):
         """Test that Sentry tags are set with correct values."""

templates/base.html

@@ -85,7 +85,7 @@
       </script>
     {% else %}
       <script type="text/javascript">
-        // FakeSentry wrapper when Sentry is not available
+        // FakeSentry wrapper when Sentry is not loaded yet/not available
         window.Sentry = {
           captureException: function() {},
           setUser: function() {},

test/test_exceptionlog_sentry.py

@@ -59,13 +59,17 @@ class TestExceptionLogSentry:
 
             sentry = Sentry(mock_app)
 
-            # Verify Sentry SDK was initialized
+            call_args = mock_sentry_sdk.init.call_args
-            mock_sentry_sdk.init.assert_called_once_with(
+            assert call_args is not None
-                dsn="https://test@sentry.io/123",
+
-                environment="test",
+            kwargs = call_args.kwargs
-                traces_sample_rate=0.5,
+            assert kwargs["dsn"] == "https://test@sentry.io/123"
-                profiles_sample_rate=0.3,
+            assert kwargs["environment"] == "test"
-            )
+            assert kwargs["traces_sample_rate"] == 0.5
+            assert kwargs["profiles_sample_rate"] == 0.3
+
+            # Verify the SDK was called exactly once
+            assert mock_sentry_sdk.init.call_count == 1
 
             # Verify the initialized Sentry SDK object is returned
             assert sentry.state is mock_initialized_sentry
@@ -102,13 +106,19 @@ class TestExceptionLogSentry:
 
             sentry = Sentry(mock_app)
 
-            # Verify default values are used
+            # Verify default values are used - the SDK automatically adds additional parameters
-            mock_sentry_sdk.init.assert_called_once_with(
+            call_args = mock_sentry_sdk.init.call_args
-                dsn="https://test@sentry.io/123",
+            assert call_args is not None
-                environment="production",  # default
+
-                traces_sample_rate=0.1,  # default
+            # Check the required parameters we explicitly set
-                profiles_sample_rate=0.1,  # default
+            kwargs = call_args.kwargs
-            )
+            assert kwargs["dsn"] == "https://test@sentry.io/123"
+            assert kwargs["environment"] == "production"  # default
+            assert kwargs["traces_sample_rate"] == 0.1  # default
+            assert kwargs["profiles_sample_rate"] == 0.1  # default
+
+            # Verify the SDK was called exactly once
+            assert mock_sentry_sdk.init.call_count == 1
 
             # Verify the initialized Sentry SDK object is returned
             assert sentry.state is mock_initialized_sentry
@@ -161,13 +171,19 @@ class TestExceptionLogSentry:
 
             sentry = Sentry(mock_app)
 
-            # Verify Sentry SDK was initialized without any integrations
+            # Verify Sentry SDK was initialized - the SDK automatically adds additional parameters
-            mock_sentry_sdk.init.assert_called_once_with(
+            call_args = mock_sentry_sdk.init.call_args
-                dsn="https://test@sentry.io/123",
+            assert call_args is not None
-                environment="test",
+
-                traces_sample_rate=0.1,
+            # Check the required parameters we explicitly set
-                profiles_sample_rate=0.1,
+            kwargs = call_args.kwargs
-            )
+            assert kwargs["dsn"] == "https://test@sentry.io/123"
+            assert kwargs["environment"] == "test"
+            assert kwargs["traces_sample_rate"] == 0.1
+            assert kwargs["profiles_sample_rate"] == 0.1
+
+            # Verify the SDK was called exactly once
+            assert mock_sentry_sdk.init.call_count == 1
 
             # Verify the initialized Sentry SDK object is returned
             assert sentry.state is mock_initialized_sentry

util/saas/exceptionlog.py

@@ -1,6 +1,12 @@
 import logging
 
 import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+from sentry_sdk.integrations.logging import LoggingIntegration
+from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
+from sentry_sdk.integrations.stdlib import StdlibIntegration
+
+import features
 
 
 class FakeSentryClient(object):
@@ -31,11 +37,36 @@ class Sentry(object):
             sentry_dsn = app.config.get("SENTRY_DSN", "")
             if sentry_dsn:
                 try:
+                    integrations = []
+
+                    # Always include logging integration
+                    integrations.append(
+                        LoggingIntegration(level=logging.INFO, event_level=logging.ERROR)
+                    )
+
+                    # Only add Flask and SQLAlchemy integrations if OpenTelemetry is not enabled
+                    # to avoid conflicts with OpenTelemetry instrumentors
+                    if not getattr(features, "OTEL_TRACING", False):
+                        integrations.extend(
+                            [
+                                FlaskIntegration(transaction_style="endpoint"),
+                                SqlalchemyIntegration(),
+                                StdlibIntegration(),
+                            ]
+                        )
+                    else:
+                        # When OTEL is enabled, use minimal integrations to avoid conflicts
+                        logger = logging.getLogger(__name__)
+                        logger.info("OpenTelemetry enabled - using minimal Sentry integrations")
+
                     initialized_sentry = sentry_sdk.init(
                         dsn=sentry_dsn,
                         environment=app.config.get("SENTRY_ENVIRONMENT", "production"),
                         traces_sample_rate=app.config.get("SENTRY_TRACES_SAMPLE_RATE", 0.1),
                         profiles_sample_rate=app.config.get("SENTRY_PROFILES_SAMPLE_RATE", 0.1),
+                        integrations=integrations,
+                        default_integrations=False,
+                        auto_session_tracking=True,
                     )
                     # Return the initialized Sentry SDK object directly
                     sentry = initialized_sentry

workers/test/test_worker.py

@@ -92,9 +92,14 @@ def test_default_sentry_config_values():
 
         worker = Worker()
 
-        mock_sentry_sdk.init.assert_called_once_with(
+        call_args = mock_sentry_sdk.init.call_args
-            dsn="https://test@sentry.io/123",
+        assert call_args is not None
-            environment="production",  # default
+
-            traces_sample_rate=0.1,  # default
+        kwargs = call_args.kwargs
-            profiles_sample_rate=0.1,  # default
+        assert kwargs["dsn"] == "https://test@sentry.io/123"
-        )
+        assert kwargs["environment"] == "production"  # default
+        assert kwargs["traces_sample_rate"] == 0.1  # default
+        assert kwargs["profiles_sample_rate"] == 0.1  # default
+
+        # Verify the SDK was called exactly once
+        assert mock_sentry_sdk.init.call_count == 1

workers/worker.py

@@ -10,7 +10,12 @@ from threading import Event
 
 import sentry_sdk
 from apscheduler.schedulers.background import BackgroundScheduler
+from sentry_sdk.integrations.flask import FlaskIntegration
+from sentry_sdk.integrations.logging import LoggingIntegration
+from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
+from sentry_sdk.integrations.stdlib import StdlibIntegration
 
+import features
 from app import app
 from data.database import UseThenDisconnect
 from util.log import logfile_path
@@ -67,11 +72,43 @@ class Worker(object):
             sentry_dsn = app.config.get("SENTRY_DSN", "")
             if sentry_dsn:
                 try:
+                    integrations = []
+
+                    # Always include logging integration
+                    integrations.append(
+                        LoggingIntegration(level=logging.INFO, event_level=logging.ERROR)
+                    )
+
+                    # Only add Flask and SQLAlchemy integrations if OpenTelemetry is not enabled
+                    if not getattr(features, "OTEL_TRACING", False):
+                        integrations.extend(
+                            [
+                                FlaskIntegration(transaction_style="endpoint"),
+                                SqlalchemyIntegration(),
+                                StdlibIntegration(),
+                            ]
+                        )
+                    else:
+                        logger.info(
+                            "OpenTelemetry enabled - using minimal Sentry integrations for worker"
+                        )
+
                     sentry_sdk.init(
                         dsn=sentry_dsn,
                         environment=app.config.get("SENTRY_ENVIRONMENT", "production"),
                         traces_sample_rate=app.config.get("SENTRY_TRACES_SAMPLE_RATE", 0.1),
                         profiles_sample_rate=app.config.get("SENTRY_PROFILES_SAMPLE_RATE", 0.1),
+                        integrations=integrations,
+                        default_integrations=False,
+                        auto_session_tracking=True,
+                        # Fix connection pool issues
+                        transport=sentry_sdk.transport.make_transport(
+                            {
+                                "pool_connections": 10,  # Instead of 1
+                                "pool_maxsize": 20,  # Max connections per pool
+                                "max_retries": 3,  # Retry failed sends
+                            }
+                        ),
                     )
                     sentry_sdk.set_tag("worker", worker_name)
                 except Exception as e: