Tracing fixes:

Don't log trace_params in tracing logger, because it already has all this data
Don't print span attrs, it can contain lots of headers
Save small part of the response into the span, if the response was a JSON string
Added JSON logging of trace details into a separate file (to not spam the main log with machine readable stuff)
Record part of the response into the span
Set duration attribute in server spans
Log 404 errors
Colorize the traces (each span slightly changes the color of the parent span)
Improve trace visualization with duration formatting and notes for request/response pairs

cmapi/tracing/traced_session.py

@@ -1,9 +1,16 @@
 """Customized requests.Session that automatically traces outbound HTTP calls."""
 from typing import Any, Optional
 
+import logging
+import time
 import requests
 
-from tracing.tracer import get_tracer
+from tracing.tracer import get_tracer, TraceSpan
+
+# Limit for raw JSON string preview (in characters)
+_PREVIEW_MAX_CHARS = 512
+
+logger = logging.getLogger("tracer")
 
 
 class TracedSession(requests.Session):
@@ -19,6 +26,7 @@ class TracedSession(requests.Session):
         with tracer.start_as_current_span(span_name, kind="CLIENT") as span:
             span.set_attribute("http.method", method)
             span.set_attribute("http.url", url)
+            span.set_attribute("request_is_sync", True)
 
             tracer.inject_outbound_headers(headers)
             try:
@@ -28,7 +36,11 @@ class TracedSession(requests.Session):
                 raise
             else:
                 span.set_attribute("http.status_code", response.status_code)
+                _record_outbound_json_preview(response, span)
                 return response
+            finally:
+                duration_ms = (time.time_ns() - span.start_ns) / 1_000_000.0
+                span.set_attribute("request_duration_ms", duration_ms)
 
 
 _default_session: Optional[TracedSession] = None
@@ -41,4 +53,14 @@ def get_traced_session() -> TracedSession:
     return _default_session
 
 
-
+def _record_outbound_json_preview(response: requests.Response, span: TraceSpan) -> None:
+    """If response is JSON, attach small part of it to span"""
+    try:
+        content_type = str(response.headers.get('Content-Type', '')).lower()
+        if 'application/json' not in content_type:
+            return
+        text = response.text  # requests will decode using inferred/declared encoding
+        span.set_attribute('http.response.body.size', len(text))
+        span.set_attribute('http.response.json', text[:_PREVIEW_MAX_CHARS])
+    except Exception:
+        logger.exception("Could not extract JSON response body")