Refactor telemetry integration to use Azure Monitor, removing Application Insights dependency and enhancing event tracking

Author: Abdul-Microsoft

src/backend/api/event_utils.py

@@ -3,69 +3,18 @@
 import os
 
 # Third-party
-from applicationinsights import TelemetryClient
-from applicationinsights.channel import SynchronousQueue, SynchronousSender, TelemetryChannel
+from azure.monitor.events.extension import track_event
 
 from dotenv import load_dotenv
 
 load_dotenv()
 
-# Global telemetry client (initialized once)
-_telemetry_client = None
-
-
-def _get_telemetry_client():
-    """Get or create the Application Insights telemetry client."""
-    global _telemetry_client
-
-    if _telemetry_client is None:
-        connection_string = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING")
-        if connection_string:
-            try:
-                # Extract instrumentation key from connection string
-                # Format: InstrumentationKey=xxx;IngestionEndpoint=https://...
-                parts = dict(part.split('=', 1) for part in connection_string.split(';') if '=' in part)
-                instrumentation_key = parts.get('InstrumentationKey')
-
-                if instrumentation_key:
-                    # Create a synchronous channel for immediate sending
-                    sender = SynchronousSender()
-                    queue = SynchronousQueue(sender)
-                    channel = TelemetryChannel(None, queue)
-
-                    _telemetry_client = TelemetryClient(instrumentation_key, channel)
-                    logging.info("Application Insights TelemetryClient initialized successfully")
-                else:
-                    logging.error("Could not extract InstrumentationKey from connection string")
-            except Exception as e:
-                logging.error(f"Failed to initialize TelemetryClient: {e}")
-
-    return _telemetry_client
-
 
 def track_event_if_configured(event_name: str, event_data: dict):
-    """Track a custom event to Application Insights customEvents table.
-
-    This uses the Application Insights SDK TelemetryClient which properly
-    sends custom events to the customEvents table in Application Insights.
-    """
+    """Track a custom event to Application Insights customEvents table."""
     instrumentation_key = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING")
     if instrumentation_key:
-        try:
-            client = _get_telemetry_client()
-            if client:
-                # Convert all values to strings to ensure compatibility
-                properties = {k: str(v) for k, v in event_data.items()}
-
-                # Track the custom event
-                client.track_event(event_name, properties=properties)
-                client.flush()  # Ensure immediate sending
-
-                logging.debug(f"Tracked custom event: {event_name} with data: {event_data}")
-            else:
-                logging.warning("TelemetryClient not available, custom event not tracked")
-        except Exception as e:
-            logging.error(f"Failed to track event {event_name}: {e}")
+        track_event(event_name, event_data)
     else:
         logging.warning(
             f"Skipping track_event for {event_name} as Application Insights is not configured"

src/backend/app.py

@@ -5,10 +5,11 @@
 
 from api.api_routes import router as backend_router
 
-from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter, AzureMonitorTraceExporter
+from azure.monitor.opentelemetry import configure_azure_monitor
 
 from common.config.config import app_config
 from common.logger.app_logger import AppLogger
+from common.telemetry import patch_instrumentors
 
 from dotenv import load_dotenv
 
@@ -17,13 +18,7 @@
 
 from helper.azure_credential_utils import get_azure_credential
 
-from opentelemetry import trace
-from opentelemetry._logs import set_logger_provider
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
-from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
-from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
 from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent  # pylint: disable=E0611
 
@@ -138,52 +133,23 @@ def create_app() -> FastAPI:
     # This must happen AFTER app creation but BEFORE route registration
     instrumentation_key = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING")
     if instrumentation_key:
-        # SOLUTION: Use manual telemetry setup instead of configure_azure_monitor
-        # This gives us precise control over what gets instrumented, avoiding interference
-        # with Semantic Kernel's async generators while still tracking Azure SDK calls
-
-        # Set up Azure Monitor exporter for traces
-        azure_trace_exporter = AzureMonitorTraceExporter(connection_string=instrumentation_key)
-
-        # Create a tracer provider and add the Azure Monitor exporter
-        tracer_provider = TracerProvider()
-        tracer_provider.add_span_processor(BatchSpanProcessor(azure_trace_exporter))
-
-        # Set the global tracer provider
-        trace.set_tracer_provider(tracer_provider)
-
-        # Set up Azure Monitor exporter for logs (appears in traces table)
-        azure_log_exporter = AzureMonitorLogExporter(connection_string=instrumentation_key)
-
-        # Create a logger provider and add the Azure Monitor exporter
-        logger_provider = LoggerProvider()
-        logger_provider.add_log_record_processor(BatchLogRecordProcessor(azure_log_exporter))
-        set_logger_provider(logger_provider)
-
-        # Attach OpenTelemetry handler to Python's root logger
-        handler = LoggingHandler(logger_provider=logger_provider)
-        logging.getLogger().addHandler(handler)
+        # Fix: Patch buggy instrumentors before Azure Monitor loads them
+        # See: https://github.com/microsoft/semantic-kernel/issues/13715
+        patch_instrumentors()
+
+        # Configure Azure Monitor with FULL auto-instrumentation
+        configure_azure_monitor(
+            connection_string=instrumentation_key,
+            enable_live_metrics=True
+        )
 
-        # Instrument ONLY FastAPI for HTTP request/response tracing
-        # This is safe because it only wraps HTTP handlers, not internal async operations
+        # Instrument FastAPI for HTTP request/response tracing
         FastAPIInstrumentor.instrument_app(
             app,
-            excluded_urls="socket,ws",  # Exclude WebSocket URLs to reduce noise
-            tracer_provider=tracer_provider
+            excluded_urls="health,socket,ws",
         )
 
-        # Optional: Add manual spans in your code for Azure SDK operations using:
-        # from opentelemetry import trace
-        # tracer = trace.get_tracer(__name__)
-        # with tracer.start_as_current_span("operation_name"):
-        #     # your Azure SDK call here
-
-        logger.logger.info("Application Insights configured with selective instrumentation")
-        logger.logger.info("✓ FastAPI HTTP tracing enabled")
-        logger.logger.info("✓ Python logging export to Application Insights enabled")
-        logger.logger.info("✓ Manual span support enabled for Azure SDK operations")
-        logger.logger.info("✓ Custom events via OpenTelemetry enabled")
-        logger.logger.info("✓ Semantic Kernel async generators unaffected")
+        logger.logger.info("Application Insights configured with full auto-instrumentation")
     else:
         logger.logger.warning("No Application Insights connection string found. Telemetry disabled.")
 

src/backend/common/telemetry/__init__.py

@@ -1,17 +1,7 @@
 """Telemetry utilities for Application Insights integration."""
 
-from common.telemetry.telemetry_helper import (
-    add_span_attributes,
-    get_tracer,
-    trace_context,
-    trace_operation,
-    trace_sync_context,
-)
+from common.telemetry.patch_instrumentor import patch_instrumentors
 
 __all__ = [
-    "trace_operation",
-    "trace_context",
-    "trace_sync_context",
-    "get_tracer",
-    "add_span_attributes",
+    "patch_instrumentors",
 ]

src/backend/common/telemetry/patch_instrumentor.py

@@ -0,0 +1,45 @@
+"""\nPatch for Azure AI telemetry instrumentors.\n\nFixes GitHub issue: https://github.com/microsoft/semantic-kernel/issues/13715\n\nThe bug: agent_api_response_to_str() in both azure.ai.agents and azure.ai.projects\nraises ValueError when response_format is a dict (e.g. from Semantic Kernel's AzureAIAgent).\nIt only handles str and None types.\n\nThe fix: Monkey-patch that method to convert dict/other types to JSON string instead of raising.\nMust be called BEFORE configure_azure_monitor() triggers any instrumentation.\n"""
+
+import json
+import logging
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _fixed_response_to_str(response_format: Any) -> Optional[str]:
+    """Convert response_format to string, handling dict types."""
+    if response_format is None:
+        return None
+    if isinstance(response_format, str):
+        return response_format
+    try:
+        return json.dumps(response_format, default=str)
+    except (TypeError, ValueError):
+        return str(response_format)
+
+
+def patch_instrumentors():
+    """
+    Patch Azure AI telemetry instrumentors to handle dict response_format.
+
+    This fixes the ValueError: \"Unknown response format <class 'dict'>\" error\n    that occurs when Semantic Kernel's AzureAIAgent passes a dict as response_format\n    and Azure Monitor telemetry instrumentor tries to serialize it.\n\n    Patches both azure.ai.agents and azure.ai.projects packages.\n    Must be called BEFORE configure_azure_monitor().\n    """
+    # Patch azure.ai.agents (primary package with the bug)
+    try:
+        from azure.ai.agents.telemetry._ai_agents_instrumentor import (
+            _AIAgentsInstrumentorPreview as _AgentsPreview,
+        )
+        _AgentsPreview.agent_api_response_to_str = staticmethod(_fixed_response_to_str)
+        logger.info("Patched azure.ai.agents instrumentor")
+    except ImportError:
+        logger.debug("azure.ai.agents telemetry not installed, skipping patch")
+
+    # Patch azure.ai.projects (in case it exists in the environment)
+    try:
+        from azure.ai.projects.telemetry._ai_project_instrumentor import (
+            _AIAgentsInstrumentorPreview as _ProjectsPreview,
+        )
+        _ProjectsPreview.agent_api_response_to_str = staticmethod(_fixed_response_to_str)
+        logger.info("Patched azure.ai.projects instrumentor")
+    except ImportError:
+        logger.debug("azure.ai.projects telemetry not installed, skipping patch")