MCP Telemetry, Observability and Distributed Tracing

sequenceDiagram
    autonumber
    participant Host as AI Host (Agent / LLM)
    participant Gateway as MCP Gateway (Reverse Proxy)
    participant Server as Remote MCP Server
    participant DB as Production DB / Tool Target

    Note over Host,DB: Trace ID: 4bf92f3577b34da6a3ce929d0e0e4736
    Host->>Gateway: POST /messages (Span: agent.tool_execution)
    Note over Host,Gateway: Injects W3C traceparent into JSON-RPC _meta
    Gateway->>Server: tools/call [db_select] (Span: gateway.forward)
    Server->>DB: SQL SELECT (Span: db.query)
    DB-->>Server: Query Result Rows (24ms)
    Server-->>Gateway: JSON-RPC Result (Span: mcp.tool_eval)
    Gateway-->>Host: Streamed SSE Chunk (Span: response.deliver)

{
  "jsonrpc": "2.0",
  "id": "trace-job-7712",
  "method": "tools/call",
  "params": {
    "name": "enterprise_db_query",
    "arguments": {
      "sql": "SELECT account_id, billing_tier FROM accounts WHERE status = 'active' LIMIT 10;"
    },
    "_meta": {
      "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
      "tracestate": "rojo=1,congo=2",
      "tenant_id": "org_enterprise_99",
      "agent_model": "claude-3-7-sonnet"
    }
  }
}

{
  "jsonrpc": "2.0",
  "id": "trace-job-7712",
  "result": {
    "content": [
      {
        "type": "text",
        "text": "[{\"account_id\": \"acc_01\", \"billing_tier\": \"enterprise\"}]"
      }
    ],
    "_meta": {
      "execution_time_ms": 38.4,
      "span_id": "9a38f72c0199e4b1",
      "estimated_tokens": 42
    }
  }
}

import { NodeSDK } from '@opentelemetry/sdk-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
import { 
  trace, 
  context, 
  propagation, 
  SpanStatusCode, 
  ROOT_CONTEXT 
} from '@opentelemetry/api';

// 1. Initialize OpenTelemetry SDK
const exporter = new OTLPTraceExporter({
  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'https://otel-collector.internal:4317'
});

const sdk = new NodeSDK({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'mcp-database-service',
    [SemanticResourceAttributes.SERVICE_VERSION]: '1.2.0',
    [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: 'production'
  }),
  traceExporter: exporter
});

sdk.start();

const tracer = trace.getTracer('mcp-instrumentation', '1.0.0');

// 2. Middleware: Extract Context and Trace Tool Execution
export async function executeMonitoredTool(
  toolName: string,
  args: Record<string, any>,
  meta: Record<string, any> = {},
  handler: (args: any) => Promise<any>
): Promise<any> {
  // Extract W3C traceparent from JSON-RPC _meta if available
  let parentContext = ROOT_CONTEXT;
  if (meta?.traceparent) {
    parentContext = propagation.extract(ROOT_CONTEXT, {
      traceparent: meta.traceparent,
      tracestate: meta.tracestate
    });
  }

  return context.with(parentContext, async () => {
    const span = tracer.startSpan(`mcp.tool_call:${toolName}`, {
      attributes: {
        'mcp.tool.name': toolName,
        'mcp.tenant.id': meta?.tenant_id || 'anonymous',
        'mcp.agent.model': meta?.agent_model || 'unknown',
        'rpc.system': 'mcp-jsonrpc'
      }
    });

    const startTime = performance.now();

    try {
      // Execute the actual tool logic
      const result = await handler(args);

      const duration = performance.now() - startTime;
      span.setAttribute('mcp.execution_duration_ms', duration);
      span.setStatus({ code: SpanStatusCode.OK });

      return {
        result,
        meta: {
          traceparent: span.spanContext().traceId,
          execution_time_ms: duration
        }
      };
    } catch (error: any) {
      span.recordException(error);
      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: error.message || 'Tool execution failure'
      });
      throw error;
    } finally {
      span.end();
    }
  });
}

groups:
  - name: MCP_Observability_Alerts
    rules:
      - alert: MCPToolHighLatencyP99
        expr: histogram_quantile(0.99, sum(rate(mcp_tool_execution_duration_seconds_bucket[5m])) by (le, tool_name)) > 3.0
        for: 2m
        labels:
          severity: warning
          team: ai-platform
        annotations:
          summary: "MCP tool {{ $labels.tool_name }} p99 latency exceeds 3 seconds"
          description: "High tool latency directly blocks host LLM agent generation and exhausts connection pools."

      - alert: MCPHighToolFailureRate
        expr: (sum(rate(mcp_tool_invocation_total{status="error"}[5m])) / sum(rate(mcp_tool_invocation_total[5m]))) * 100 > 2.0
        for: 1m
        labels:
          severity: critical
          team: ai-platform
        annotations:
          summary: "MCP tool execution error rate > 2% across cluster"
          description: "Agents encountering repeated tool failures risk hallucination cascades and runaway retries."

{
  "timestamp": "2026-09-12T19:45:10.112Z",
  "level": "warn",
  "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736",
  "span_id": "9a38f72c0199e4b1",
  "service": "mcp-database-service",
  "event": "tool_execution_slow",
  "tool": "enterprise_db_query",
  "duration_ms": 4120,
  "tenant": "org_enterprise_99",
  "query_fingerprint": "SELECT * FROM billing_events WHERE event_date = ?",
  "message": "Query exceeded soft threshold (3000ms). Index scan missing."
}