DevOps and Cloud SRE Tooling with Scoped MCP Agents

graph TD
    Agent[SRE Incident Agent] --> MCP[DevOps MCP Server]
    
    subgraph Read-Only Diagnostic Enclave (Auto-Approved)
        MCP --> K8sInspect[k8s_inspect_pod_crashes: Read-Only K8s API]
        MCP --> CWQuery[aws_query_cloudwatch_insights: AWS Logs API]
        MCP --> TFDiff[terraform_inspect_plan_drift: State Store]
    end

    subgraph Remediation Enclave (Requires Approval Token)
        MCP --> MutateGate{Approval Token Validated?}
        MutateGate -->|Token Missing / Expired| Require2FA[Return -32003 Action Confirmation Required]
        MutateGate -->|Validated by SRE Lead| ExecuteRestart[k8s_restart_deployment]
    end

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: mcp-sre-diagnostic-reader
rules:
  - apiGroups: [""]
    resources: ["pods", "pods/log", "events", "services", "namespaces"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["apps"]
    resources: ["deployments", "replicasets", "statefulsets"]
    verbs: ["get", "list", "watch"]
  # Explicitly DENY mutations (create, delete, patch) in this role
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: mcp-sre-binding
subjects:
  - kind: ServiceAccount
    name: mcp-sre-sa
    namespace: ai-infrastructure
roleRef:
  kind: ClusterRole
  name: mcp-sre-diagnostic-reader
  apiGroup: rbac.authorization.k8s.io

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "AllowCloudWatchLogsTriage",
      "Effect": "Allow",
      "Action": [
        "logs:DescribeLogGroups",
        "logs:DescribeLogStreams",
        "logs:FilterLogEvents",
        "logs:StartQuery",
        "logs:GetQueryResults",
        "logs:StopQuery"
      ],
      "Resource": "arn:aws:logs:us-east-1:123456789012:log-group:/aws/eks/prod-cluster/*"
    },
    {
      "Sid": "DenyLogDeletionAndMutation",
      "Effect": "Deny",
      "Action": [
        "logs:DeleteLogGroup",
        "logs:DeleteLogStream",
        "logs:PutRetentionPolicy"
      ],
      "Resource": "*"
    }
  ]
}

import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
import * as k8s from '@kubernetes/client-node';
import { CloudWatchLogsClient, StartQueryCommand, GetQueryResultsCommand } from '@aws-sdk/client-cloudwatch-logs';

// 1. Initialize Clients
const kc = new k8s.KubeConfig();
kc.loadFromDefault();
const k8sCoreApi = kc.makeApiClient(k8s.CoreV1Api);

const cwClient = new CloudWatchLogsClient({ region: process.env.AWS_REGION || 'us-east-1' });

const server = new Server(
  { name: 'devops-sre-mcp', version: '1.5.0' },
  { capabilities: { tools: {} } }
);

// 2. Register Scoped Tools
server.setRequestHandler(ListToolsRequestSchema, async () => {
  return {
    tools: [
      {
        name: 'k8s_diagnose_failing_pods',
        description: 'Scan a Kubernetes namespace for pods in CrashLoopBackOff, Error, or OOMKilled states.',
        inputSchema: {
          type: 'object',
          properties: {
            namespace: { type: 'string', description: 'Target Kubernetes namespace.' }
          },
          required: ['namespace']
        }
      },
      {
        name: 'aws_query_cloudwatch_insights',
        description: 'Run CloudWatch Logs Insights queries to triage recent error bursts.',
        inputSchema: {
          type: 'object',
          properties: {
            log_group_name: { type: 'string', description: 'Target CloudWatch log group.' },
            query_string: { type: 'string', description: 'Insights query (e.g. fields @timestamp, @message | filter @message like /Exception/ | limit 20)' },
            time_window_minutes: { type: 'integer', default: 15 }
          },
          required: ['log_group_name', 'query_string']
        }
      }
    ]
  };
});

// 3. Tool Execution Handlers
server.setRequestHandler(CallToolRequestSchema, async (request) => {
  const { name, arguments: args } = request.params;

  if (name === 'k8s_diagnose_failing_pods') {
    const namespace = (args?.namespace as string) || 'default';
    const podsRes = await k8sCoreApi.listNamespacedPod(namespace);

    const failingPods = [];
    for (const pod of podsRes.body.items) {
      const status = pod.status;
      const containerStatuses = status?.containerStatuses || [];

      for (const cs of containerStatuses) {
        if (cs.state?.waiting && ['CrashLoopBackOff', 'ErrImagePull', 'ImagePullBackOff'].includes(cs.state.waiting.reason || '')) {
          failingPods.push({
            pod_name: pod.metadata?.name,
            restart_count: cs.restartCount,
            reason: cs.state.waiting.reason,
            message: cs.state.waiting.message
          });
        }
      }
    }

    if (failingPods.length === 0) {
      return { content: [{ type: 'text', text: `All pods in namespace '${namespace}' are healthy.` }] };
    }

    return {
      content: [{ type: 'text', text: `Found ${failingPods.length} failing pods:\n` + JSON.stringify(failingPods, null, 2) }]
    };
  }

  if (name === 'aws_query_cloudwatch_insights') {
    const { log_group_name, query_string, time_window_minutes } = args as any;
    const startTime = Math.floor((Date.now() - (time_window_minutes * 60 * 1000)) / 1000);
    const endTime = Math.floor(Date.now() / 1000);

    const startCmd = new StartQueryCommand({
      logGroupName: log_group_name,
      queryString: query_string,
      startTime,
      endTime
    });

    const startRes = await cwClient.send(startCmd);
    const queryId = startRes.queryId;

    // Poll for query completion
    let isComplete = false;
    let results: any = null;

    for (let i = 0; i < 15; i++) {
      await new Promise(r => setTimeout(r, 1000));
      const getRes = await cwClient.send(new GetQueryResultsCommand({ queryId }));
      if (getRes.status === 'Complete') {
        isComplete = true;
        results = getRes.results;
        break;
      }
    }

    if (!isComplete) {
      return { isError: true, content: [{ type: 'text', text: 'CloudWatch query timed out after 15 seconds.' }] };
    }

    // Format output as condensed log lines
    const formatted = (results || []).map((row: any[]) => {
      const entry: Record<string, string> = {};
      row.forEach(field => { if (field.field) entry[field.field] = field.value || ''; });
      return `${entry['@timestamp']} - ${entry['@message']}`;
    }).join('\n');

    return { content: [{ type: 'text', text: formatted || 'Query completed: 0 matching log lines found.' }] };
  }

  throw new Error(`Tool not found: ${name}`);
});

async function main() {
  const transport = new StdioServerTransport();
  await server.connect(transport);
}

main();

{
  "jsonrpc": "2.0",
  "id": "remediate-41",
  "method": "tools/call",
  "params": {
    "name": "k8s_restart_deployment",
    "arguments": {
      "namespace": "production",
      "deployment_name": "checkout-service"
    }
  }
}

{
  "jsonrpc": "2.0",
  "id": "remediate-41",
  "result": {
    "isError": true,
    "content": [
      {
        "type": "text",
        "text": "ACTION REQUIRED: Restarting 'production/checkout-service' requires SRE human approval. Challenge Token: 'CHALLENGE_9a81f3'. Please approve in the SRE Slack portal or re-invoke with argument: confirmation_token."
      }
    ]
  }
}