Benchmarking 20 Popular MCP Servers: Latency, Throughput & Cost Analysis

// benchmark-runner.ts
import { spawn } from 'child_process';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';

interface BenchmarkConfig {
  server: {
    name: string;
    transport: 'stdio' | 'sse';
    command?: string;
    args?: string[];
    url?: string;
  };
  tool: string;
  args: any;
  iterations: number;
  concurrency: number;
  warmup: number;
}

async function runBenchmark(config: BenchmarkConfig): Promise<BenchmarkResult> {
  const latencies: number[] = [];
  const errors: Error[] = [];
  let success = 0;
  
  // Create client pool
  const clients = await createClientPool(config.server, config.concurrency);
  
  // Warmup
  for (let i = 0; i < config.warmup; i++) {
    await clients[0].callTool({ name: config.tool, arguments: config.args });
  }
  
  // Benchmark
  const start = process.hrtime.bigint();
  
  await Promise.all(
    Array.from({ length: config.iterations }, (_, i) => 
      executeRequest(clients[i % clients.length], config.tool, config.args, latencies, errors)
    )
  );
  
  const end = process.hrtime.bigint();
  const durationMs = Number(end - start) / 1_000_000;
  
  return {
    server: config.server.name,
    transport: config.server.transport,
    tool: config.tool,
    totalRequests: config.iterations,
    successful: success,
    failed: errors.length,
    durationMs,
    throughput: config.iterations / (durationMs / 1000),
    latencies: {
      p50: percentile(latencies, 50),
      p90: percentile(latencies, 90),
      p95: percentile(latencies, 95),
      p99: percentile(latencies, 99),
      max: Math.max(...latencies),
    },
    coldStartMs: await measureColdStart(config.server),
    memoryMB: process.memoryUsage().heapUsed / 1024 / 1024,
  };
}

// Client-side: Reuse SSE connection across tool calls
class PooledSSEClient {
  private transport: SSEClientTransport | null = null;
  private client: Client | null = null;
  private lastUsed = 0;
  private readonly TTL = 5 * 60 * 1000; // 5 min
  
  async getClient(): Promise<Client> {
    const now = Date.now();
    
    if (this.transport && now - this.lastUsed < this.TTL) {
      this.lastUsed = now;
      return this.client!;
    }
    
    // Create new connection
    await this.close();
    
    this.transport = new SSEClientTransport(new URL(this.serverUrl));
    this.client = new Client({ name: 'benchmark-client', version: '1.0' });
    await this.client.connect(this.transport);
    this.lastUsed = now;
    
    return this.client;
  }
  
  async callTool(name: string, args: any) {
    const client = await this.getClient();
    return client.callTool({ name, arguments: args });
  }
}

Developer → Cursor/Claude → stdio → MCP Server → Local DB

Team → Cursor/Claude → SSE → MCP Gateway → Pooled Servers → Shared DB

Enterprise → Claude/Cursor → SSE → MCP Gateway (auth, rate limit, observability) → 
  → Server Fleet (K8s, auto-scaling) → Managed Services

server,transport,tool,p50_ms,p90_ms,p95_ms,p99_ms,max_ms,throughput_rps,cold_start_ms,memory_mb,cost_per_million_usd
filesystem,stdio,read_file,1.8,4.2,6.1,11.3,89,5200,38,42,0.12
filesystem,sse,read_file,14.2,38.7,56.3,108.4,445,1100,1840,68,2.80
memory,stdio,read_memory,0.9,2.1,3.0,5.8,42,8900,31,28,0.12
memory,sse,read_memory,12.8,34.1,49.8,95.2,398,1200,1720,52,2.80
postgres,stdio,query,3.2,8.7,12.4,24.1,198,2800,52,89,0.12
postgres,sse,query,22.4,61.3,89.7,172.8,834,680,2140,134,2.80
git,stdio,git_log,4.5,12.3,18.9,35.2,287,1900,67,78,0.12
github,sse,search_code,45.6,124.8,182.3,341.2,1823,290,2890,156,2.80
chromadb,stdio,search,18.7,48.2,71.3,134.2,891,420,156,234,0.12
chromadb,sse,search,134.5,367.2,521.8,943.4,3421,95,3420,412,2.80

# Clone benchmark suite
git clone https://github.com/mcp-codex/mcp-benchmarks
cd mcp-benchmarks

# Configure servers to test
cp config.example.yaml config.yaml
# Edit config.yaml with your server commands/URLs

# Run full suite
npm run benchmark -- --iterations 10000 --concurrency 50

# Generate report
npm run report -- --output benchmarks-$(date +%Y%m%d).html