Speculative Tool Execution in MCP Runtimes

Standard Sequential Execution:
LLM Stream Arguments ────► Finish JSON ────► Dispatch MCP ────► Wait DB ────► LLM Turn 2
[ 1,200 ms ]              [ 0 ms ]         [ 50 ms ]         [ 400 ms ]     [ 800 ms ]
Total Latency: 2,450 ms

Speculative Concurrent Execution:
LLM Stream (Prefix Match) ────► Speculative Pre-fetch (DB)
[ 400 ms ]                     [ Runs in parallel 400ms ]
                         └────► Argument Match Confirmed ────► Instant Cache Hit!
Total Latency: 1,350 ms (45% Speedup!)

// speculative_mcp_runtime.ts
export interface SpeculativeToolPolicy {
  name: string;
  is_idempotent: boolean;
  can_speculate: boolean;
  max_prefetch_ms: number;
}

export const SPECULATIVE_POLICIES: Record<string, SpeculativeToolPolicy> = {
  "postgres_query": {
    name: "postgres_query",
    is_idempotent: true,
    can_speculate: true,
    max_prefetch_ms: 1000
  },
  "brave_web_search": {
    name: "brave_web_search",
    is_idempotent: true,
    can_speculate: true,
    max_prefetch_ms: 1500
  },
  "github_create_pull_request": {
    name: "github_create_pull_request",
    is_idempotent: false,
    can_speculate: false, // Strictly blocked from speculative execution
    max_prefetch_ms: 0
  }
};

// streaming_speculator.ts
import { Client } from "@modelcontextprotocol/sdk/client/index.js";

export class SpeculativeMCPRunner {
  private mcpClient: Client;
  private speculativePromise: Promise<any> | null = null;
  private speculativeArgsHash: string = "";

  constructor(client: Client) {
    this.mcpClient = client;
  }

  onStreamingToken(partialJsonText: string) {
    // Attempt early extraction of tool parameters from streaming buffer
    if (this.speculativePromise) return; // Speculation already in-flight

    try {
      // Look for completed query fields early in the stream
      const queryMatch = partialJsonText.match(/"sql":\s*"([^"]{10,})"/);
      if (queryMatch && queryMatch[1]) {
        const speculativeSql = queryMatch[1];
        this.speculativeArgsHash = speculativeSql;

        console.log(`⚡ Speculatively pre-fetching SQL: ${speculativeSql}`);
        this.speculativePromise = this.mcpClient.callTool({
          name: "postgres_query",
          arguments: { sql: speculativeSql }
        });
      }
    } catch {
      // Parsing intermediate stream tokens
    }
  }

  async resolveFinalToolCall(finalToolName: string, finalArgs: any) {
    // Verify whether our speculative pre-fetch matches the final arguments
    if (this.speculativePromise && finalArgs.sql === this.speculativeArgsHash) {
      console.log("🎯 Speculative Cache HIT! Returning pre-computed result.");
      const result = await this.speculativePromise;
      this.reset();
      return result;
    }

    // Cache miss or non-speculative tool: execute standard synchronous call
    console.log("⚠️ Speculative Cache MISS: Executing standard call.");
    this.reset();
    return await this.mcpClient.callTool({
      name: finalToolName,
      arguments: finalArgs
    });
  }

  private reset() {
    this.speculativePromise = null;
    this.speculativeArgsHash = "";
  }
}