Continuous Background Loops for AI Agents

┌────────────────────────────────────────────────────────────────────────┐
│                   INTERACTIVE VS. BACKGROUND AGENT LOOPS               │
├────────────────────────────┬───────────────────────────────────────────┤
│ Dimension                  │ Interactive Agent (/goal)                 │
├────────────────────────────┼───────────────────────────────────────────┤
│ Invocation Mode            │ Manual developer prompt via CLI / IDE     │
│ Lifecycle Duration         │ Ephemeral (minutes to hours)              │
│ Execution Concurrency      │ Single active session, synchronous wait   │
│ Failure Tolerance          │ Low: halts and prompts user directly      │
│ Human Intervention         │ Synchronous, blocking approval            │
├────────────────────────────┼───────────────────────────────────────────┤
│ Dimension                  │ Background Agent Daemon (/loop /schedule) │
├────────────────────────────┼───────────────────────────────────────────┤
│ Invocation Mode            │ Event-driven (Webhooks, PubSub, Cron)     │
│ Lifecycle Duration         │ Persistent / Always-on (days to months)   │
│ Execution Concurrency      │ Highly concurrent, asynchronous pool      │
│ Failure Tolerance          │ High: automated backoff, retry, isolation │
│ Human Intervention         │ Asynchronous, non-blocking via Slack/PR   │
└────────────────────────────┴───────────────────────────────────────────┘

Background Agent Daemon Architecture:
                                                                         
  Webhook Events ───────┐                                                
  (Sentry / GitHub)     │                                                
                        ▼                                                
  Cron Triggers ───▶ [Event Ingestion & Deduplication Gateway]           
  (Nightly Scans)       │                                                
                        ▼                                                
                     [Redis BullMQ Task Queue]                           
                        │                                                
         ┌──────────────┴──────────────┐                                 
         ▼                             ▼                                 
  [Worker Node A]               [Worker Node B]                          
  ┌────────────────────────┐    ┌────────────────────────┐               
  │ Parent Supervisor Daemon│   │ Parent Supervisor Daemon│               
  │   │                    │    │   │                    │               
  │   ├── Subagent 1 (Git) │    │   ├── Subagent 3 (DB)  │               
  │   └── Subagent 2 (Test)│    │   └── Subagent 4 (PR)  │               
  └───────────┬────────────┘    └───────────┬────────────┘               
              │                             │                            
              └──────────────┬──────────────┘                            
                             ▼                                           
              [Centralized MCP Gateway]                                  
         (Auth, Rate Limits, Sandboxed Stdio/SSE)                        
                             │                                           
        ┌────────────────────┼────────────────────┐                      
        ▼                    ▼                    ▼                      
  [GitHub MCP]         [Sentry MCP]         [Docker Test MCP]            

# Supervisor spawns an isolated worktree for Agent Task 8841
git worktree add -b agent/fix-issue-8841 /tmp/worktrees/task-8841 origin/main

git worktree remove --force /tmp/worktrees/task-8841
git branch -D agent/fix-issue-8841

import Redis from 'ioredis';
import { v4 as uuidv4 } from 'uuid';

export class DistributedAgentLock {
  private redis: Redis;
  private lockKey: string;
  private lockValue: string;
  private ttlMs: number;

  constructor(redis: Redis, resourceName: string, ttlMs: number = 60000) {
    this.redis = redis;
    this.lockKey = `locks:mcp:resource:${resourceName}`;
    this.lockValue = uuidv4();
    this.ttlMs = ttlMs;
  }

  public async acquire(): Promise<boolean> {
    // Acquire lock only if key does not exist (NX) with auto-expiry (PX)
    const result = await this.redis.set(this.lockKey, this.lockValue, 'PX', this.ttlMs, 'NX');
    return result === 'OK';
  }

  public async release(): Promise<boolean> {
    // Release lock only if the current value matches to prevent deleting expired locks
    const luaScript = `
      if redis.call("get", KEYS[1]) == ARGV[1] then
        return redis.call("del", KEYS[1])
      else
        return 0
      end
    `;
    const result = await this.redis.eval(luaScript, 1, this.lockKey, this.lockValue);
    return result === 1;
  }
}

┌────────────────────────────────────────────────────────────────────────┐
│                   SUPERVISOR AGENT DAEMON (ALWAYS ON)                  │
│  - Listens to task queue                                               │
│  - Tracks token budgets & timeouts                                     │
│  - Maintains liveness heartbeats                                       │
└───────────────────────────────────┬────────────────────────────────────┘
                                    │
           ┌────────────────────────┼────────────────────────┐
           ▼ Spawns                 ▼ Spawns                 ▼ Spawns
┌──────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐
│ INVESTIGATOR AGENT   │ │ BUILDER AGENT        │ │ REVIEWER AGENT       │
│ Scope: Read-Only     │ │ Scope: Write Local   │ │ Scope: Verification  │
│ Tools: Ripgrep, Tree │ │ Tools: Filesystem,   │ │ Tools: Docker, Jest, │
│ Lifespan: ~30s       │ │        AST Replace   │ │        Static Lint   │
│ Context: Clean 8k    │ │ Lifespan: ~60s       │ │ Lifespan: ~45s       │
└──────────┬───────────┘ └──────────┬───────────┘ └──────────┬───────────┘
           │                        │                        │
           └────────────────────────┼────────────────────────┘
                                    │ Emits Compact Result
                                    ▼
                         Supervisor Commits / PRs

/**
 * Production Continuous Background Agent Daemon.
 * Integrates Redis queues, Git Worktree isolation, and MCP Tool invocation.
 */

import { execSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import Redis from 'ioredis';

interface BackgroundTask {
  id: string;
  type: 'ERROR_TRIAGE' | 'DEPENDENCY_AUDIT' | 'PERFORMANCE_SCAN';
  targetRepo: string;
  issueContext: {
    title: string;
    stackTrace?: string;
    failingFile?: string;
  };
}

export class BackgroundAgentDaemon {
  private redis: Redis;
  private queueKey = 'queue:agent:background_tasks';
  private isRunning = false;
  private rootRepoPath: string;

  constructor(redisUrl: string, rootRepoPath: string) {
    this.redis = new Redis(redisUrl);
    this.rootRepoPath = rootRepoPath;
  }

  public async start(): Promise<void> {
    this.isRunning = true;
    console.log(`🤖 [DAEMON STARTED] Listening for background agent tasks on ${this.queueKey}...`);

    while (this.isRunning) {
      try {
        // Blocking pop with 5s timeout to allow clean shutdown
        const taskRaw = await this.redis.brpop(this.queueKey, 5);
        if (!taskRaw) continue;

        const task: BackgroundTask = JSON.parse(taskRaw[1]);
        await this.processTask(task);
      } catch (err) {
        console.error(`🚨 [DAEMON ERROR] Unhandled loop exception:`, err);
        await new Promise((res) => setTimeout(res, 2000));
      }
    }
  }

  public stop(): void {
    this.isRunning = false;
    console.log(`🛑 [DAEMON STOPPING] Awaiting in-flight subagents...`);
  }

  private async processTask(task: BackgroundTask): Promise<void> {
    console.log(`\n======================================================`);
    console.log(`⚡ [TASK RECEIVED] ID: ${task.id} | Type: ${task.type}`);
    console.log(`Context: ${task.issueContext.title}`);
    console.log(`======================================================`);

    const worktreePath = `/tmp/agent_worktrees/${task.id}`;
    const branchName = `agent/auto-fix-${task.id}`;

    try {
      // 1. Setup Isolated Git Worktree
      console.log(`  📁 Provisioning isolated Git worktree at: ${worktreePath}`);
      execSync(`git worktree add -b ${branchName} ${worktreePath} origin/main`, {
        cwd: this.rootRepoPath,
        stdio: 'pipe',
      });

      // 2. Invoke Autonomous Subagent Loop (Simulated MCP interaction)
      const success = await this.runSubagentWorkflow(worktreePath, task);

      if (success) {
        // 3. Push branch and open Pull Request
        console.log(`  🚀 Pushing branch ${branchName} to origin...`);
        execSync(`git push origin ${branchName}`, { cwd: worktreePath, stdio: 'pipe' });

        console.log(`  📬 Creating GitHub Pull Request via MCP...`);
        // In production: dispatch call to GitHub MCP (create_pull_request)
        console.log(`  ✅ PR Successfully Created: "fix(auto): resolve ${task.issueContext.title}"`);
      } else {
        console.warn(`  ⚠️ Subagent failed to safely resolve issue. Abandoning branch.`);
      }
    } catch (err: any) {
      console.error(`❌ Task processing failed: ${err.message}`);
    } finally {
      // 4. Teardown Worktree
      console.log(`  🧹 Cleaning up worktree: ${worktreePath}`);
      try {
        execSync(`git worktree remove --force ${worktreePath}`, {
          cwd: this.rootRepoPath,
          stdio: 'pipe',
        });
        execSync(`git branch -D ${branchName}`, {
          cwd: this.rootRepoPath,
          stdio: 'pipe',
        });
      } catch (cleanupErr) {
        console.warn(`Warning during worktree cleanup:`, cleanupErr);
      }
    }
  }

  private async runSubagentWorkflow(worktree: string, task: BackgroundTask): Promise<boolean> {
    console.log(`  🔍 [SUBAGENT INVESTIGATOR] Locating source of error in worktree...`);
    // Simulated investigation and patch application
    if (task.issueContext.failingFile) {
      const targetFilePath = path.join(worktree, task.issueContext.failingFile);
      if (fs.existsSync(targetFilePath)) {
        console.log(`  📝 [SUBAGENT BUILDER] Applying targeted bugfix patch...`);
        // Simulate safe patch application
        const content = fs.readFileSync(targetFilePath, 'utf8');
        fs.writeFileSync(targetFilePath, content + '\n// Hotfix applied by Background Agent Loop\n');

        console.log(`  🧪 [SUBAGENT REVIEWER] Running regression test suite in sandbox...`);
        // Verify changes compile
        execSync('git diff --stat', { cwd: worktree, stdio: 'inherit' });
        execSync('git commit -am "fix(agent): autonomous hotfix for triage"', { cwd: worktree });
        return true;
      }
    }
    return false;
  }
}

// Execution Entrypoint
if (require.main === module) {
  const REDIS_URL = process.env.REDIS_URL || 'redis://';
  const REPO_ROOT = path.resolve(__dirname, '..');

  const daemon = new BackgroundAgentDaemon(REDIS_URL, REPO_ROOT);
  daemon.start().catch(console.error);

  process.on('SIGINT', () => {
    daemon.stop();
    process.exit(0);
  });
}

                        HITL DECISION MATRIX
                        
       Autonomous Action Proposed by Background Loop
                            │
              ┌─────────────┴─────────────┐
              ▼                           ▼
      Low-Risk Action             High-Risk Action
  - Typo fix                  - Database schema drop
  - Dev-dependency bump       - IAM role mutation
  - Non-critical unit test    - Payment logic alteration
              │                           │
              ▼                           ▼
     [Auto-Merge & Commit]       [Non-Blocking HITL Gate]
                                          │
                                          ▼
                               Dispatches Slack / Email
                               Interactive Approval Card
                                          │
                             ┌────────────┴────────────┐
                             ▼                         ▼
                         [Approve]                  [Reject]
                             │                         │
                             ▼                         ▼
                        Execute PR                Discard Tree

{
  "channel": "#infra-approvals",
  "text": "🤖 *Background Agent Requesting Approval*",
  "attachments": [
    {
      "color": "#f2c744",
      "blocks": [
        {
          "type": "section",
          "text": {
            "type": "mrkdwn",
            "text": "*Task*: Auto-scale RDS Read Replicas (Sentry Lag Detected)\n*Reasoning*: Read queries on `/api/analytics` exceeded 800ms latency across 5 consecutive minutes.\n*Proposed Action*: Launch 1 new `db.m6g.xlarge` Aurora read replica."
          }
        },
        {
          "type": "actions",
          "elements": [
            {
              "type": "button",
              "text": { "type": "plain_text", "text": "Approve Action" },
              "style": "primary",
              "value": "approve_task_8841"
            },
            {
              "type": "button",
              "text": { "type": "plain_text", "text": "Reject & Rollback" },
              "style": "danger",
              "value": "reject_task_8841"
            }
          ]
        }
      ]
    }
  ]
}