content: [{
        type: "text",
        text: JSON.stringify({
          total_events: events.length,
          cumulative_off_cpu_ms: events.reduce((sum, e) => sum + e.duration_ms, 0),
          top_contenders: this.extractTopContenders(events)
        })
      }]
    };
  }
);

this.server.tool(
  "resolve_causal_chain",
  "Maps a latency spike to kernel-level blocking events and stack traces",
  {
    target_pid: z.number().int().positive(),
    latency_window_s: z.number().min(1).max(300),
    include_stacks: z.boolean().default(true)
  },
  async ({ target_pid, latency_window_s, include_stacks }) => {
    const chain = await this.eventStore.buildCausalChain({
      pid: target_pid,
      windowSeconds: latency_window_s,
      attachStacks: include_stacks
    });

    return {
      content: [{
        type: "text",
        text: JSON.stringify({
          severity: chain.severity,
          blocking_mechanism: chain.mechanism,
          contention_source: chain.source_process,
          stack_trace: include_stacks ? chain.stack : null,
          recommended_action: this.deriveRemediation(chain)
        })
      }]
    };
  }
);

}

private extractTopContenders(events: OffCpuEvent[]): string[] { const contentionMap = new Map<string, number>(); events.forEach(e => { const key = ${e.blocking_pid}:${e.blocking_comm}; contentionMap.set(key, (contentionMap.get(key) || 0) + e.duration_ms); }); return Array.from(contentionMap.entries()) .sort((a, b) => b[1] - a[1]) .slice(0, 3) .map(([key]) => key); }

private deriveRemediation(chain: CausalChain): string { if (chain.mechanism.includes("futex_wait")) { return "Isolate competing threads via cgroup cpuset or pin critical paths to dedicated cores."; } if (chain.mechanism.includes("sched_switch")) { return "Reduce host-level CPU contention; consider workload co-location adjustments."; } return "Review kernel scheduler tuning and verify hardware interrupt affinity."; }

public getServer(): McpServer { return this.server; } }

### Step 4: Agent Orchestration Flow
The agent receives a symptom (e.g., TTFT spike) and routes calls sequentially:
1. Query application MCP for metrics/logs to establish the time window.
2. Query kernel MCP for off-CPU events and causal chains within that window.
3. Synthesize findings into a root-cause statement with actionable remediation.

### Architecture Rationale
- **Read-Only Enforcement**: Kernel-layer tools must never expose write capabilities. Agents investigate; human operators or separate automation pipelines execute fixes.
- **Event-Shaped Schema**: Pre-aggregated metrics lose causal context. Storing raw events allows the agent to reconstruct execution paths dynamically.
- **Per-Host Boundary**: eBPF data is inherently host-local. The MCP server should operate per-node, with cluster-wide visibility achieved through fan-out aggregation, not centralized indexing.
- **Fixed Overhead Targeting**: Probes attach only to high-signal libraries and tracepoints. This maintains sub-2% CPU overhead regardless of process count, making it safe for production inference clusters.

## Pitfall Guide

### 1. Metric-Bucket Schema Contamination
**Explanation**: Developers often force eBPF event data into pre-bucketed time series formats (e.g., 1-minute aggregates). This destroys tail latency visibility and breaks causal chain reconstruction.
**Fix**: Store raw events with millisecond precision. Use SQL aggregation tools within the MCP server to generate summaries on-demand, preserving the underlying event stream.

### 2. Over-Probing the Kernel
**Explanation**: Attaching to every syscall or tracepoint creates massive data volume, triggering storage bottlenecks and pushing overhead past safe thresholds.
**Fix**: Target specific uprobes (`libcudart.so`, `libcuda.so`) and high-signal tracepoints (`sched_switch`, `block:rq_issue`). Use PID/TID filters to scope probes to relevant workloads.

### 3. Cluster-Wide Indexing Assumption
**Explanation**: Teams attempt to centralize eBPF data into a single distributed index. This introduces network serialization costs, breaks per-host isolation, and complicates security boundaries.
**Fix**: Design the MCP server as a per-host daemon. Implement a lightweight fan-out orchestrator that aggregates per-node responses only when the agent requests cluster-wide context.

### 4. Ignoring CPU Pinning & Cgroup Isolation
**Explanation**: Monitoring threads, eBPF collectors, and agent runtimes compete with production workloads for CPU cycles. This creates artificial contention that the agent misdiagnoses as application issues.
**Fix**: Pin critical monitoring processes to dedicated cores using `taskset` or cgroup `cpuset`. Reserve isolated CPU sets for inference engines and diagnostic tooling.

### 5. Write-Access Exposure in Kernel Tools
**Explanation**: Granting agents remediation capabilities (e.g., `kill_process`, `modify_cgroup`) through the kernel MCP layer violates the principle of least privilege and introduces blast radius risks.
**Fix**: Enforce strict read-only contracts on kernel diagnostic tools. Separate investigation MCP servers from execution/automation pipelines. Require human approval or policy-gated automation for state changes.

### 6. Latency Blindness in Aggregation
**Explanation**: Relying on p50 or p90 metrics masks the actual bottleneck. GPU inference stalls and scheduler preemptions manifest in the p99/p999 tail.
**Fix**: Always expose p99 and p999 latency distributions alongside causal chain data. Configure the MCP server to return tail-event samples by default, not just aggregate counts.

### 7. Cross-Layer Schema Mismatch
**Explanation**: Application MCP servers return metric-shaped data (timestamps, values, labels). Kernel MCP servers return event-shaped data (PIDs, stacks, durations). Agents struggle to correlate them without explicit mapping.
**Fix**: Implement a correlation layer that maps application request IDs to kernel thread IDs and process boundaries. Include `trace_id` or `span_id` in eBPF uprobes where framework support exists.

## Production Bundle

### Action Checklist
- [ ] Deploy eBPF probes targeting `libcudart.so`, `libcuda.so`, and `sched_switch` tracepoints
- [ ] Configure event storage with millisecond precision and raw event retention
- [ ] Implement read-only MCP server with event-shaped tool contracts
- [ ] Pin monitoring and agent processes to isolated CPU sets via cgroups
- [ ] Establish per-host MCP daemon architecture with fan-out aggregation
- [ ] Map application trace IDs to kernel thread boundaries for cross-layer correlation
- [ ] Enforce strict separation between diagnostic (read-only) and remediation (write) tool surfaces
- [ ] Validate kernel overhead remains below 2% under peak workload conditions

### Decision Matrix

| Scenario | Recommended Approach | Why | Cost Impact |
|----------|---------------------|-----|-------------|
| Microservice latency spike | Standard App-Layer MCP + Distributed Tracing | Application boundaries are clear; SDK instrumentation is sufficient | Low (existing observability stack) |
| GPU inference TTFT stall | Kernel-Enhanced eBPF MCP + Application MCP | Bottleneck lives in scheduler preemption or CUDA sync waits; app telemetry is blind | Medium (eBPF deployment + per-host storage) |
| Network retransmit storm | eBPF `tcp:tcp_retransmit_skb` tracepoints + MCP | Kernel sees packet drops before application logs; causal chain requires socket-level context | Medium (targeted probe deployment) |
| Multi-tenant CPU contention | eBPF `sched_switch` + cgroup cpuset isolation | Cross-process contention requires kernel scheduler visibility; app metrics only show aggregate CPU | High (requires host-level policy enforcement) |
| Compliance/audit investigation | Standard App-Layer MCP with immutable logs | Audit trails require application-level context; kernel data is noisy and irrelevant | Low (leverages existing SIEM/audit pipelines) |

### Configuration Template

```yaml
# kernel-mcp-config.yaml
server:
  name: "kernel-diagnostic-bridge"
  version: "1.0.0"
  transport: "stdio"
  read_only: true

probes:
  uprobes:
    - library: "libcudart.so"
      symbol: "cudaLaunchKernel"
      attach: "entry"
    - library: "libcudart.so"
      symbol: "cudaDeviceSynchronize"
      attach: "entry"
  tracepoints:
    - name: "sched:sched_switch"
      filter: "prev_pid != 0 && next_pid != 0"
    - name: "block:block_rq_issue"
      filter: "rwbs ~ '.*W.*'"

storage:
  backend: "columnar"
  retention_hours: 72
  precision_ms: 1
  max_events_per_second: 50000

tools:
  - name: "query_off_cpu_events"
    description: "Retrieves scheduler preemption events"
    schema: "event"
  - name: "resolve_causal_chain"
    description: "Maps latency to kernel blocking mechanics"
    schema: "event"
  - name: "analyze_gpu_launch_latency"
    description: "Returns CUDA kernel launch distribution"
    schema: "event"

security:
  write_access: false
  pid_filtering: true
  rate_limit_per_second: 100

Quick Start Guide

1. Install eBPF Collector: Deploy the probe binary to the target host. Verify kernel headers are available and bpf syscall is unblocked.
2. Start Event Store: Initialize the columnar storage backend with the provided configuration. Confirm raw event ingestion is active.
3. Launch MCP Server: Run the TypeScript MCP bridge in stdio mode. Test tool discovery using mcp-cli list-tools.
4. Connect Agent: Configure your LLM agent to register both the application MCP and the kernel MCP. Execute a diagnostic query against a known latency window.
5. Validate Overhead: Monitor host CPU usage during probe execution. Confirm overhead remains under 2% and event throughput matches expected workload volume.