import { CronJob } from 'cron';
import { OracleTelemetryClient } from './clients/OracleTelemetryClient';
import { MetricAggregator } from './processors/MetricAggregator';
import { TraceDiagnosticsEngine } from './diagnostics/TraceDiagnosticsEngine';

export class AgentTelemetryPipeline {
  private telemetryClient: OracleTelemetryClient;
  private aggregator: MetricAggregator;
  private diagnostics: TraceDiagnosticsEngine;

  constructor(config: TelemetryConfig) {
    this.telemetryClient = new OracleTelemetryClient(config.endpoint, config.credentials);
    this.aggregator = new MetricAggregator();
    this.diagnostics = new TraceDiagnosticsEngine();
  }

  public startScheduledAggregation(): void {
    // Oracle recommends 1-2 executions per day for metric consolidation
    const job = new CronJob('0 2,14 * * *', async () => {
      console.info('[Telemetry] Initiating ESS aggregation cycle...');
      await this.telemetryClient.triggerAggregationJob('Aggregate AI Agent Usage and Metrics');
      console.info('[Telemetry] ESS job dispatched successfully.');
    });
    job.start();
  }
}

export class MetricA

ggregator { public async processSessionBatch(rawSessions: RawSessionRecord[]): Promise<NormalizedMetrics> { const tokenBudget = await this.fetchTokenPricing(); const latencySamples: number[] = []; const costProjections: CostBreakdown = { inputTokens: 0, outputTokens: 0, estimatedCost: 0 };

const normalized = rawSessions.map(session => {
  const totalLatency = session.toolCalls.reduce((acc, call) => acc + call.durationMs, 0);
  latencySamples.push(totalLatency);

  const sessionTokens = session.llmInvocations.reduce((acc, inv) => acc + inv.tokenCount, 0);
  costProjections.inputTokens += sessionTokens;
  costProjections.estimatedCost += this.calculateTokenCost(sessionTokens, tokenBudget);

  return {
    sessionId: session.id,
    turnCount: session.turns,
    status: session.status,
    totalTokens: sessionTokens,
    p99LatencyMs: this.computeP99(latencySamples),
    costProjection: costProjections.estimatedCost
  };
});

return { sessions: normalized, aggregatedMetrics: costProjections };

}

private computeP99(samples: number[]): number { const sorted = [...samples].sort((a, b) => a - b); const index = Math.ceil(sorted.length * 0.99) - 1; return sorted[index] ?? 0; } }

**Architecture Rationale:** P99 latency is calculated per batch to reflect worst-case user experience rather than mathematical averages that mask tail latency. Token cost projection runs synchronously with metric normalization to enable real-time budget alerts. The separation of `MetricAggregator` and `TraceDiagnosticsEngine` ensures that high-level dashboards remain lightweight while deep trace analysis runs asynchronously.

### Step 3: Implement Trace-Level Bottleneck Detection

Drill-down diagnostics require parsing individual session traces to identify slow tool executions, redundant LLM calls, or token-heavy prompt cycles. The diagnostics engine extracts execution timelines and flags anomalies.

```typescript
export class TraceDiagnosticsEngine {
  public async analyzeSessionTrace(traceId: string): Promise<TraceReport> {
    const rawTrace = await this.telemetryClient.fetchSessionTrace(traceId);
    const bottlenecks: BottleneckAlert[] = [];

    for (const step of rawTrace.executionSteps) {
      if (step.type === 'tool_call' && step.durationMs > 5000) {
        bottlenecks.push({
          type: 'SLOW_TOOL',
          component: step.componentName,
          durationMs: step.durationMs,
          recommendation: 'Evaluate tool caching strategy or timeout thresholds.'
        });
      }
      if (step.type === 'llm_invocation' && step.tokenCount > 15000) {
        bottlenecks.push({
          type: 'HIGH_TOKEN_BURN',
          component: step.modelName,
          tokenCount: step.tokenCount,
          recommendation: 'Review system prompt length and tool output verbosity.'
        });
      }
    }

    return {
      traceId,
      totalTurns: rawTrace.turnCount,
      sessionStatus: rawTrace.status,
      bottlenecks,
      optimizationScore: this.calculateOptimizationScore(bottlenecks)
    };
  }
}

Architecture Rationale: Trace analysis is isolated from aggregation to prevent dashboard latency. The engine applies deterministic thresholds (5s tool timeout, 15k token limit per invocation) that align with enterprise SLAs. Recommendations are generated programmatically, enabling automated remediation workflows or CI/CD gating for prompt changes.

Pitfall Guide

1. Skipping the ESS Aggregation Schedule

Explanation: The monitoring dashboard remains empty until the Aggregate AI Agent Usage and Metrics job runs. Teams often assume real-time streaming is enabled by default. Fix: Implement a cron-based scheduler or CI/CD post-deployment hook that triggers the ESS job at least twice daily. Verify job completion status before querying metrics.

2. Optimizing for Average Latency Instead of P99

Explanation: Average latency masks tail failures. A 2-second average with a 16-second P99 means 1% of users experience severe degradation, which disproportionately impacts satisfaction and support volume. Fix: Configure alerts on P99 thresholds. Use trace diagnostics to isolate slow tool calls or model retries that drive tail latency.

3. Ignoring Draft Agent Telemetry

Explanation: Draft agents generate traffic during internal testing and QA. Excluding them from monitoring creates blind spots where inefficient prompts or misconfigured tools burn tokens before promotion. Fix: Enable draft agent tracking in the monitoring configuration. Run performance regression tests against draft versions before merging to production.

4. Treating Token Count as a Single Metric

Explanation: Total tokens obscure distribution patterns. High input tokens indicate verbose prompts or large context windows. High output tokens suggest inefficient tool routing or redundant LLM calls. Fix: Split token tracking into input/output categories. Map each category to specific optimization levers (prompt compression, tool output filtering, model selection).

5. Overlooking Tool-Level Latency Spikes

Explanation: Teams frequently blame the LLM for slow responses when external tools (APIs, databases, vector stores) are the actual bottleneck. Fix: Use the session trace view to isolate execution steps. Implement tool-level timeouts, connection pooling, and response caching. Alert on tool duration > 5s.

6. Alerting on Session Errors Without Context

Explanation: Session error spikes often correlate with upstream dependency failures, rate limits, or schema changes. Alerting without trace context leads to misdiagnosis. Fix: Correlate error rates with tool invocation logs and LLM response codes. Implement circuit breakers for failing tools and fallback prompt strategies.

7. Neglecting Token-to-Cost Mapping

Explanation: Oracle's pricing model ties compute costs directly to token consumption. Engineering teams often track tokens as a technical metric without translating them to financial impact. Fix: Build a cost projection layer that maps token volume to Oracle pricing tiers. Integrate budget thresholds into deployment pipelines to prevent uncontrolled spend.

Production Bundle

Action Checklist

• Schedule ESS job: Configure Aggregate AI Agent Usage and Metrics to run twice daily via cron or orchestration platform
• Enable draft tracking: Verify monitoring includes both published and draft agent teams
• Set time-window baselines: Establish 1-day, 7-day, 1-month, and 3-month performance benchmarks
• Configure P99 alerts: Trigger notifications when session latency exceeds 10,000ms for 99% of users
• Map token costs: Integrate Oracle pricing tiers into your telemetry pipeline for real-time cost projection
• Implement trace gating: Block prompt or tool changes in CI/CD if trace diagnostics show regression in optimization score
• Audit tool timeouts: Set maximum execution thresholds for external integrations and cache repetitive responses

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
High token consumption with stable latency	Prompt compression & tool output filtering	Reduces input/output volume without degrading response quality	Direct reduction in monthly Oracle compute billing
P99 latency spikes with low error rate	Tool-level caching & connection pooling	Addresses tail latency from external dependencies rather than model inference	Minimal cost increase; improves UX and reduces retry overhead
Draft agent showing 40% higher token burn	A/B test prompt variants & trace diagnostics	Isolates inefficient routing or verbose system instructions before promotion	Prevents cost overruns post-deployment; optimizes production baseline
Session error rate > 5%	Circuit breaker implementation & fallback prompts	Isolates failing tools and maintains session continuity	Reduces support ticket volume; avoids wasted token spend on failed turns

Configuration Template

# telemetry-scheduler.config.yaml
ess_job:
  name: "Aggregate AI Agent Usage and Metrics"
  schedule: "0 2,14 * * *"
  timeout_seconds: 300
  retry_policy:
    max_attempts: 3
    backoff_ms: 5000

monitoring:
  time_windows: [1d, 7d, 30d, 90d]
  include_draft_agents: true
  alert_thresholds:
    p99_latency_ms: 10000
    error_rate_percent: 5.0
    token_budget_daily: 500000

cost_mapping:
  pricing_tier: "oracle-ai-agent-standard"
  input_token_rate: 0.000005
  output_token_rate: 0.000015
  currency: "USD"

Quick Start Guide

1. Deploy the scheduler: Copy the configuration template into your orchestration environment. Ensure credentials for Oracle AI Agent Studio are injected securely.
2. Trigger initial aggregation: Manually run the ESS job once to populate the monitoring dashboard. Verify that draft and published agents appear in the aggregated view.
3. Validate trace diagnostics: Open a recent session, drill into the trace view, and confirm that tool calls, LLM invocations, and per-step latency are visible.
4. Configure alerts: Set P99 latency and token budget thresholds in your monitoring platform. Link alerts to your incident response channel for immediate visibility.
5. Integrate into CI/CD: Add a post-deployment step that triggers the ESS job and runs trace diagnostics against smoke-test sessions. Gate merges if optimization scores degrade.

Monitoring is not a post-launch checkbox. It is the operational backbone that transforms AI agents from experimental prototypes into governed, cost-aware, and performance-optimized production systems. By institutionalizing telemetry aggregation, trace-level diagnostics, and token-to-cost mapping, engineering teams gain the visibility required to scale LLM deployments without sacrificing reliability or budget control.