m("checkout.latency_ms", metric.WithDescription("Checkout transaction latency in milliseconds"), metric.WithExplicitBucketBoundaries(50, 100, 250, 500, 1000, 2500, 5000)) }

func checkoutHandler(w http.ResponseWriter, r *http.Request) { ctx, span := tracer.Start(r.Context(), "checkout.process") defer span.End()

start := time.Now()
requests.Add(ctx, 1, attribute.String("region", r.Header.Get("X-Region")))

// Simulate checkout logic
err := processCheckout(ctx)
duration := time.Since(start).Seconds() * 1000
latency.Record(ctx, duration, attribute.String("status", map[bool]string{true: "success", false: "failure"}[err == nil]))

if err != nil {
    http.Error(w, "checkout failed", http.StatusInternalServerError)
    return
}
w.WriteHeader(http.StatusOK)

}

This instrumentation captures business-level success/failure and latency, tagged with user-relevant dimensions (region, status). Infrastructure metrics are deliberately excluded from the SLI definition.

#### 2. Prometheus Recording Rules for SLI Calculation

```yaml
# prometheus/rules/sli_checkout.yaml
groups:
  - name: checkout_sli_rules
    interval: 30s
    rules:
      # Success rate SLI (rolling 30d)
      - record: checkout:success_rate:30d
        expr: |
          sum(rate(checkout_requests_total{status="success"}[30d]))
          /
          sum(rate(checkout_requests_total[30d]))

      # p95 latency SLI (rolling 30d)
      - record: checkout:latency_p95:30d
        expr: |
          histogram_quantile(0.95,
            sum(rate(checkout_latency_ms_bucket[30d])) by (le)
          )

      # Composite SLI: both conditions must hold
      - record: checkout:sli_composite:30d
        expr: |
          (checkout:success_rate:30d >= 0.995) and (checkout:latency_p95:30d <= 2500)

Recording rules precompute SLIs, reducing query latency and ensuring consistent calculations across dashboards and alerting.

3. Multi-Burn-Rate Alerting Configuration

# prometheus/rules/burn_rate_alerts.yaml
groups:
  - name: checkout_burn_rate
    interval: 1m
    rules:
      # Fast burn: 1h window, 14.4x burn rate
      - alert: CheckoutErrorBudgetFastBurn
        expr: |
          (
            sum(rate(checkout_requests_total{status="failure"}[1h]))
            /
            sum(rate(checkout_requests_total[1h]))
          ) > (1 - 0.995) * 14.4
        for: 2m
        labels:
          severity: page
          burn_window: 1h
        annotations:
          summary: "Checkout error budget burning fast (14.4x)"
          description: "At current rate, error budget exhausted in <1h. Investigate immediately."

      # Slow burn: 6h window, 6x burn rate
      - alert: CheckoutErrorBudgetSlowBurn
        expr: |
          (
            sum(rate(checkout_requests_total{status="failure"}[6h]))
            /
            sum(rate(checkout_requests_total[6h]))
          ) > (1 - 0.995) * 6
        for: 5m
        labels:
          severity: ticket
          burn_window: 6h
        annotations:
          summary: "Checkout error budget burning slowly (6x)"
          description: "Error budget will deplete in ~6h if trend continues. Review recent deployments."

This configuration implements Google's multi-window burn rate strategy. Fast burn pages engineers; slow burn creates tickets. Both reference the same SLO (99.5% success) but measure consumption velocity differently.

4. CI/CD SLO Gate (GitHub Actions Example)

# .github/workflows/slo-validation.yml
name: SLO Validation Gate
on:
  pull_request:
    paths:
      - 'src/**'
      - 'deploy/**'

jobs:
  validate-slo:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Check SLO Compliance
        run: |
          # Query Prometheus for current SLO status
          SLO_STATUS=$(curl -s "http://prometheus:9090/api/v1/query?query=checkout:sli_composite:30d" | jq -r '.data.result[0].value[1]')
          if [ "$SLO_STATUS" != "1" ]; then
            echo "::error::SLO violation detected. Current composite SLI: $SLO_STATUS"
            exit 1
          fi
          echo "SLO validation passed. Proceeding with deployment."

This gate prevents merges when the SLO is currently violated, enforcing reliability as a first-class CI/CD concern.

Pitfall Guide (6 Critical Mistakes)

Production Bundle

1. Implementation Checklist

• Identify top 3 user journeys generating 80% of revenue/support tickets
• Map existing telemetry to journey steps; instrument missing business events
• Collect 30-90 days of historical success/latency data
• Calculate baseline p95/p99 metrics and correlate with support volume
• Negotiate SLO targets with product, support, and engineering leads
• Define composite SLIs (success + latency + throughput) with clear weightings
• Implement recording rules for SLI precomputation
• Configure multi-window burn rate alerting (fast/slow/medium)
• Integrate SLO validation into CI/CD pipeline (gate or warning)
• Document error budget policy: release pauses, rollback triggers, review cadence
• Schedule quarterly SLO calibration and business alignment review
• Audit alerting rules: remove threshold-based alerts, enforce burn rate only

2. Decision Matrix: SLI Selection Guide

3. Config Template: SLO Definition (YAML)

# slo-checkout.yaml
apiVersion: slo.monitoring.io/v1
kind: ServiceLevelObjective
metadata:
  name: checkout-slo
  namespace: production
spec:
  service: checkout-api
  window: 30d
  target: 0.995
  description: "Checkout success rate must remain above 99.5% over 30 days"
  indicators:
    - name: success_rate
      query: sum(rate(checkout_requests_total{status="success"}[$__window])) / sum(rate(checkout_requests_total[$__window]))
      weight: 0.7
    - name: latency_p95
      query: histogram_quantile(0.95, sum(rate(checkout_latency_ms_bucket[$__window])) by (le))
      threshold: 2500
      weight: 0.3
  errorBudget:
    total: 0.005
    burnRateAlerts:
      - window: 1h
        multiplier: 14.4
        severity: page
      - window: 6h
        multiplier: 6
        severity: ticket
  governance:
    owner: platform-reliability
    reviewers: [product-checkout, support-ecommerce]
    reviewCadence: quarterly

4. Quick Start: Zero to First SLO in 2 Hours

1. Hour 0:00-0:30 - Select one critical user journey (e.g., login or checkout). Identify the top 3 failure modes from support tickets.
2. Hour 0:30-1:00 - Instrument business-level success/latency metrics using OpenTelemetry or your APM vendor. Tag with user/session identifiers.
3. Hour 1:00-1:30 - Query historical data. Calculate baseline success rate and p95 latency. Set SLO at max(baseline - 0.005, business_minimum).
4. Hour 1:30-1:45 - Create Prometheus recording rules for SLI calculation. Validate with promtool check rules.
5. Hour 1:45-2:00 - Deploy burn rate alerting rules. Test with synthetic traffic or historical replay. Confirm alert routing matches severity policy. Document error budget guidelines in team wiki.

Closing Thoughts

SLO and SLI design is not a monitoring exercise; it is a reliability contract between engineering and the business. When designed correctly, SLOs transform telemetry from noise into signal, replace reactive firefighting with proactive budget management, and align engineering velocity with user experience. The principles outlined here—user-centric measurement, error budget policy, burn rate alerting, and continuous calibration—form the foundation of modern reliability engineering.

Implement them iteratively. Start with one critical journey. Validate with real user data. Enforce through automation. Review with business stakeholders. Reliability is not a destination; it is a continuously negotiated equilibrium between risk, velocity, and user trust. Build your SLOs to reflect that reality, and your systems will follow.