From Anomaly to
Root Cause in Minutes

id: "restart-payment-pods"
name: "Restart Payment Service Pods"
description: "Restart payment pods on connection pool exhaustion (approval-gated)"

triggers:
  - type: anomaly
    service_pattern: "payment-service"
    min_severity: warning
    cooldown_mins: 15

steps:
  - name: "Notify team"
    action: slack_notify
    params:
      channel: "#incidents"
      message: "Restarting payment-service pods (auto)"

  - name: "Approval gate"
    action: approval
    params:
      approvers: ["sre-team"]
      require_human: true

  - name: "Restart pods"
    action: kubernetes_restart
    params:
      namespace: "production"
      deployment: "payment-service"
      strategy: "rolling"

  - name: "Verify recovery"
    action: wait_healthy
    params:
      service: "payment-service"
      timeout: "3m"

// LLM-generated root cause analysis
{
  "incident_id": "inc-1847",
  "service": "payment-service",
  "verdict": "PUBLISHED",   // adversarially judged

  "root_cause": "Connection pool exhaustion on PostgreSQL.
    db.insert span latency rose well above baseline while the
    connection pool saturated under load. Upstream api-gateway is
    timing out and retrying, amplifying the problem.",

  "evidence": [
    "db.insert spans elevated above baseline [ev:1]",
    "logs: 'connection pool exhausted' [ev:2]",
    "error rate rising on payment-service [ev:3]"
  ],

  "blast_radius": [
    "api-gateway (upstream, retrying)",
    "order-service (downstream, blocked)"
  ],

  "recommended_actions": [
    "Immediate: Restart payment-service pods (approve in Slack)",
    "Short-term: Increase the connection pool size",
    "Long-term: Add connection-pool metrics to monitoring"
  ]
}

groups:
  - name: infrasage-anomalies
    rules:
      - alert: ServiceAnomalyDetected
        expr: |
          infrasage_anomaly_weirdness_score
            > 0.7
          and
          infrasage_anomaly_consecutive_hits
            >= 3
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: >-
            Anomaly on {{ $labels.service_id }}
            (score: {{ $value | printf "%.2f" }})
          runbook: "https://wiki/runbooks/anomaly"

# Get current anomaly status for all services
$ curl -s http://infrasage:8080/api/v1/watchdog/summary \
    -H "X-API-Key: $API_KEY" | jq .

{
  "timestamp": "2025-01-15T14:32:15Z",
  "services": {
    "payment-service": {
      "status": "anomaly",
      "rca_status": "completed",
      "incident_id": "inc-1847"
    },
    "api-gateway": {
      "status": "healthy"
    }
  },
  "total_services": 12,
  "anomalies": 1
}