Your Infrastructure
Heals Itself

id: "restart-payment-pods"
name: "Restart Payment Service Pods"
description: "Auto-restart payment pods on connection pool exhaustion"

triggers:
  - type: anomaly
    service_pattern: "payment-service"
    min_severity: warning
    cooldown_mins: 15

steps:
  - name: "Notify team"
    action: slack_notify
    params:
      channel: "#incidents"
      message: "Restarting payment-service pods (auto)"

  - name: "Approval gate"
    action: approval
    params:
      approvers: ["sre-team"]
      timeout: "5m"
      auto_approve_if: "confidence > 0.85"

  - name: "Restart pods"
    action: kubernetes_restart
    params:
      namespace: "production"
      deployment: "payment-service"
      strategy: "rolling"

  - name: "Verify recovery"
    action: wait_healthy
    params:
      service: "payment-service"
      timeout: "3m"

// LLM-generated root cause analysis
{
  "incident_id": "inc-1847",
  "service": "payment-service",
  "weirdness_score": 0.87,

  "root_cause": "Connection pool exhaustion on PostgreSQL.
    The db.insert span shows p99 latency of 2.4s (baseline: 45ms).
    Connection pool max_open_conns=25 is saturated under current
    load of 340 req/s. Upstream api-gateway is timing out and
    retrying, amplifying the problem.",

  "confidence": 0.91,
  "evidence": [
    "23 spans with db.insert > 2s in last 5min",
    "8 error logs: 'connection pool exhausted'",
    "Error rate jumped 12% → 67% at 14:31:45"
  ],

  "blast_radius": [
    "api-gateway (upstream, retrying)",
    "order-service (downstream, blocked)"
  ],

  "recommended_actions": [
    "Immediate: Restart payment-service pods",
    "Short-term: Increase max_open_conns to 50",
    "Long-term: Add connection pool metrics to monitoring"
  ]
}

groups:
  - name: infrasage-anomalies
    rules:
      - alert: ServiceAnomalyDetected
        expr: |
          infrasage_anomaly_weirdness_score
            > 0.7
          and
          infrasage_anomaly_consecutive_hits
            >= 3
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: >-
            Anomaly on {{ $labels.service_id }}
            (score: {{ $value | printf "%.2f" }})
          runbook: "https://wiki/runbooks/anomaly"

# Get current anomaly status for all services
$ curl -s http://infrasage:8080/api/v1/watchdog/summary \
    -H "X-API-Key: $API_KEY" | jq .

{
  "timestamp": "2025-01-15T14:32:15Z",
  "services": {
    "payment-service": {
      "status": "anomaly",
      "weirdness_score": 0.87,
      "rca_status": "completed",
      "incident_id": "inc-1847"
    },
    "api-gateway": {
      "status": "healthy",
      "weirdness_score": 0.12
    }
  },
  "total_services": 12,
  "anomalies": 1
}