SLOs, SLIs und Error Budgets: ein praktischer Leitfaden

// Calculate error budget remaining
interface SLOConfig {
  target: number;       // e.g., 0.999 (99.9%)
  windowDays: number;   // e.g., 30
}
 
interface SLIMetrics {
  totalRequests: number;
  successfulRequests: number;
}
 
function calculateErrorBudget(config: SLOConfig, metrics: SLIMetrics) {
  const currentSLI = metrics.successfulRequests / metrics.totalRequests;
  const allowedFailureRate = 1 - config.target;
  const allowedFailures = Math.floor(metrics.totalRequests * allowedFailureRate);
  const actualFailures = metrics.totalRequests - metrics.successfulRequests;
  const budgetRemaining = allowedFailures - actualFailures;
  const budgetPercentUsed = (actualFailures / allowedFailures) * 100;
 
  return {
    currentSLI: (currentSLI * 100).toFixed(3) + "%",
    allowedFailures,
    actualFailures,
    budgetRemaining,
    budgetPercentUsed: budgetPercentUsed.toFixed(1) + "%",
    withinBudget: budgetRemaining >= 0,
  };
}

# ❌ Infrastructure metrics as SLIs — don't reflect user experience
slis:
  - name: "CPU utilization"
    target: "below 70%"
  - name: "Memory usage"
    target: "below 80%"
  # CPU can be at 90% while users are perfectly happy
 
# ✅ User-facing metrics as SLIs
slis:
  - name: "Availability"
    description: "Proportion of requests that return non-5xx responses"
    good_event: "response.status_code < 500"
    total_event: "all requests"
    target: 99.9%
    window: 30d
 
  - name: "Latency"
    description: "Proportion of requests served within 500ms"
    good_event: "response.duration_ms <= 500"
    total_event: "all requests"
    target: 95%
    window: 30d
 
  - name: "Correctness"
    description: "Proportion of responses that pass data validation"
    good_event: "response.body passes schema validation"
    total_event: "all successful requests"
    target: 99.99%
    window: 30d

# Prometheus recording rules for SLI computation
groups:
  - name: sli-recording
    interval: 1m
    rules:
      # Availability SLI - ratio of successful requests
      - record: sli:availability:ratio_rate5m
        expr: |
          sum(rate(http_requests_total{status_code!~"5.."}[5m]))
          /
          sum(rate(http_requests_total[5m]))
 
      # Latency SLI - ratio of fast requests
      - record: sli:latency:ratio_rate5m
        expr: |
          sum(rate(http_request_duration_seconds_bucket{le="0.5"}[5m]))
          /
          sum(rate(http_request_duration_seconds_count[5m]))
 
  - name: slo-alerts
    rules:
      # Alert when error budget burn rate is too high
      # Burns through 5% of monthly budget in 1 hour
      - alert: SLOBudgetBurnHigh
        expr: |
          (
            1 - sli:availability:ratio_rate5m
          ) > (14.4 * (1 - 0.999))
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error budget burn rate - paging on-call"

## Error Budget Policy
 
### When budget is healthy (>50% remaining):
- Ship features at normal velocity
- Run chaos engineering experiments
- Perform infrastructure migrations
 
### When budget is concerning (25-50% remaining):
- Reduce deployment frequency
- Increase testing requirements for changes
- Review recent incidents for patterns
 
### When budget is critical (<25% remaining):
- Feature freeze — only reliability improvements and critical fixes
- All engineering effort focused on error reduction
- Post-incident reviews required for every budget-consuming event
 
### When budget is exhausted (0% remaining):
- Complete deployment freeze
- Roll back any recent changes that correlate with budget consumption
- Leadership review of reliability roadmap

# Fast burn: consuming budget 14.4x faster than sustainable
# Detected in 1 hour, confirmed over 5 minutes
- alert: SLOBudgetBurnFast
  expr: |
    (
      (1 - sli:availability:ratio_rate1h) > 14.4 * 0.001
    )
    and
    (
      (1 - sli:availability:ratio_rate5m) > 14.4 * 0.001
    )
  labels:
    severity: critical
 
# Slow burn: consuming budget 3x faster than sustainable
# Detected over 6 hours, confirmed over 30 minutes
- alert: SLOBudgetBurnSlow
  expr: |
    (
      (1 - sli:availability:ratio_rate6h) > 3 * 0.001
    )
    and
    (
      (1 - sli:availability:ratio_rate30m) > 3 * 0.001
    )
  labels:
    severity: warning