Monitoring- und Alerting-Strategien, die wirklich funktionieren

// Express middleware that captures the four golden signals
import { Counter, Histogram, Gauge } from "prom-client";
 
// 1. Latency — how long requests take
const httpDuration = new Histogram({
  name: "http_request_duration_seconds",
  help: "Duration of HTTP requests in seconds",
  labelNames: ["method", "route", "status_code"],
  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
});
 
// 2. Traffic — how many requests per second
const httpRequests = new Counter({
  name: "http_requests_total",
  help: "Total number of HTTP requests",
  labelNames: ["method", "route", "status_code"],
});
 
// 3. Errors — rate of failed requests
const httpErrors = new Counter({
  name: "http_errors_total",
  help: "Total number of HTTP errors",
  labelNames: ["method", "route", "error_type"],
});
 
// 4. Saturation — how full your resources are
const activeConnections = new Gauge({
  name: "http_active_connections",
  help: "Number of active HTTP connections",
});
 
app.use((req, res, next) => {
  activeConnections.inc();
  const end = httpDuration.startTimer({
    method: req.method,
    route: req.route?.path ?? req.path,
  });
 
  res.on("finish", () => {
    const labels = {
      method: req.method,
      route: req.route?.path ?? req.path,
      status_code: res.statusCode.toString(),
    };
    end(labels);
    httpRequests.inc(labels);
    activeConnections.dec();
 
    if (res.statusCode >= 500) {
      httpErrors.inc({ ...labels, error_type: "server_error" });
    }
  });
 
  next();
});

# Prometheus alerting rules based on SLIs
groups:
  - name: sli-alerts
    rules:
      # Availability SLI: proportion of successful requests
      - alert: HighErrorRate
        expr: |
          (
            sum(rate(http_errors_total[5m]))
            /
            sum(rate(http_requests_total[5m]))
          ) > 0.01
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Error rate exceeds 1% for 5 minutes"
          runbook: "https://wiki.internal/runbooks/high-error-rate"
 
      # Latency SLI: proportion of requests served within threshold
      - alert: HighLatency
        expr: |
          (
            sum(rate(http_request_duration_seconds_bucket{le="0.5"}[5m]))
            /
            sum(rate(http_request_duration_seconds_count[5m]))
          ) < 0.95
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Less than 95% of requests served within 500ms"

# ❌ Noisy alert — fires on any single error
- alert: AnyServerError
  expr: http_errors_total > 0
  labels:
    severity: critical
 
# ✅ Meaningful alert — fires on sustained elevated error rate
- alert: ElevatedErrorRate
  expr: |
    (
      sum(rate(http_errors_total[10m]))
      /
      sum(rate(http_requests_total[10m]))
    ) > 0.005
  for: 10m  # Must sustain for 10 minutes
  labels:
    severity: warning
  annotations:
    summary: "Error rate above 0.5% for 10 minutes"
    impact: "Approximately {{ $value | humanizePercentage }} of users affected"
    runbook: "https://wiki.internal/runbooks/elevated-errors"
    dashboard: "https://grafana.internal/d/api-overview"

// ❌ Unstructured logs — impossible to search or correlate
console.log("Error processing order for user " + userId);
 
// ✅ Structured JSON logs with correlation ID
import { randomUUID } from "crypto";
 
app.use((req, res, next) => {
  req.correlationId = req.headers["x-correlation-id"] as string ?? randomUUID();
  res.setHeader("x-correlation-id", req.correlationId);
  next();
});
 
function log(level: string, message: string, context: Record<string, unknown>) {
  console.log(JSON.stringify({
    timestamp: new Date().toISOString(),
    level,
    message,
    correlationId: context.correlationId,
    service: "order-service",
    ...context,
  }));
}
 
// Usage in route handler
app.post("/orders", async (req, res) => {
  log("info", "Processing order", {
    correlationId: req.correlationId,
    userId: req.body.userId,
    itemCount: req.body.items.length,
  });
 
  try {
    const order = await createOrder(req.body);
    log("info", "Order created", {
      correlationId: req.correlationId,
      orderId: order.id,
    });
    res.json(order);
  } catch (error) {
    log("error", "Order creation failed", {
      correlationId: req.correlationId,
      error: error instanceof Error ? error.message : String(error),
    });
    res.status(500).json({ error: "Order processing failed" });
  }
});