Monitoring and Alerting Strategies That Reduce Alert Fatigue

// ❌ Cause-based alert — fires constantly, rarely actionable
const cpuAlert = {
  name: "High CPU Usage",
  condition: "cpu_usage > 80%",
  // Fires 50 times a week, ignored by day 3
};
 
// ✅ Symptom-based alert — fires when users are affected
interface SLOAlert {
  name: string;
  sli: string; // Service Level Indicator
  objective: number; // e.g., 99.9%
  window: string; // Measurement window
  burnRate: number; // How fast we're consuming error budget
  severity: "warning" | "critical";
}
 
const checkoutAvailability: SLOAlert = {
  name: "Checkout success rate below SLO",
  sli: "successful_checkouts / total_checkout_attempts",
  objective: 99.9,
  window: "30d",
  burnRate: 14.4, // Exhausts 30d budget in 2 hours
  severity: "critical",
};
 
const apiLatency: SLOAlert = {
  name: "API p99 latency exceeds SLO",
  sli: "requests_under_500ms / total_requests",
  objective: 99.0,
  window: "30d",
  burnRate: 6, // Exhausts 30d budget in 5 hours
  severity: "warning",
};

interface MultiWindowBurnRate {
  longWindow: { duration: string; burnRate: number };
  shortWindow: { duration: string; burnRate: number };
  severity: "page" | "ticket";
}
 
// Both windows must fire simultaneously to trigger the alert
const alertPolicies: MultiWindowBurnRate[] = [
  {
    // Fast burn: detect outages within minutes
    longWindow: { duration: "1h", burnRate: 14.4 },
    shortWindow: { duration: "5m", burnRate: 14.4 },
    severity: "page", // Wake someone up
  },
  {
    // Slow burn: detect gradual degradation
    longWindow: { duration: "6h", burnRate: 6 },
    shortWindow: { duration: "30m", burnRate: 6 },
    severity: "page",
  },
  {
    // Very slow burn: create a ticket, don't page
    longWindow: { duration: "3d", burnRate: 1 },
    shortWindow: { duration: "6h", burnRate: 1 },
    severity: "ticket",
  },
];
 
function evaluateBurnRate(
  errorCount: number,
  totalCount: number,
  sloTarget: number,
  windowHours: number,
  budgetWindowDays: number
): number {
  const errorRate = totalCount === 0 ? 0 : errorCount / totalCount;
  const errorBudget = 1 - sloTarget / 100;
  const windowFraction = windowHours / (budgetWindowDays * 24);
 
  return errorRate / (errorBudget * windowFraction);
}

interface AlertRoute {
  match: {
    severity: string[];
    services?: string[];
    labels?: Record<string, string>;
  };
  receivers: AlertReceiver[];
  muteWindows?: MuteWindow[];
  repeatInterval: string;
}
 
interface AlertReceiver {
  type: "pagerduty" | "slack" | "email" | "ticket";
  target: string;
}
 
interface MuteWindow {
  weekdays?: number[];
  startTime?: string;
  endTime?: string;
}
 
const routingConfig: AlertRoute[] = [
  {
    // Critical: always page
    match: { severity: ["critical"] },
    receivers: [
      { type: "pagerduty", target: "primary-oncall" },
      { type: "slack", target: "#incidents" },
    ],
    repeatInterval: "5m",
  },
  {
    // Warning: Slack during business hours, suppress overnight
    match: { severity: ["warning"] },
    receivers: [
      { type: "slack", target: "#alerts-warning" },
    ],
    muteWindows: [
      {
        weekdays: [1, 2, 3, 4, 5],
        startTime: "22:00",
        endTime: "08:00",
      },
      { weekdays: [6, 7] }, // Mute all weekend
    ],
    repeatInterval: "30m",
  },
  {
    // Info: create ticket, never page
    match: { severity: ["info"] },
    receivers: [
      { type: "ticket", target: "ops-backlog" },
    ],
    repeatInterval: "24h",
  },
];

// ❌ Alert with no context — responder starts from zero
// "ALERT: checkout_error_rate > 1%"
 
// ✅ Alert with full context for fast response
interface EnrichedAlert {
  title: string;
  description: string;
  severity: "critical" | "warning" | "info";
  runbookUrl: string;
  dashboardUrl: string;
  recentDeployments: Deployment[];
  impactEstimate: string;
  suggestedActions: string[];
}
 
function enrichAlert(
  rawAlert: RawAlert,
  deployments: Deployment[]
): EnrichedAlert {
  const recentDeploys = deployments.filter(
    (d) => Date.now() - d.timestamp < 60 * 60 * 1000
  );
 
  return {
    title: rawAlert.title,
    description: rawAlert.description,
    severity: rawAlert.severity,
    runbookUrl: `https://wiki.internal/runbooks/${rawAlert.alertName}`,
    dashboardUrl: `https://grafana.internal/d/${rawAlert.service}`,
    recentDeployments: recentDeploys,
    impactEstimate: estimateImpact(rawAlert),
    suggestedActions: [
      recentDeploys.length > 0
        ? `Recent deploy detected — consider rollback: ${recentDeploys[0].sha}`
        : "No recent deployments — investigate infrastructure",
      `Check dependency health: ${rawAlert.service}-dependencies`,
    ],
  };
}
 
function estimateImpact(alert: RawAlert): string {
  if (alert.affectedUsersPercent > 50) return "Major — >50% users affected";
  if (alert.affectedUsersPercent > 10) return "Moderate — 10-50% users affected";
  return "Minor — <10% users affected";
}

interface AlertGroup {
  groupKey: string;
  alerts: Alert[];
  firstFired: number;
  lastFired: number;
}
 
class AlertGrouper {
  private groups = new Map<string, AlertGroup>();
  private readonly GROUP_WINDOW = 5 * 60 * 1000; // 5 minutes
 
  addAlert(alert: Alert): AlertGroup {
    const groupKey = this.computeGroupKey(alert);
    const existing = this.groups.get(groupKey);
 
    if (existing && Date.now() - existing.lastFired < this.GROUP_WINDOW) {
      existing.alerts.push(alert);
      existing.lastFired = Date.now();
      return existing;
    }
 
    const group: AlertGroup = {
      groupKey,
      alerts: [alert],
      firstFired: Date.now(),
      lastFired: Date.now(),
    };
    this.groups.set(groupKey, group);
    return group;
  }
 
  private computeGroupKey(alert: Alert): string {
    // Group by service and alert type — not by instance
    return `${alert.service}:${alert.alertName}`;
  }
 
  getSummary(group: AlertGroup): string {
    return `[${group.alerts.length} alerts] ${group.groupKey} — ` +
      `first fired ${new Date(group.firstFired).toISOString()}`;
  }
}