Effective Incident Postmortems That Drive Change

# Incident Postmortem: [Title]
 
## Summary
- **Severity**: SEV-1 / SEV-2 / SEV-3
- **Duration**: [start time] to [resolution time]
- **Impact**: [number of affected users, revenue impact, data loss]
- **Detection**: [how was the incident discovered]
- **On-call responder**: [name]
 
## Timeline (UTC)
| Time  | Event |
|-------|-------|
| 14:23 | Deploy of payment-service v2.4.1 begins |
| 14:27 | Error rate spikes on /api/checkout (5% → 42%) |
| 14:31 | PagerDuty alert fires for payment-service p99 latency |
| 14:35 | On-call acknowledges alert, begins investigation |
| 14:42 | Identifies new deploy as probable cause |
| 14:45 | Initiates rollback to v2.4.0 |
| 14:51 | Rollback complete, error rate normalizing |
| 14:58 | Error rate back to baseline (0.3%) |
| 15:10 | Incident declared resolved |
 
## Contributing Factors
1. Missing integration test for payment retry logic
2. Feature flag not used for new payment provider path
3. Staging environment uses mock payment gateway (different behavior)
 
## Action Items
| Action | Owner | Priority | Due Date |
|--------|-------|----------|----------|
| Add integration test for payment retry with real sandbox | @alice | P1 | 2022-09-06 |
| Implement feature flag for payment provider switching | @bob | P1 | 2022-09-13 |
| Set up staging payment sandbox matching production | @infra | P2 | 2022-09-30 |

// ❌ Blame-oriented questions that shut people down
const badQuestions = [
  "Why didn't you check the logs before deploying?",
  "Who approved this PR without tests?",
  "Didn't you know this would break production?",
  "Why wasn't this caught in code review?",
];
 
// ✅ Learning-oriented questions that surface systemic issues
const goodQuestions = [
  "What information did you have at the time of the decision?",
  "What signals would have helped you catch this earlier?",
  "What made this failure mode possible in the first place?",
  "How can we make this class of error detectable before production?",
  "If this happened again tomorrow, what would help us resolve it faster?",
  "What surprised you about how the incident unfolded?",
];

interface PostmortemActionItem {
  id: string;
  description: string;
  owner: string;
  priority: "P0" | "P1" | "P2" | "P3";
  dueDate: string;
  category: ActionCategory;
  status: "open" | "in-progress" | "completed" | "wont-fix";
  incidentId: string;
}
 
type ActionCategory =
  | "detection"     // Improve monitoring/alerting
  | "prevention"    // Prevent this class of incident
  | "mitigation"    // Reduce impact when it happens
  | "process"       // Improve response procedures
  | "documentation"; // Update runbooks
 
// Good action items are specific and verifiable
const actionItemExamples = {
  // ❌ Vague action items that never get done
  bad: [
    "Improve monitoring",
    "Add more tests",
    "Be more careful with deploys",
    "Review processes",
  ],
  // ✅ Specific, measurable action items
  good: [
    "Add alert when payment error rate exceeds 5% for 2 minutes",
    "Write integration test covering payment retry with 503 response",
    "Add pre-deploy check that runs payment smoke test in staging",
    "Update payment service runbook with rollback command",
  ],
};

interface IncidentMetrics {
  incidentId: string;
  severity: "SEV-1" | "SEV-2" | "SEV-3";
  detectedAt: Date;
  acknowledgedAt: Date;
  resolvedAt: Date;
  impactMinutes: number;
  affectedUsers: number;
  detectionMethod: "alert" | "customer-report" | "internal-report" | "automated";
  actionItemsCreated: number;
  actionItemsCompleted: number;
}
 
function calculateIncidentMetrics(
  incidents: IncidentMetrics[]
): Record<string, number> {
  const metrics: Record<string, number> = {};
 
  // Mean time to detect (MTTD)
  const detectionTimes = incidents.map(
    (i) => i.acknowledgedAt.getTime() - i.detectedAt.getTime()
  );
  metrics.mttdMinutes =
    detectionTimes.reduce((a, b) => a + b, 0) /
    detectionTimes.length /
    60000;
 
  // Mean time to resolve (MTTR)
  const resolutionTimes = incidents.map(
    (i) => i.resolvedAt.getTime() - i.detectedAt.getTime()
  );
  metrics.mttrMinutes =
    resolutionTimes.reduce((a, b) => a + b, 0) /
    resolutionTimes.length /
    60000;
 
  // Action item completion rate
  const totalCreated = incidents.reduce(
    (sum, i) => sum + i.actionItemsCreated, 0
  );
  const totalCompleted = incidents.reduce(
    (sum, i) => sum + i.actionItemsCompleted, 0
  );
  metrics.actionCompletionRate =
    totalCreated > 0 ? totalCompleted / totalCreated : 0;
 
  // Percentage detected by alerts vs humans
  const alertDetected = incidents.filter(
    (i) => i.detectionMethod === "alert" || i.detectionMethod === "automated"
  ).length;
  metrics.automatedDetectionRate = alertDetected / incidents.length;
 
  return metrics;
}

## Five Whys — Payment Outage 2022-08-30
 
**Why did checkout fail for 28 minutes?**
→ The payment service returned 500 errors for requests
  to the new payment provider endpoint.
 
**Why did the payment service return 500 errors?**
→ The new provider retry logic had a bug that caused
  an infinite retry loop, exhausting connection pool.
 
**Why was the bug not caught before production?**
→ Integration tests use a mock payment gateway that
  always returns success. The retry path is not tested.
 
**Why don't integration tests cover the retry path?**
→ The staging environment uses a mock because setting up
  a payment sandbox requires coordination with the provider.
 
**Why hasn't the payment sandbox been set up in staging?**
→ No one owned the relationship with the payment provider
  for test environments. It was assumed mocks were sufficient.
 
→ **Root cause**: No realistic test environment for payment
  integration. Fix: Set up provider sandbox in staging and
  add integration tests for error/retry paths.

// Automate recurrence checking
interface RecurrenceCheck {
  incidentId: string;
  category: string;
  tags: string[];
  similarPastIncidents: string[];
}
 
function findSimilarIncidents(
  current: RecurrenceCheck,
  past: RecurrenceCheck[]
): RecurrenceCheck[] {
  return past.filter((p) => {
    const sharedTags = p.tags.filter((t) => current.tags.includes(t));
    const sameCategory = p.category === current.category;
    return sameCategory && sharedTags.length >= 2;
  });
}
 
// If similar incidents exist, the postmortem should reference them
// and explain why previous action items didn't prevent recurrence.
// This is the most valuable part of the postmortem: identifying
// patterns across incidents.

// A simple review cadence for postmortem follow-up
interface PostmortemReview {
  frequency: "weekly" | "biweekly";
  agenda: string[];
}
 
const weeklyReview: PostmortemReview = {
  frequency: "weekly",
  agenda: [
    "Review open action items from last 30 days",
    "Check if any P1 items are overdue",
    "Review incident count trend (are we improving?)",
    "Celebrate completed preventive measures",
    "Identify recurring incident patterns",
  ],
};
 
// Track the metrics that matter
const healthIndicators = {
  leading: [
    "Action item completion rate (target: >80%)",
    "Time from incident to postmortem (target: <5 business days)",
    "Percentage of incidents with postmortems (target: 100% for SEV-1/2)",
  ],
  lagging: [
    "Incident frequency per month (trending down?)",
    "Mean time to detect (MTTD) (trending down?)",
    "Mean time to resolve (MTTR) (trending down?)",
    "Recurrence rate (same root cause within 90 days)",
  ],
};