Chaos Engineering: Breaking Things on Purpose

// Structure of a chaos experiment
interface ChaosExperiment {
  name: string;
  hypothesis: string;
  steadyState: {
    metric: string;
    threshold: number;
    window: string;
  };
  method: {
    type: "latency" | "failure" | "resource" | "network";
    target: string;
    parameters: Record<string, unknown>;
    duration: string;
  };
  rollback: {
    automatic: boolean;
    trigger: string;
  };
}
 
const experiment: ChaosExperiment = {
  name: "Database failover under load",
  hypothesis: "When the primary database becomes unreachable, the application fails over to the replica within 30 seconds with less than 1% error rate increase",
  steadyState: {
    metric: "error_rate",
    threshold: 0.01,  // 1% error rate
    window: "5m",
  },
  method: {
    type: "network",
    target: "primary-db.internal",
    parameters: { action: "block", port: 5432 },
    duration: "5m",
  },
  rollback: {
    automatic: true,
    trigger: "error_rate > 0.05 for 2m",  // Abort if error rate exceeds 5%
  },
};

// Middleware that simulates downstream service failures
// Only active when CHAOS_MODE environment variable is set
function chaosMiddleware(serviceName: string) {
  return (req: Request, res: Response, next: NextFunction) => {
    if (process.env.CHAOS_MODE !== "true") {
      return next();
    }
 
    const chaosConfig = getChaosConfig(serviceName);
 
    // Simulate latency injection
    if (chaosConfig.latency && Math.random() < chaosConfig.latency.probability) {
      const delay = chaosConfig.latency.minMs +
        Math.random() * (chaosConfig.latency.maxMs - chaosConfig.latency.minMs);
      setTimeout(next, delay);
      return;
    }
 
    // Simulate error injection
    if (chaosConfig.error && Math.random() < chaosConfig.error.probability) {
      res.status(chaosConfig.error.statusCode).json({
        error: "Chaos injection: simulated failure",
        service: serviceName,
      });
      return;
    }
 
    next();
  };
}

# ❌ Testing only the happy path
# curl http://api-server:3000/health → 200 OK
# "Ship it, the service works!"
 
# ✅ Simulating real network conditions
 
# Add 200ms latency to traffic going to the database
tc qdisc add dev eth0 root netem delay 200ms 50ms distribution normal
 
# Simulate 10% packet loss to downstream service
tc qdisc add dev eth0 root netem loss 10%
 
# Simulate network partition (complete connectivity loss)
iptables -A OUTPUT -d database.internal -j DROP
 
# Clean up after experiment
tc qdisc del dev eth0 root
iptables -D OUTPUT -d database.internal -j DROP

// ❌ Assuming the circuit breaker works because the code looks right
const breaker = new CircuitBreaker(callExternalService, {
  timeout: 3000,
  errorThresholdPercentage: 50,
  resetTimeout: 30000,
});
 
// ✅ Chaos experiment to verify circuit breaker behavior
async function verifyCiruitBreaker() {
  const metrics = {
    totalRequests: 0,
    successfulRequests: 0,
    circuitOpenRejections: 0,
    timeouts: 0,
  };
 
  // Phase 1: Normal operation — circuit should be closed
  console.log("Phase 1: Verifying normal operation");
  for (let i = 0; i < 100; i++) {
    try {
      await breaker.fire();
      metrics.successfulRequests++;
    } catch (error) {
      // Should not happen in normal operation
    }
    metrics.totalRequests++;
  }
 
  // Phase 2: Inject failures — circuit should open
  console.log("Phase 2: Injecting failures");
  enableChaosMode({ error: { probability: 1.0, statusCode: 500 } });
 
  for (let i = 0; i < 50; i++) {
    try {
      await breaker.fire();
    } catch (error) {
      if (error.message === "Breaker is open") {
        metrics.circuitOpenRejections++;
      } else {
        metrics.timeouts++;
      }
    }
    metrics.totalRequests++;
  }
 
  // Phase 3: Recovery — circuit should close after reset timeout
  console.log("Phase 3: Verifying recovery");
  disableChaosMode();
  await sleep(35000); // Wait for reset timeout
 
  const recovered = await breaker.fire();
  console.log("Circuit recovered:", recovered !== undefined);
 
  return metrics;
}

## Gameday Checklist
 
### Before
- [ ] Define 3-5 specific experiments with hypotheses
- [ ] Ensure monitoring dashboards are visible to all participants
- [ ] Verify rollback procedures for each experiment
- [ ] Brief all participants on the plan and abort criteria
- [ ] Confirm the on-call rotation is staffed
 
### During
- [ ] Run one experiment at a time
- [ ] Record observations in a shared document
- [ ] Note any unexpected behaviors or cascading effects
- [ ] Use the incident response process (even if the "incident" is planned)
- [ ] Abort immediately if abort criteria are met
 
### After
- [ ] Document findings for each experiment
- [ ] File tickets for discovered issues (with severity based on blast radius)
- [ ] Update runbooks based on what was learned
- [ ] Schedule follow-up gameday to verify fixes

# Chaos experiment configuration with safety controls
experiment:
  name: "API latency injection"
  scope:
    environment: "production"
    percentage_of_traffic: 5    # Only affect 5% of requests
    target_service: "order-api"
    excluded_endpoints:         # Never inject chaos on critical paths
      - "/api/payments"
      - "/api/health"
 
  abort_conditions:
    - metric: "error_rate"
      threshold: 0.05           # Abort if errors exceed 5%
      window: "2m"
    - metric: "p99_latency_ms"
      threshold: 5000           # Abort if p99 exceeds 5 seconds
      window: "2m"
 
  duration: "15m"
  auto_rollback: true