Chaos Engineering: Breaking Things on Purpose

## Chaos experiment template
 
### 1. Steady State Hypothesis
Define what "normal" looks like using measurable metrics.
"Our checkout flow processes orders with P99 latency 
under 2 seconds and error rate below 0.1%"
 
### 2. Hypothesis
What do you expect to happen when the failure occurs?
"If the payment service becomes unreachable, the 
checkout flow should retry 3 times and then return 
a user-friendly error within 10 seconds. No orders 
should be double-charged."
 
### 3. Experiment Design
- What failure are you injecting?
- What's the blast radius? (percentage of traffic)
- How long will the experiment run?
- What's the abort condition?
 
### 4. Run the Experiment
Inject the failure and observe.
 
### 5. Analyze Results
Did the system behave as hypothesized?
If not, what failed and why?
 
### 6. Fix and Rerun
Address any issues discovered and run the 
experiment again to verify the fix.

// Game day experiment: kill a service instance
interface GameDayExperiment {
  name: string;
  description: string;
  steadyState: SteadyStateDefinition;
  action: FailureAction;
  duration: Duration;
  abortConditions: AbortCondition[];
  owners: string[];
}
 
const firstExperiment: GameDayExperiment = {
  name: 'Payment service instance failure',
  description:
    'Kill one of three payment service replicas and verify ' +
    'traffic redistributes without user impact.',
  steadyState: {
    metrics: [
      { name: 'checkout_success_rate', operator: 'gte', value: 99.9 },
      { name: 'checkout_p99_latency_ms', operator: 'lte', value: 2000 },
      { name: 'payment_error_rate', operator: 'lte', value: 0.1 },
    ],
    verifyBeforeStart: true,
    verifyDuringExperiment: true,
  },
  action: {
    type: 'kill-pod',
    target: 'payment-service',
    count: 1,
    totalReplicas: 3,
  },
  duration: { minutes: 15 },
  abortConditions: [
    { metric: 'checkout_success_rate', operator: 'lt', value: 99.0 },
    { metric: 'checkout_p99_latency_ms', operator: 'gt', value: 5000 },
  ],
  owners: ['platform-team', 'payments-team'],
};

# Kubernetes-based failure injection examples
 
# 1. Pod failure: kill a running instance
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: payment-pod-kill
spec:
  action: pod-kill
  mode: one
  selector:
    namespaces: [production]
    labelSelectors:
      app: payment-service
  duration: "5m"
  scheduler:
    cron: "@every 2h"  # Recurring experiment
 
---
# 2. Network delay: add latency to service calls
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: database-latency
spec:
  action: delay
  mode: all
  selector:
    namespaces: [production]
    labelSelectors:
      app: order-service
  delay:
    latency: "500ms"
    jitter: "100ms"
  direction: to
  target:
    selector:
      namespaces: [production]
      labelSelectors:
        app: postgres
  duration: "10m"
 
---
# 3. Network partition: isolate a service
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: cache-partition
spec:
  action: partition
  mode: all
  selector:
    namespaces: [production]
    labelSelectors:
      app: api-gateway
  direction: both
  target:
    selector:
      namespaces: [production]
      labelSelectors:
        app: redis-cache
  duration: "5m"

// Application-level chaos: inject failures in code
// Useful for testing specific error paths
 
class ChaosMiddleware {
  private experiments: Map<string, ExperimentConfig> = new Map();
 
  middleware() {
    return (req: Request, res: Response, next: NextFunction) => {
      for (const [name, config] of this.experiments) {
        if (this.shouldApply(config, req)) {
          switch (config.type) {
            case 'latency':
              return setTimeout(next, config.delayMs);
            case 'error':
              return res.status(config.statusCode).json({
                error: 'Chaos experiment: simulated failure',
              });
            case 'timeout':
              // Don't respond at all — tests client timeout handling
              return;
          }
        }
      }
      next();
    };
  }
 
  private shouldApply(config: ExperimentConfig, req: Request): boolean {
    // Only apply to configured percentage of requests
    return (
      config.enabled &&
      Math.random() * 100 < config.percentageAffected &&
      config.pathPattern.test(req.path)
    );
  }
}

// Experiment monitoring dashboard
interface ExperimentMonitor {
  checkSteadyState(): Promise<SteadyStateResult>;
  shouldAbort(): Promise<boolean>;
  captureSnapshot(): Promise<MetricSnapshot>;
}
 
class ExperimentRunner {
  async run(experiment: GameDayExperiment): Promise<ExperimentResult> {
    const monitor = new ExperimentMonitorImpl(experiment);
 
    // Verify steady state before starting
    const baseline = await monitor.checkSteadyState();
    if (!baseline.healthy) {
      return {
        status: 'skipped',
        reason: 'System not in steady state before experiment',
        baseline,
      };
    }
 
    console.log(`Starting experiment: ${experiment.name}`);
    const startSnapshot = await monitor.captureSnapshot();
 
    // Inject the failure
    await this.injectFailure(experiment.action);
 
    // Monitor throughout the experiment
    const observations: MetricSnapshot[] = [];
    const checkInterval = setInterval(async () => {
      const snapshot = await monitor.captureSnapshot();
      observations.push(snapshot);
 
      // Auto-abort if conditions are breached
      if (await monitor.shouldAbort()) {
        console.log('ABORT: conditions breached, rolling back');
        await this.rollback(experiment.action);
        clearInterval(checkInterval);
      }
    }, 10_000); // Check every 10 seconds
 
    // Wait for experiment duration
    await this.wait(experiment.duration);
    clearInterval(checkInterval);
 
    // Remove the failure
    await this.rollback(experiment.action);
 
    // Capture recovery metrics
    await this.wait({ minutes: 2 });
    const recoverySnapshot = await monitor.captureSnapshot();
 
    return {
      status: 'completed',
      baseline: startSnapshot,
      observations,
      recovery: recoverySnapshot,
      hypothesis: experiment.steadyState,
    };
  }
}

// Continuous chaos experiment pipeline
interface ContinuousChaosConfig {
  experiments: GameDayExperiment[];
  schedule: {
    // Run during business hours when engineers are available
    timezone: string;
    hours: { start: number; end: number };
    daysOfWeek: number[]; // 1-5 for weekdays
  };
  notifications: {
    onStart: string[];      // Slack channels
    onAbort: string[];      // PagerDuty + Slack
    onComplete: string[];   // Slack + email summary
  };
  safetyControls: {
    // Never run during deployments
    pauseDuringDeploys: boolean;
    // Never run during incidents
    pauseDuringIncidents: boolean;
    // Maximum concurrent experiments
    maxConcurrent: number;
    // Global kill switch
    emergencyStop: boolean;
  };
}

## Top findings from chaos experiments
 
### 1. Timeouts are wrong
- Default timeouts are too high (30s when 2s is appropriate)
- Some services have no timeout at all
- Cascading timeouts: A calls B calls C, each with 30s 
  timeout = 90s total wait
 
### 2. Retries amplify failures
- Service retries 3x on failure
- 10 callers each retry 3x = 30 requests to a struggling 
  service that can barely handle 10
- Missing exponential backoff and jitter
 
### 3. Circuit breakers don't trip
- Configured but never tested
- Thresholds set too high to ever trigger
- No fallback behavior defined
 
### 4. Health checks lie
- Service reports healthy while database is unreachable
- Liveness probe passes but readiness should fail
- Health endpoint doesn't check critical dependencies
 
### 5. Graceful degradation paths don't exist
- Cache fails → error instead of slower database query
- Search fails → blank page instead of basic list view
- Payment fails → no way to queue order for retry