The Circuit Breaker Pattern: Failing Gracefully

type CircuitState = "closed" | "open" | "half-open";
 
class CircuitBreaker<T> {
  private state: CircuitState = "closed";
  private failureCount = 0;
  private successCount = 0;
  private lastFailureTime = 0;
 
  constructor(
    private readonly fn: () => Promise<T>,
    private readonly options: {
      failureThreshold: number;   // Failures before opening
      resetTimeout: number;       // Ms before trying half-open
      halfOpenRequests: number;   // Successful requests to close
    }
  ) {}
 
  async execute(): Promise<T> {
    if (this.state === "open") {
      if (Date.now() - this.lastFailureTime > this.options.resetTimeout) {
        this.state = "half-open";
        this.successCount = 0;
      } else {
        throw new CircuitOpenError("Circuit breaker is open");
      }
    }
 
    try {
      const result = await this.fn();
 
      if (this.state === "half-open") {
        this.successCount++;
        if (this.successCount >= this.options.halfOpenRequests) {
          this.state = "closed";
          this.failureCount = 0;
        }
      } else {
        this.failureCount = 0;
      }
 
      return result;
    } catch (error) {
      this.failureCount++;
      this.lastFailureTime = Date.now();
 
      if (this.failureCount >= this.options.failureThreshold) {
        this.state = "open";
      }
 
      throw error;
    }
  }
 
  getState(): CircuitState {
    return this.state;
  }
}
 
class CircuitOpenError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "CircuitOpenError";
  }
}

// ❌ Direct call — 30s timeout when payment service is down
async function createOrder(order: OrderData): Promise<OrderResult> {
  const payment = await fetch("http://payment-service/charge", {
    method: "POST",
    body: JSON.stringify({ amount: order.total }),
    signal: AbortSignal.timeout(30000), // 30 seconds of waiting
  });
  return payment.json();
}
 
// ✅ Circuit breaker — fails immediately when service is known to be down
const paymentBreaker = new CircuitBreaker(
  () => fetch("http://payment-service/charge", {
    method: "POST",
    body: JSON.stringify({ amount: order.total }),
    signal: AbortSignal.timeout(5000),
  }).then(r => r.json()),
  {
    failureThreshold: 5,      // Open after 5 consecutive failures
    resetTimeout: 30000,       // Try again after 30 seconds
    halfOpenRequests: 3,       // Need 3 successes to fully close
  }
);
 
async function createOrder(order: OrderData): Promise<OrderResult> {
  try {
    return await paymentBreaker.execute();
  } catch (error) {
    if (error instanceof CircuitOpenError) {
      // Fail fast — return a meaningful response
      return {
        status: "pending",
        message: "Payment processing is temporarily unavailable. Your order has been saved and will be processed shortly.",
      };
    }
    throw error;
  }
}

// Strategy 1: Cached response
async function getProductPrice(productId: string): Promise<number> {
  try {
    return await pricingBreaker.execute();
  } catch (error) {
    if (error instanceof CircuitOpenError) {
      // Return cached price
      const cached = await cache.get(`price:${productId}`);
      if (cached) return Number(cached);
    }
    throw error;
  }
}
 
// Strategy 2: Queue for retry
async function processPayment(order: OrderData): Promise<PaymentResult> {
  try {
    return await paymentBreaker.execute();
  } catch (error) {
    if (error instanceof CircuitOpenError) {
      // Queue for later processing
      await paymentQueue.enqueue({
        orderId: order.id,
        amount: order.total,
        retryAfter: Date.now() + 60000,
      });
      return { status: "queued", message: "Payment will be processed when service recovers" };
    }
    throw error;
  }
}
 
// Strategy 3: Degraded response
async function getRecommendations(userId: string): Promise<Product[]> {
  try {
    return await recommendationBreaker.execute();
  } catch (error) {
    if (error instanceof CircuitOpenError) {
      // Return popular products instead of personalized recommendations
      return await getPopularProducts();
    }
    throw error;
  }
}

import { Counter, Gauge } from "prom-client";
 
const circuitStateGauge = new Gauge({
  name: "circuit_breaker_state",
  help: "Current state of circuit breaker (0=closed, 1=open, 2=half-open)",
  labelNames: ["service"],
});
 
const circuitTripsCounter = new Counter({
  name: "circuit_breaker_trips_total",
  help: "Number of times the circuit breaker has opened",
  labelNames: ["service"],
});
 
// Enhanced circuit breaker with metrics
class MonitoredCircuitBreaker<T> extends CircuitBreaker<T> {
  constructor(
    fn: () => Promise<T>,
    options: CircuitBreakerOptions,
    private readonly serviceName: string
  ) {
    super(fn, options);
  }
 
  async execute(): Promise<T> {
    const stateMap = { closed: 0, open: 1, "half-open": 2 };
    circuitStateGauge.set({ service: this.serviceName }, stateMap[this.getState()]);
 
    try {
      return await super.execute();
    } catch (error) {
      if (this.getState() === "open") {
        circuitTripsCounter.inc({ service: this.serviceName });
      }
      throw error;
    }
  }
}

// Low-latency, high-reliability service (payment processing)
const paymentBreaker = new CircuitBreaker(paymentCall, {
  failureThreshold: 3,        // Open quickly — payments are critical
  resetTimeout: 15000,         // Retry soon
  halfOpenRequests: 5,         // Require more successes to restore trust
});
 
// High-latency, best-effort service (recommendations)
const recommendationBreaker = new CircuitBreaker(recommendationCall, {
  failureThreshold: 10,        // More tolerant — recommendations are optional
  resetTimeout: 60000,          // Check less frequently
  halfOpenRequests: 2,          // Fewer successes needed
});