Resiliente Retry-Policies für verteilte Systeme entwerfen

interface RetryConfig {
  maxRetries: number;
  baseDelayMs: number;
  maxDelayMs: number;
  jitterStrategy: "full" | "equal" | "decorrelated";
}
 
function calculateDelay(
  attempt: number,
  config: RetryConfig
): number {
  const exponentialDelay = config.baseDelayMs * Math.pow(2, attempt);
  const capped = Math.min(exponentialDelay, config.maxDelayMs);
 
  switch (config.jitterStrategy) {
    case "full":
      // Random between 0 and capped delay
      return Math.random() * capped;
 
    case "equal":
      // Half fixed, half random
      return capped / 2 + (Math.random() * capped) / 2;
 
    case "decorrelated":
      // Each delay is random between base and 3× the previous delay
      return Math.min(
        config.maxDelayMs,
        config.baseDelayMs + Math.random() * (capped * 3 - config.baseDelayMs)
      );
  }
}
 
// ❌ Immediate retry — hammers the failing service
async function badRetry<T>(fn: () => Promise<T>): Promise<T> {
  for (let i = 0; i < 5; i++) {
    try { return await fn(); }
    catch { continue; }
  }
  throw new Error("All retries failed");
}
 
// ✅ Exponential backoff with jitter
async function retryWithBackoff<T>(
  fn: () => Promise<T>,
  config: RetryConfig
): Promise<T> {
  let lastError: Error | undefined;
 
  for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
    try {
      return await fn();
    } catch (error) {
      lastError = error as Error;
 
      if (!isRetryable(error)) throw error;
      if (attempt === config.maxRetries) break;
 
      const delay = calculateDelay(attempt, config);
      await sleep(delay);
    }
  }
 
  throw lastError;
}
 
function isRetryable(error: unknown): boolean {
  if (error instanceof HttpError) {
    // Retry server errors and rate limits, not client errors
    return error.status >= 500 || error.status === 429;
  }
  if (error instanceof NetworkError) return true;
  if (error instanceof TimeoutError) return true;
  return false;
}

type CircuitState = "closed" | "open" | "half-open";
 
class CircuitBreaker {
  private state: CircuitState = "closed";
  private failures: number = 0;
  private lastFailureTime: number = 0;
  private successesSinceHalfOpen: number = 0;
 
  constructor(
    private readonly config: {
      failureThreshold: number;
      resetTimeoutMs: number;
      halfOpenMaxAttempts: number;
    }
  ) {}
 
  async execute<T>(fn: () => Promise<T>): Promise<T> {
    if (this.state === "open") {
      if (Date.now() - this.lastFailureTime < this.config.resetTimeoutMs) {
        throw new CircuitOpenError("Circuit is open — request rejected");
      }
      // Transition to half-open
      this.state = "half-open";
      this.successesSinceHalfOpen = 0;
    }
 
    try {
      const result = await fn();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }
 
  private onSuccess(): void {
    if (this.state === "half-open") {
      this.successesSinceHalfOpen++;
      if (this.successesSinceHalfOpen >= this.config.halfOpenMaxAttempts) {
        this.state = "closed";
        this.failures = 0;
      }
    } else {
      this.failures = 0;
    }
  }
 
  private onFailure(): void {
    this.failures++;
    this.lastFailureTime = Date.now();
 
    if (this.state === "half-open") {
      this.state = "open";
    } else if (this.failures >= this.config.failureThreshold) {
      this.state = "open";
    }
  }
 
  getState(): CircuitState {
    return this.state;
  }
}

class RetryBudget {
  private requestCount: number = 0;
  private retryCount: number = 0;
  private window: number[] = [];
 
  constructor(
    private readonly config: {
      maxRetryRatio: number;     // e.g., 0.2 = 20% of requests can be retries
      windowMs: number;          // Rolling window size
      minRequestsForBudget: number; // Need this many requests before enforcing
    }
  ) {}
 
  recordRequest(): void {
    this.cleanup();
    this.requestCount++;
    this.window.push(Date.now());
  }
 
  canRetry(): boolean {
    this.cleanup();
 
    if (this.requestCount < this.config.minRequestsForBudget) {
      return true; // Not enough data to enforce budget
    }
 
    const retryRatio = this.retryCount / this.requestCount;
    return retryRatio < this.config.maxRetryRatio;
  }
 
  recordRetry(): void {
    this.retryCount++;
  }
 
  private cleanup(): void {
    const cutoff = Date.now() - this.config.windowMs;
    this.window = this.window.filter((t) => t > cutoff);
    this.requestCount = this.window.length;
  }
}

class ResilientHttpClient {
  private circuitBreakers: Map<string, CircuitBreaker> = new Map();
  private retryBudget: RetryBudget;
 
  constructor(
    private readonly retryConfig: RetryConfig,
    budgetConfig: { maxRetryRatio: number; windowMs: number }
  ) {
    this.retryBudget = new RetryBudget({
      ...budgetConfig,
      minRequestsForBudget: 20,
    });
  }
 
  async request<T>(serviceKey: string, fn: () => Promise<T>): Promise<T> {
    const breaker = this.getCircuitBreaker(serviceKey);
    this.retryBudget.recordRequest();
 
    let lastError: Error | undefined;
 
    for (let attempt = 0; attempt <= this.retryConfig.maxRetries; attempt++) {
      try {
        return await breaker.execute(fn);
      } catch (error) {
        lastError = error as Error;
 
        if (error instanceof CircuitOpenError) throw error;
        if (!isRetryable(error)) throw error;
        if (attempt === this.retryConfig.maxRetries) break;
 
        if (!this.retryBudget.canRetry()) {
          throw new RetryBudgetExhaustedError(
            "Retry budget exhausted — too many retries system-wide"
          );
        }
 
        this.retryBudget.recordRetry();
        const delay = calculateDelay(attempt, this.retryConfig);
        await sleep(delay);
      }
    }
 
    throw lastError;
  }
 
  private getCircuitBreaker(key: string): CircuitBreaker {
    if (!this.circuitBreakers.has(key)) {
      this.circuitBreakers.set(
        key,
        new CircuitBreaker({
          failureThreshold: 5,
          resetTimeoutMs: 30_000,
          halfOpenMaxAttempts: 3,
        })
      );
    }
    return this.circuitBreakers.get(key)!;
  }
}

function instrumentRetries(client: ResilientHttpClient, metrics: Metrics) {
  const originalRequest = client.request.bind(client);
 
  client.request = async function <T>(
    serviceKey: string,
    fn: () => Promise<T>
  ): Promise<T> {
    const start = performance.now();
    try {
      const result = await originalRequest(serviceKey, fn);
      metrics.increment("http_request_total", {
        service: serviceKey,
        status: "success",
      });
      return result;
    } catch (error) {
      metrics.increment("http_request_total", {
        service: serviceKey,
        status: "failure",
        error_type: (error as Error).constructor.name,
      });
      throw error;
    } finally {
      metrics.histogram("http_request_duration_ms", performance.now() - start, {
        service: serviceKey,
      });
    }
  };
}