Designing Resilient Retry Policies for Distributed Systems
Retry policies that recover from transient failures without overwhelming downstream services: exponential backoff, jitter, circuit breakers and budgets.
Retries Are Dangerous by Default
A naive retry—"if it fails, try again immediately"—is one of the most dangerous patterns in distributed systems. When a downstream service is overloaded, retries from hundreds of clients multiply the load, turning a partial degradation into a complete outage. Good retry policies help systems recover. Bad retry policies accelerate failure.
Exponential Backoff with Jitter
Exponential backoff spaces retries further apart with each attempt. Jitter adds randomness so clients do not retry in synchronized waves.
interface RetryConfig {
maxRetries: number;
baseDelayMs: number;
maxDelayMs: number;
jitterStrategy: "full" | "equal" | "decorrelated";
}
function calculateDelay(
attempt: number,
config: RetryConfig
): number {
const exponentialDelay = config.baseDelayMs * Math.pow(2, attempt);
const capped = Math.min(exponentialDelay, config.maxDelayMs);
switch (config.jitterStrategy) {
case "full":
// Random between 0 and capped delay
return Math.random() * capped;
case "equal":
// Half fixed, half random
return capped / 2 + (Math.random() * capped) / 2;
case "decorrelated":
// Each delay is random between base and 3× the previous delay
return Math.min(
config.maxDelayMs,
config.baseDelayMs + Math.random() * (capped * 3 - config.baseDelayMs)
);
}
}
// ❌ Immediate retry — hammers the failing service
async function badRetry<T>(fn: () => Promise<T>): Promise<T> {
for (let i = 0; i < 5; i++) {
try { return await fn(); }
catch { continue; }
}
throw new Error("All retries failed");
}
// ✅ Exponential backoff with jitter
async function retryWithBackoff<T>(
fn: () => Promise<T>,
config: RetryConfig
): Promise<T> {
let lastError: Error | undefined;
for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
try {
return await fn();
} catch (error) {
lastError = error as Error;
if (!isRetryable(error)) throw error;
if (attempt === config.maxRetries) break;
const delay = calculateDelay(attempt, config);
await sleep(delay);
}
}
throw lastError;
}
function isRetryable(error: unknown): boolean {
if (error instanceof HttpError) {
// Retry server errors and rate limits, not client errors
return error.status >= 500 || error.status === 429;
}
if (error instanceof NetworkError) return true;
if (error instanceof TimeoutError) return true;
return false;
}Circuit Breakers: Stop Calling What Is Broken
Retries keep calling a failing service. Circuit breakers stop calling it entirely for a cooldown period, giving the service time to recover.
type CircuitState = "closed" | "open" | "half-open";
class CircuitBreaker {
private state: CircuitState = "closed";
private failures: number = 0;
private lastFailureTime: number = 0;
private successesSinceHalfOpen: number = 0;
constructor(
private readonly config: {
failureThreshold: number;
resetTimeoutMs: number;
halfOpenMaxAttempts: number;
}
) {}
async execute<T>(fn: () => Promise<T>): Promise<T> {
if (this.state === "open") {
if (Date.now() - this.lastFailureTime < this.config.resetTimeoutMs) {
throw new CircuitOpenError("Circuit is open — request rejected");
}
// Transition to half-open
this.state = "half-open";
this.successesSinceHalfOpen = 0;
}
try {
const result = await fn();
this.onSuccess();
return result;
} catch (error) {
this.onFailure();
throw error;
}
}
private onSuccess(): void {
if (this.state === "half-open") {
this.successesSinceHalfOpen++;
if (this.successesSinceHalfOpen >= this.config.halfOpenMaxAttempts) {
this.state = "closed";
this.failures = 0;
}
} else {
this.failures = 0;
}
}
private onFailure(): void {
this.failures++;
this.lastFailureTime = Date.now();
if (this.state === "half-open") {
this.state = "open";
} else if (this.failures >= this.config.failureThreshold) {
this.state = "open";
}
}
getState(): CircuitState {
return this.state;
}
}Retry Budgets: System-Wide Limits
Individual retry configs do not account for aggregate load. A retry budget limits the total percentage of requests that can be retries, preventing amplification at the system level.
class RetryBudget {
private requestCount: number = 0;
private retryCount: number = 0;
private window: number[] = [];
constructor(
private readonly config: {
maxRetryRatio: number; // e.g., 0.2 = 20% of requests can be retries
windowMs: number; // Rolling window size
minRequestsForBudget: number; // Need this many requests before enforcing
}
) {}
recordRequest(): void {
this.cleanup();
this.requestCount++;
this.window.push(Date.now());
}
canRetry(): boolean {
this.cleanup();
if (this.requestCount < this.config.minRequestsForBudget) {
return true; // Not enough data to enforce budget
}
const retryRatio = this.retryCount / this.requestCount;
return retryRatio < this.config.maxRetryRatio;
}
recordRetry(): void {
this.retryCount++;
}
private cleanup(): void {
const cutoff = Date.now() - this.config.windowMs;
this.window = this.window.filter((t) => t > cutoff);
this.requestCount = this.window.length;
}
}Composing Retry Strategies
Combine retries, circuit breakers, and budgets into a resilient client that handles failures at multiple levels.
class ResilientHttpClient {
private circuitBreakers: Map<string, CircuitBreaker> = new Map();
private retryBudget: RetryBudget;
constructor(
private readonly retryConfig: RetryConfig,
budgetConfig: { maxRetryRatio: number; windowMs: number }
) {
this.retryBudget = new RetryBudget({
...budgetConfig,
minRequestsForBudget: 20,
});
}
async request<T>(serviceKey: string, fn: () => Promise<T>): Promise<T> {
const breaker = this.getCircuitBreaker(serviceKey);
this.retryBudget.recordRequest();
let lastError: Error | undefined;
for (let attempt = 0; attempt <= this.retryConfig.maxRetries; attempt++) {
try {
return await breaker.execute(fn);
} catch (error) {
lastError = error as Error;
if (error instanceof CircuitOpenError) throw error;
if (!isRetryable(error)) throw error;
if (attempt === this.retryConfig.maxRetries) break;
if (!this.retryBudget.canRetry()) {
throw new RetryBudgetExhaustedError(
"Retry budget exhausted — too many retries system-wide"
);
}
this.retryBudget.recordRetry();
const delay = calculateDelay(attempt, this.retryConfig);
await sleep(delay);
}
}
throw lastError;
}
private getCircuitBreaker(key: string): CircuitBreaker {
if (!this.circuitBreakers.has(key)) {
this.circuitBreakers.set(
key,
new CircuitBreaker({
failureThreshold: 5,
resetTimeoutMs: 30_000,
halfOpenMaxAttempts: 3,
})
);
}
return this.circuitBreakers.get(key)!;
}
}Observability for Retry Behavior
Without metrics, you cannot tell if retries are healing or hurting. Track retry rates, circuit breaker state transitions, and budget consumption.
function instrumentRetries(client: ResilientHttpClient, metrics: Metrics) {
const originalRequest = client.request.bind(client);
client.request = async function <T>(
serviceKey: string,
fn: () => Promise<T>
): Promise<T> {
const start = performance.now();
try {
const result = await originalRequest(serviceKey, fn);
metrics.increment("http_request_total", {
service: serviceKey,
status: "success",
});
return result;
} catch (error) {
metrics.increment("http_request_total", {
service: serviceKey,
status: "failure",
error_type: (error as Error).constructor.name,
});
throw error;
} finally {
metrics.histogram("http_request_duration_ms", performance.now() - start, {
service: serviceKey,
});
}
};
}Key Takeaways
Naive retries amplify failures. Exponential backoff with jitter spaces retries apart and prevents thundering herds. Always classify errors as retryable or not—never retry 400 Bad Request or 404 Not Found. Circuit breakers stop calling services that are clearly broken, giving them time to recover.
Retry budgets limit aggregate retry load across the system, preventing individual retry configs from causing amplification. Compose these patterns into a resilient client: retries with backoff, gated by circuit breakers, limited by budgets. Track retry rates and circuit state transitions in your metrics—if your retry rate exceeds 10% of total traffic, something is wrong beyond what retries can fix.
