Building Robust Webhook Delivery Systems
Design webhook delivery that handles retries with exponential backoff, signature verification, idempotency, delivery logging and endpoint health checks.
Webhooks seem simple—send an HTTP POST when something happens. In practice, reliable webhook delivery is a distributed systems problem. Endpoints go down. Networks partition. Receivers process slowly. Your system must handle all of this while guaranteeing that events reach their destinations without loss or excessive duplication.
The difference between a toy webhook system and a production one is retry logic, signature verification, idempotency, and endpoint health management.
Webhook Event Queuing
Events should be queued immediately and delivered asynchronously. The operation that triggers the webhook shouldn't block on delivery.
// ❌ Synchronous webhook delivery — blocks the operation
async function createOrder(order: Order): Promise<void> {
await database.insert(order);
// If this fails or times out, the order creation hangs
await fetch(webhookUrl, {
method: "POST",
body: JSON.stringify({ event: "order.created", data: order }),
});
}// ✅ Queue-based async delivery
interface WebhookEvent {
id: string;
type: string;
payload: Record<string, unknown>;
createdAt: Date;
subscriptionId: string;
endpoint: string;
attempts: number;
maxAttempts: number;
nextAttemptAt: Date;
status: "pending" | "delivered" | "failed" | "exhausted";
}
class WebhookQueue {
private events: WebhookEvent[] = [];
enqueue(
type: string,
payload: Record<string, unknown>,
subscriptions: WebhookSubscription[]
): string[] {
const eventIds: string[] = [];
for (const sub of subscriptions) {
if (!sub.events.includes(type)) continue;
const event: WebhookEvent = {
id: crypto.randomUUID(),
type,
payload,
createdAt: new Date(),
subscriptionId: sub.id,
endpoint: sub.url,
attempts: 0,
maxAttempts: 8,
nextAttemptAt: new Date(),
status: "pending",
};
this.events.push(event);
eventIds.push(event.id);
}
return eventIds;
}
getDeliverable(limit: number): WebhookEvent[] {
const now = new Date();
return this.events
.filter(
(e) =>
e.status === "pending" &&
e.nextAttemptAt <= now
)
.slice(0, limit);
}
}
interface WebhookSubscription {
id: string;
url: string;
events: string[];
secret: string;
active: boolean;
}
// The order creation is now non-blocking
async function createOrder(
order: Order,
webhookQueue: WebhookQueue,
subscriptions: WebhookSubscription[]
): Promise<void> {
await database.insert(order);
webhookQueue.enqueue("order.created", { order }, subscriptions);
// Returns immediately — delivery happens asynchronously
}Retry Strategy with Exponential Backoff
When delivery fails, retry with increasing delays. This prevents overwhelming a recovering endpoint while ensuring eventual delivery.
class WebhookDeliveryWorker {
private queue: WebhookQueue;
constructor(queue: WebhookQueue) {
this.queue = queue;
}
async processNextBatch(batchSize: number = 10): Promise<void> {
const events = this.queue.getDeliverable(batchSize);
for (const event of events) {
await this.deliver(event);
}
}
private async deliver(event: WebhookEvent): Promise<void> {
const signature = this.sign(event);
try {
const controller = new AbortController();
const timeout = setTimeout(
() => controller.abort(),
10_000
);
const response = await fetch(event.endpoint, {
method: "POST",
headers: {
"Content-Type": "application/json",
"X-Webhook-ID": event.id,
"X-Webhook-Signature": signature,
"X-Webhook-Timestamp": event.createdAt.toISOString(),
},
body: JSON.stringify({
id: event.id,
type: event.type,
data: event.payload,
created_at: event.createdAt.toISOString(),
}),
signal: controller.signal,
});
clearTimeout(timeout);
if (response.ok) {
event.status = "delivered";
this.logDelivery(event, "success", response.status);
} else if (response.status >= 500) {
this.scheduleRetry(event);
} else if (response.status >= 400) {
// Client error — don't retry
event.status = "failed";
this.logDelivery(event, "client_error", response.status);
}
} catch (error) {
this.scheduleRetry(event);
}
}
private scheduleRetry(event: WebhookEvent): void {
event.attempts++;
if (event.attempts >= event.maxAttempts) {
event.status = "exhausted";
this.logDelivery(event, "exhausted", 0);
return;
}
// Exponential backoff: 1m, 2m, 4m, 8m, 16m, 32m, 64m, 128m
const delayMs = Math.min(
60_000 * Math.pow(2, event.attempts),
128 * 60_000
);
// Add jitter to prevent thundering herd
const jitter = Math.random() * delayMs * 0.1;
event.nextAttemptAt = new Date(
Date.now() + delayMs + jitter
);
event.status = "pending";
}
private sign(event: WebhookEvent): string {
// HMAC-SHA256 signature for verification
const payload = JSON.stringify({
id: event.id,
type: event.type,
data: event.payload,
created_at: event.createdAt.toISOString(),
});
return `sha256=${computeHmac(payload, event.subscriptionId)}`;
}
private logDelivery(
event: WebhookEvent,
result: string,
statusCode: number
): void {
console.log(
JSON.stringify({
eventId: event.id,
type: event.type,
endpoint: event.endpoint,
attempt: event.attempts,
result,
statusCode,
timestamp: new Date().toISOString(),
})
);
}
}
function computeHmac(payload: string, secret: string): string {
// Placeholder for HMAC-SHA256
return `hmac_${payload.length}_${secret.slice(0, 4)}`;
}Signature Verification on the Receiver Side
Receivers must verify that webhooks actually came from your system and weren't tampered with in transit.
import { createHmac, timingSafeEqual } from "crypto";
function verifyWebhookSignature(
payload: string,
signature: string,
secret: string,
toleranceSeconds: number = 300
): { valid: boolean; reason?: string } {
// Check timestamp to prevent replay attacks
const timestampHeader = extractTimestamp(signature);
if (timestampHeader) {
const eventTime = new Date(timestampHeader).getTime();
const now = Date.now();
const age = Math.abs(now - eventTime);
if (age > toleranceSeconds * 1000) {
return {
valid: false,
reason: `Event too old: ${Math.round(age / 1000)}s`,
};
}
}
// Compute expected signature
const expectedSignature = createHmac("sha256", secret)
.update(payload)
.digest("hex");
const expected = `sha256=${expectedSignature}`;
// Timing-safe comparison prevents timing attacks
const signatureBuffer = Buffer.from(signature);
const expectedBuffer = Buffer.from(expected);
if (signatureBuffer.length !== expectedBuffer.length) {
return { valid: false, reason: "Signature length mismatch" };
}
const isValid = timingSafeEqual(signatureBuffer, expectedBuffer);
return {
valid: isValid,
reason: isValid ? undefined : "Signature mismatch",
};
}
function extractTimestamp(headers: string): string | null {
// Extract from X-Webhook-Timestamp header
return null; // Simplified
}
// Express middleware for webhook verification
function webhookVerificationMiddleware(secret: string) {
return (req: Request, res: Response, next: NextFunction) => {
const signature = req.headers["x-webhook-signature"] as string;
if (!signature) {
return res.status(401).json({ error: "Missing signature" });
}
const result = verifyWebhookSignature(
JSON.stringify(req.body),
signature,
secret
);
if (!result.valid) {
return res.status(403).json({
error: "Invalid signature",
reason: result.reason,
});
}
next();
};
}Endpoint Health Monitoring
Track endpoint health to avoid wasting resources delivering to consistently failing endpoints.
interface EndpointHealth {
url: string;
consecutiveFailures: number;
lastSuccessAt: Date | null;
lastFailureAt: Date | null;
totalDeliveries: number;
totalFailures: number;
status: "healthy" | "degraded" | "disabled";
}
class EndpointMonitor {
private health: Map<string, EndpointHealth> = new Map();
recordSuccess(url: string): void {
const h = this.getHealth(url);
h.consecutiveFailures = 0;
h.lastSuccessAt = new Date();
h.totalDeliveries++;
h.status = "healthy";
}
recordFailure(url: string): void {
const h = this.getHealth(url);
h.consecutiveFailures++;
h.lastFailureAt = new Date();
h.totalDeliveries++;
h.totalFailures++;
if (h.consecutiveFailures >= 10) {
h.status = "disabled";
} else if (h.consecutiveFailures >= 3) {
h.status = "degraded";
}
}
shouldDeliver(url: string): boolean {
const h = this.health.get(url);
if (!h) return true;
if (h.status === "disabled") {
// Check if enough time has passed to retry
const cooldownMs = 30 * 60 * 1000; // 30 minutes
if (
h.lastFailureAt &&
Date.now() - h.lastFailureAt.getTime() > cooldownMs
) {
// Allow one probe delivery
return true;
}
return false;
}
return true;
}
getHealthReport(): EndpointHealth[] {
return [...this.health.values()];
}
private getHealth(url: string): EndpointHealth {
let h = this.health.get(url);
if (!h) {
h = {
url,
consecutiveFailures: 0,
lastSuccessAt: null,
lastFailureAt: null,
totalDeliveries: 0,
totalFailures: 0,
status: "healthy",
};
this.health.set(url, h);
}
return h;
}
}Key Takeaways
Webhook delivery is an asynchronous distributed systems problem—queue events immediately and deliver them outside the request path so the triggering operation never blocks on endpoint availability. Retry with exponential backoff and jitter to handle transient failures without overwhelming recovering endpoints, capping at a reasonable maximum delay and attempt count. Sign every webhook payload with HMAC-SHA256 so receivers can verify authenticity, and include timestamps to prevent replay attacks. Receivers must verify signatures using timing-safe comparison to prevent timing side-channel attacks. Monitor endpoint health with consecutive failure tracking, automatically disabling delivery to persistently failing endpoints and probing periodically for recovery. Log every delivery attempt with event ID, attempt number, and result to provide a complete audit trail for debugging delivery issues. The goal is a system where events are never lost, endpoints are never overwhelmed, and both sender and receiver can verify the integrity of every delivery.
