Distributed Tracing: Following Requests Across Services

import { trace, SpanKind, context, propagation } from "@opentelemetry/api";
 
const tracer = trace.getTracer("order-service");
 
// Each function call creates a span within the trace
async function createOrder(userId: string, items: OrderItem[]): Promise<Order> {
  return tracer.startActiveSpan("createOrder", async (span) => {
    span.setAttribute("user.id", userId);
    span.setAttribute("order.item_count", items.length);
 
    try {
      // Each of these creates child spans
      const inventory = await checkInventory(items);
      const payment = await processPayment(userId, calculateTotal(items));
      const order = await saveOrder(userId, items, payment.id);
 
      span.setAttribute("order.id", order.id);
      span.setStatus({ code: 0 }); // OK
 
      return order;
    } catch (error) {
      span.setStatus({
        code: 2, // ERROR
        message: error instanceof Error ? error.message : "Unknown error",
      });
      throw error;
    } finally {
      span.end();
    }
  });
}

// ❌ No context propagation — spans from different services are disconnected
async function callInventoryService(items: OrderItem[]) {
  const response = await fetch("http://inventory-service/api/check", {
    method: "POST",
    body: JSON.stringify({ items }),
  });
  return response.json();
}
 
// ✅ Inject trace context into outgoing request headers
async function callInventoryService(items: OrderItem[]) {
  const headers: Record<string, string> = {
    "Content-Type": "application/json",
  };
 
  // OpenTelemetry injects W3C Trace Context headers automatically
  // traceparent: 00-<trace-id>-<span-id>-01
  propagation.inject(context.active(), headers);
 
  const response = await fetch("http://inventory-service/api/check", {
    method: "POST",
    headers,
    body: JSON.stringify({ items }),
  });
  return response.json();
}
 
// Receiving service extracts the context
function tracingMiddleware(req: Request, res: Response, next: NextFunction) {
  const parentContext = propagation.extract(context.active(), req.headers);
 
  context.with(parentContext, () => {
    const span = tracer.startSpan("handleRequest", {
      kind: SpanKind.SERVER,
      attributes: {
        "http.method": req.method,
        "http.url": req.url,
      },
    });
 
    res.on("finish", () => {
      span.setAttribute("http.status_code", res.statusCode);
      span.end();
    });
 
    next();
  });
}

import { NodeSDK } from "@opentelemetry/sdk-node";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { Resource } from "@opentelemetry/resources";
 
const sdk = new NodeSDK({
  resource: new Resource({
    "service.name": "order-service",
    "service.version": "1.2.0",
    "deployment.environment": process.env.NODE_ENV ?? "development",
  }),
  traceExporter: new OTLPTraceExporter({
    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://localhost:4318/v1/traces",
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      // Auto-instrument HTTP, Express, database clients, etc.
      "@opentelemetry/instrumentation-http": { enabled: true },
      "@opentelemetry/instrumentation-express": { enabled: true },
      "@opentelemetry/instrumentation-pg": { enabled: true },
    }),
  ],
});
 
sdk.start();
 
// Graceful shutdown — flush pending spans on exit
process.on("SIGTERM", () => {
  sdk.shutdown().then(() => process.exit(0));
});

// ❌ Trace with only HTTP metadata — "a POST to /api/orders took 2s"
// Doesn't tell you WHY it was slow
 
// ✅ Custom attributes add business context
async function processPayment(userId: string, amount: Money): Promise<PaymentResult> {
  return tracer.startActiveSpan("processPayment", async (span) => {
    span.setAttribute("payment.amount", amount.value);
    span.setAttribute("payment.currency", amount.currency);
    span.setAttribute("payment.provider", "stripe");
 
    const result = await stripe.charges.create({
      amount: amount.value,
      currency: amount.currency,
    });
 
    span.setAttribute("payment.status", result.status);
    span.setAttribute("payment.id", result.id);
 
    if (result.status === "failed") {
      span.setAttribute("payment.failure_reason", result.failure_message ?? "unknown");
      span.addEvent("payment_failed", {
        reason: result.failure_message ?? "unknown",
        decline_code: result.decline_code ?? "none",
      });
    }
 
    span.end();
    return result;
  });
}

import { TraceIdRatioBasedSampler, ParentBasedSampler } from "@opentelemetry/sdk-trace-base";
 
// Sample 10% of requests, but always trace requests where the parent was traced
const sampler = new ParentBasedSampler({
  root: new TraceIdRatioBasedSampler(0.1), // 10% of root spans
  // If an incoming request already has a trace context, always trace it
  // This ensures complete traces across services
});
 
// For debugging: always trace requests with a specific header
import { Sampler, SamplingResult, SamplingDecision } from "@opentelemetry/sdk-trace-base";
 
class DebugHeaderSampler implements Sampler {
  shouldSample(parentContext: unknown, traceId: string): SamplingResult {
    // Always sample if debug header is present
    // Otherwise fall back to ratio sampling
    return {
      decision: SamplingDecision.RECORD_AND_SAMPLED,
    };
  }
 
  toString(): string {
    return "DebugHeaderSampler";
  }
}