Distributed Tracing: Following Requests Across Services

import {
  trace,
  context,
  SpanKind,
  SpanStatusCode,
} from '@opentelemetry/api';
 
// A trace is automatically created when you start the first span
const tracer = trace.getTracer('checkout-service', '1.0.0');
 
async function processCheckout(orderId: string): Promise<Order> {
  // This creates the root span for the checkout trace
  return tracer.startActiveSpan(
    'checkout.process',
    { kind: SpanKind.SERVER },
    async (span) => {
      try {
        span.setAttribute('order.id', orderId);
 
        // Each child span appears nested under the parent
        const inventory = await checkInventory(orderId);
        const payment = await processPayment(orderId);
        const confirmation = await sendConfirmation(orderId);
 
        span.setStatus({ code: SpanStatusCode.OK });
        return confirmation;
      } catch (error) {
        span.setStatus({
          code: SpanStatusCode.ERROR,
          message: error instanceof Error ? error.message : 'Unknown error',
        });
        span.recordException(error as Error);
        throw error;
      } finally {
        span.end();
      }
    }
  );
}
 
// Child span — automatically linked to parent via context
async function checkInventory(orderId: string): Promise<InventoryResult> {
  return tracer.startActiveSpan(
    'inventory.check',
    { kind: SpanKind.CLIENT },
    async (span) => {
      try {
        span.setAttribute('order.id', orderId);
        const result = await fetch(
          `http://inventory-service/check/${orderId}`
        );
        span.setAttribute('inventory.available', true);
        return result.json();
      } finally {
        span.end();
      }
    }
  );
}

// ❌ Without context propagation: separate disconnected traces
// Service A creates Trace-A, Service B creates Trace-B
// No way to correlate them
 
// ✅ With context propagation: one connected trace
// Service A's context flows to Service B via HTTP headers
 
// --- Setup (once at application startup) ---
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from
  '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from
  '@opentelemetry/exporter-trace-otlp-http';
 
const sdk = new NodeSDK({
  traceExporter: new OTLPTraceExporter({
    url: 'http://otel-collector:4318/v1/traces',
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      // Auto-instruments HTTP, Express, database clients
      // Context propagation happens automatically
      '@opentelemetry/instrumentation-http': {
        enabled: true,
      },
      '@opentelemetry/instrumentation-express': {
        enabled: true,
      },
    }),
  ],
});
 
sdk.start();
 
// When Service A makes an HTTP request, the instrumentation
// automatically adds these headers:
// traceparent: 00-<trace-id>-<span-id>-01
// tracestate: <vendor-specific-data>
 
// Service B's instrumentation reads these headers and
// continues the same trace, creating child spans under
// the calling span from Service A

// Manual context propagation for non-HTTP transports
import {
  propagation,
  context as otelContext,
  ROOT_CONTEXT,
} from '@opentelemetry/api';
 
// Injecting context into a message queue message
function publishEvent(
  queue: MessageQueue,
  event: DomainEvent
): void {
  const carrier: Record<string, string> = {};
 
  // Inject current trace context into carrier object
  propagation.inject(otelContext.active(), carrier);
 
  queue.publish({
    ...event,
    headers: carrier, // Contains traceparent, tracestate
  });
}
 
// Extracting context when consuming from the queue
function consumeEvent(message: QueueMessage): void {
  const parentContext = propagation.extract(
    ROOT_CONTEXT,
    message.headers
  );
 
  // Create spans within the extracted context
  otelContext.with(parentContext, () => {
    tracer.startActiveSpan('queue.process', (span) => {
      processMessage(message);
      span.end();
    });
  });
}

async function handlePayment(
  order: Order,
  paymentMethod: PaymentMethod
): Promise<PaymentResult> {
  return tracer.startActiveSpan('payment.process', async (span) => {
    // Business context for debugging and analysis
    span.setAttribute('payment.method', paymentMethod.type);
    span.setAttribute('payment.currency', order.currency);
    span.setAttribute('order.item_count', order.items.length);
    span.setAttribute('order.total_cents', order.totalCents);
    span.setAttribute('customer.tier', order.customer.tier);
 
    try {
      const result = await paymentGateway.charge(order);
 
      span.setAttribute('payment.gateway', result.gateway);
      span.setAttribute('payment.status', result.status);
      span.setAttribute('payment.transaction_id', result.transactionId);
 
      // Events mark significant moments within a span
      span.addEvent('payment.authorized', {
        'payment.authorization_code': result.authCode,
      });
 
      return result;
    } catch (error) {
      span.addEvent('payment.failed', {
        'payment.error_code': (error as PaymentError).code,
        'payment.retry_eligible': (error as PaymentError).retryable,
      });
      throw error;
    } finally {
      span.end();
    }
  });
}

import {
  ParentBasedSampler,
  TraceIdRatioBasedSampler,
  AlwaysOnSampler,
} from '@opentelemetry/sdk-trace-base';
 
// ❌ Trace everything — expensive, usually unnecessary
const alwaysOn = new AlwaysOnSampler();
 
// ✅ Head-based sampling: decide at trace start
// Sample 10% of traces randomly
const ratioSampler = new TraceIdRatioBasedSampler(0.1);
 
// ✅ Parent-based: respect the parent's sampling decision
// If a parent trace was sampled, continue sampling
const parentBased = new ParentBasedSampler({
  root: new TraceIdRatioBasedSampler(0.1),
});
 
// ✅ Custom sampler: always trace errors and slow requests
class SmartSampler {
  shouldSample(context: any, traceId: string, attributes: any) {
    // Always sample errors
    if (attributes?.['http.status_code'] >= 500) {
      return { decision: 1 }; // RECORD_AND_SAMPLED
    }
 
    // Always sample specific endpoints
    const path = attributes?.['http.target'] || '';
    if (path.startsWith('/api/checkout')) {
      return { decision: 1 };
    }
 
    // 5% sampling for everything else
    const hash = this.hashTraceId(traceId);
    return {
      decision: hash < 0.05 ? 1 : 0,
    };
  }
 
  private hashTraceId(traceId: string): number {
    let hash = 0;
    for (let i = 0; i < traceId.length; i++) {
      hash = ((hash << 5) - hash + traceId.charCodeAt(i)) | 0;
    }
    return Math.abs(hash) / 2147483647;
  }
}

// Common trace analysis patterns
 
// 1. Find the slowest span in a trace
function findBottleneck(spans: Span[]): Span {
  return spans.reduce((slowest, span) => {
    const duration = span.endTime - span.startTime;
    const slowestDuration = slowest.endTime - slowest.startTime;
    return duration > slowestDuration ? span : slowest;
  });
}
 
// 2. Calculate time spent in each service
function serviceBreakdown(
  spans: Span[]
): Map<string, number> {
  const breakdown = new Map<string, number>();
 
  for (const span of spans) {
    const service = span.attributes['service.name'] as string;
    const duration = span.endTime - span.startTime;
    breakdown.set(
      service,
      (breakdown.get(service) || 0) + duration
    );
  }
 
  return breakdown;
}
 
// 3. Alert on traces exceeding SLA thresholds
// In your trace collector pipeline:
// SELECT
//   trace_id,
//   duration_ms,
//   root_span_name,
//   service_name
// FROM traces
// WHERE duration_ms > 3000       -- SLA threshold
//   AND root_span_name = 'checkout.process'
//   AND timestamp > NOW() - INTERVAL '5 minutes'
// ORDER BY duration_ms DESC