Debugging Memory Issues in Production Systems

# Monitor memory patterns in Kubernetes
kubectl top pods -w --sort-by=memory
 
# Check for OOMKill events
kubectl get events --field-selector reason=OOMKilling
 
# Get memory details for a specific pod
kubectl describe pod api-server-7d5f8b6c4-abc12 | grep -A5 "Last State"

// Expose a debug endpoint for heap snapshots (behind auth)
import v8 from 'v8';
import fs from 'fs';
import path from 'path';
 
// Only expose in controlled environments
function registerDebugEndpoints(app: Express) {
  app.post('/debug/heap-snapshot', authMiddleware, async (req, res) => {
    const snapshotPath = path.join(
      '/tmp',
      `heap-${Date.now()}.heapsnapshot`
    );
 
    const snapshotStream = v8.writeHeapSnapshot(snapshotPath);
    res.json({
      path: snapshotStream,
      timestamp: new Date().toISOString(),
      heapUsed: process.memoryUsage().heapUsed,
    });
  });
 
  app.get('/debug/memory', authMiddleware, (req, res) => {
    const mem = process.memoryUsage();
    res.json({
      rss: `${(mem.rss / 1024 / 1024).toFixed(1)}MB`,
      heapTotal: `${(mem.heapTotal / 1024 / 1024).toFixed(1)}MB`,
      heapUsed: `${(mem.heapUsed / 1024 / 1024).toFixed(1)}MB`,
      external: `${(mem.external / 1024 / 1024).toFixed(1)}MB`,
      arrayBuffers: `${(mem.arrayBuffers / 1024 / 1024).toFixed(1)}MB`,
    });
  });
}

# Workflow: take two snapshots 5 minutes apart
curl -X POST https://internal-api/debug/heap-snapshot  # snapshot 1
sleep 300
curl -X POST https://internal-api/debug/heap-snapshot  # snapshot 2
 
# Download snapshots and open in Chrome DevTools:
# 1. Open chrome://inspect → Memory tab
# 2. Load both snapshots
# 3. Select "Comparison" view between snapshot 1 and 2
# 4. Sort by "Size Delta" — largest growth = likely leak

// ❌ Leak Pattern 1: Event listeners accumulating
class DataProcessor {
  constructor(private emitter: EventEmitter) {}
 
  process(data: unknown) {
    // Each call adds a NEW listener — they never get removed
    this.emitter.on('update', () => {
      this.handleUpdate(data);  // Closure captures 'data'
    });
  }
}
 
// ✅ Fixed: remove listeners when done
class DataProcessor {
  private handler: (() => void) | null = null;
 
  process(data: unknown) {
    // Remove previous listener before adding new one
    if (this.handler) {
      this.emitter.off('update', this.handler);
    }
    this.handler = () => this.handleUpdate(data);
    this.emitter.on('update', this.handler);
  }
 
  destroy() {
    if (this.handler) {
      this.emitter.off('update', this.handler);
      this.handler = null;
    }
  }
}

// ❌ Leak Pattern 2: Unbounded cache
const cache = new Map<string, UserData>();
 
async function getUser(id: string): Promise<UserData> {
  if (cache.has(id)) return cache.get(id)!;
  const user = await db.users.findById(id);
  cache.set(id, user);  // Map grows forever
  return user;
}
 
// ✅ Fixed: bounded cache with LRU eviction
import { LRUCache } from 'lru-cache';
 
const cache = new LRUCache<string, UserData>({
  max: 1000,           // Maximum entries
  ttl: 1000 * 60 * 5,  // 5-minute TTL
});
 
async function getUser(id: string): Promise<UserData> {
  const cached = cache.get(id);
  if (cached) return cached;
  const user = await db.users.findById(id);
  cache.set(id, user);
  return user;
}

// ❌ Leak Pattern 3: Closures holding references
function createHandler(largeDataset: Buffer) {
  // The returned function captures the entire largeDataset
  // Even if the handler only needs a small piece
  return (req: Request) => {
    const id = req.params.id;
    return largeDataset.slice(0, 100);
  };
  // largeDataset (possibly MB) stays in memory as long as
  // the handler exists
}
 
// ✅ Fixed: extract only what you need before the closure
function createHandler(largeDataset: Buffer) {
  const header = Buffer.from(largeDataset.slice(0, 100));
  // largeDataset can now be garbage collected
  return (req: Request) => {
    return header;
  };
}

// Expose Prometheus-compatible memory metrics
import { collectDefaultMetrics, register, Gauge } from 'prom-client';
 
collectDefaultMetrics({ prefix: 'app_' });
 
const heapGauge = new Gauge({
  name: 'app_heap_used_bytes',
  help: 'Process heap used in bytes',
});
 
const rssGauge = new Gauge({
  name: 'app_rss_bytes',
  help: 'Process resident set size in bytes',
});
 
// Update metrics every 15 seconds
setInterval(() => {
  const mem = process.memoryUsage();
  heapGauge.set(mem.heapUsed);
  rssGauge.set(mem.rss);
}, 15000);
 
// Metrics endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', register.contentType);
  res.end(await register.metrics());
});

# Prometheus alert rule: detect memory leak pattern
groups:
  - name: memory
    rules:
      - alert: MemoryLeakDetected
        expr: |
          deriv(app_heap_used_bytes[1h]) > 1048576
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Possible memory leak in {{ $labels.pod }}"
          description: "Heap memory growing at >1MB/hour for 30 minutes"

// Allocation sampling — low overhead, production-safe
import { Session } from 'inspector';
 
async function captureAllocationProfile(durationMs: number): Promise<Buffer> {
  const session = new Session();
  session.connect();
 
  return new Promise((resolve) => {
    session.post('HeapProfiler.startSampling', {
      samplingInterval: 32768,  // Sample every 32KB allocated
    });
 
    setTimeout(() => {
      session.post('HeapProfiler.stopSampling', (err, result) => {
        session.disconnect();
        const profile = JSON.stringify(result?.profile);
        resolve(Buffer.from(profile));
      });
    }, durationMs);
  });
}
 
// Capture a 30-second allocation profile
app.post('/debug/allocation-profile', authMiddleware, async (req, res) => {
  const profile = await captureAllocationProfile(30000);
  res.setHeader('Content-Disposition',
    `attachment; filename="alloc-${Date.now()}.heapprofile"`);
  res.send(profile);
});

# For non-Node.js systems: use system-level tools
# Linux memory profiling
valgrind --tool=massif ./your-binary
 
# JVM heap dump
jmap -dump:format=b,file=heap.bin <pid>
 
# Go pprof
go tool pprof http://localhost:6060/debug/pprof/heap

# ❌ Node.js default heap limit (1.7GB) exceeds container limit (512MB)
spec:
  containers:
    - name: api
      image: myregistry/api:latest
      resources:
        limits:
          memory: "512Mi"
      # Node.js will try to use 1.7GB → OOMKilled
 
# ✅ Match Node.js heap limit to container limit
spec:
  containers:
    - name: api
      image: myregistry/api:latest
      resources:
        requests:
          memory: "256Mi"
        limits:
          memory: "512Mi"
      env:
        - name: NODE_OPTIONS
          value: "--max-old-space-size=384"
          # 384MB heap + ~128MB overhead = within 512MB container limit

# Rule of thumb for Node.js:
# --max-old-space-size = container_limit * 0.75
# The remaining 25% covers:
# - V8 overhead (new space, code space)
# - Native memory (buffers, C++ objects)
# - OS page cache