Verteilte Systeme testen: Chaos, Contracts und Vertrauen

// Consumer side — defines what it needs from the provider
interface UserServiceContract {
  "GET /users/:id": {
    params: { id: string };
    response: {
      id: string;
      email: string;
      name: string;
      role: "admin" | "member";
    };
  };
  "POST /users": {
    body: { email: string; name: string };
    response: { id: string; email: string; name: string };
  };
}
 
// ❌ Integration test — requires both services running
async function testUserFetch() {
  const response = await fetch("http://user-service:3000/users/123");
  expect(response.status).toBe(200);
}
 
// ✅ Contract test — verifiable independently
function generateContractTest(contract: UserServiceContract) {
  return {
    consumer: "order-service",
    provider: "user-service",
    interactions: [
      {
        description: "Get user by ID",
        request: { method: "GET", path: "/users/123" },
        response: {
          status: 200,
          body: {
            id: "123",
            email: expect.stringMatching(/.+@.+/),
            name: expect.any(String),
            role: expect.stringMatching(/^(admin|member)$/),
          },
        },
      },
    ],
  };
}

// Provider side — verifies it satisfies all consumer contracts
class ContractVerifier {
  constructor(
    private readonly app: Express,
    private readonly contracts: Contract[]
  ) {}
 
  async verify(): Promise<VerificationResult[]> {
    const results: VerificationResult[] = [];
 
    for (const contract of this.contracts) {
      for (const interaction of contract.interactions) {
        const response = await this.executeRequest(interaction.request);
 
        const matches = this.matchesExpectedShape(
          response,
          interaction.response
        );
 
        results.push({
          consumer: contract.consumer,
          interaction: interaction.description,
          passed: matches.success,
          failures: matches.failures,
        });
      }
    }
 
    return results;
  }
 
  private matchesExpectedShape(
    actual: any,
    expected: any
  ): { success: boolean; failures: string[] } {
    const failures: string[] = [];
 
    for (const [key, expectedValue] of Object.entries(expected.body)) {
      if (!(key in actual.body)) {
        failures.push(`Missing field: ${key}`);
      }
    }
 
    if (actual.status !== expected.status) {
      failures.push(
        `Status mismatch: expected ${expected.status}, got ${actual.status}`
      );
    }
 
    return { success: failures.length === 0, failures };
  }
 
  private async executeRequest(request: ContractRequest): Promise<any> {
    // Execute against the running provider instance
    return {} as any;
  }
}

import { GenericContainer, StartedTestContainer } from "testcontainers";
 
class TestEnvironment {
  private postgres: StartedTestContainer | null = null;
  private redis: StartedTestContainer | null = null;
 
  async setup(): Promise<{
    databaseUrl: string;
    redisUrl: string;
  }> {
    const [pg, rd] = await Promise.all([
      new GenericContainer("postgres:16")
        .withEnvironment({
          POSTGRES_DB: "test",
          POSTGRES_USER: "test",
          POSTGRES_PASSWORD: "test",
        })
        .withExposedPorts(5432)
        .start(),
      new GenericContainer("redis:7")
        .withExposedPorts(6379)
        .start(),
    ]);
 
    this.postgres = pg;
    this.redis = rd;
 
    return {
      databaseUrl: `postgresql://test:test@${pg.getHost()}:${pg.getMappedPort(5432)}/test`,
      redisUrl: `redis://${rd.getHost()}:${rd.getMappedPort(6379)}`,
    };
  }
 
  async teardown(): Promise<void> {
    await Promise.all([
      this.postgres?.stop(),
      this.redis?.stop(),
    ]);
  }
}
 
// Usage in tests
describe("OrderService integration", () => {
  let env: TestEnvironment;
  let service: OrderService;
 
  beforeAll(async () => {
    env = new TestEnvironment();
    const urls = await env.setup();
    service = new OrderService(urls.databaseUrl, urls.redisUrl);
    await service.runMigrations();
  });
 
  afterAll(() => env.teardown());
 
  it("processes order through full pipeline", async () => {
    const order = await service.create({
      userId: "user-1",
      items: [{ productId: "prod-1", quantity: 2 }],
    });
 
    expect(order.status).toBe("pending");
 
    await service.process(order.id);
    const updated = await service.findById(order.id);
 
    expect(updated.status).toBe("confirmed");
  });
});

interface ChaosExperiment {
  name: string;
  hypothesis: string;
  target: string;
  faultType: "latency" | "error" | "kill" | "partition";
  duration: number;
  steadyState: SteadyStateCheck[];
  rollback: () => Promise<void>;
}
 
interface SteadyStateCheck {
  metric: string;
  condition: "above" | "below" | "equals";
  threshold: number;
}
 
const paymentLatencyExperiment: ChaosExperiment = {
  name: "Payment service high latency",
  hypothesis:
    "When payment service responds slowly, checkout degrades to " +
    "a queued state instead of timing out with an error",
  target: "payment-service",
  faultType: "latency",
  duration: 300000, // 5 minutes
  steadyState: [
    { metric: "checkout.success_rate", condition: "above", threshold: 0.95 },
    { metric: "checkout.p99_latency_ms", condition: "below", threshold: 5000 },
    { metric: "error_rate.5xx", condition: "below", threshold: 0.01 },
  ],
  rollback: async () => {
    await removeFaultInjection("payment-service");
  },
};
 
async function runExperiment(
  experiment: ChaosExperiment
): Promise<ExperimentResult> {
  // 1. Verify steady state before injection
  const baseline = await checkSteadyState(experiment.steadyState);
  if (!baseline.healthy) {
    return { status: "aborted", reason: "System not in steady state" };
  }
 
  // 2. Inject fault
  await injectFault(experiment.target, experiment.faultType);
 
  // 3. Observe during experiment
  const observations = await observe(experiment.duration, experiment.steadyState);
 
  // 4. Always rollback
  await experiment.rollback();
 
  // 5. Evaluate
  const hypothesisHeld = observations.every((o) => o.withinThreshold);
 
  return {
    status: hypothesisHeld ? "passed" : "failed",
    observations,
    recommendations: hypothesisHeld
      ? []
      : generateRecommendations(observations),
  };
}

interface TestPortfolio {
  unit: { coverage: number; runTimeSeconds: number };
  contract: { servicesCovered: number; totalServices: number };
  integration: { criticalPaths: number; coveredPaths: number };
  chaos: { experimentsRun: number; lastRunDate: string };
}
 
function assessTestConfidence(portfolio: TestPortfolio): string[] {
  const gaps: string[] = [];
 
  if (portfolio.contract.servicesCovered < portfolio.contract.totalServices) {
    const uncovered =
      portfolio.contract.totalServices - portfolio.contract.servicesCovered;
    gaps.push(`${uncovered} services lack contract tests`);
  }
 
  if (portfolio.chaos.experimentsRun === 0) {
    gaps.push("No chaos experiments — resilience is untested");
  }
 
  if (portfolio.integration.coveredPaths < portfolio.integration.criticalPaths) {
    gaps.push("Critical integration paths missing test coverage");
  }
 
  return gaps;
}