LLM-Ausgabequalität bewerten: Metriken und Testframeworks

interface EvalMetric {
  name: string;
  compute: (prediction: string, reference: string, input: string) => Promise<number>;
  threshold: number;
  weight: number;
}
 
interface EvalResult {
  testCase: string;
  scores: Record<string, number>;
  weightedScore: number;
  passed: boolean;
}
 
class LLMEvaluator {
  constructor(private readonly metrics: EvalMetric[]) {}
 
  async evaluate(
    prediction: string,
    reference: string,
    input: string,
    testCaseName: string
  ): Promise<EvalResult> {
    const scores: Record<string, number> = {};
    let weightedSum = 0;
    let totalWeight = 0;
 
    for (const metric of this.metrics) {
      const score = await metric.compute(prediction, reference, input);
      scores[metric.name] = score;
      weightedSum += score * metric.weight;
      totalWeight += metric.weight;
    }
 
    const weightedScore = weightedSum / totalWeight;
    const passed = this.metrics.every(
      (m) => scores[m.name] >= m.threshold
    );
 
    return { testCase: testCaseName, scores, weightedScore, passed };
  }
}

// ❌ String-based comparison — fails for valid paraphrases
function exactMatch(prediction: string, reference: string): boolean {
  return prediction.trim() === reference.trim();
  // "The cat sat on the mat" !== "A cat was sitting atop the mat"
  // Both are valid but this returns false
}
 
// ✅ Embedding-based semantic similarity
async function cosineSimilarity(
  embedding1: number[],
  embedding2: number[]
): Promise<number> {
  let dotProduct = 0;
  let norm1 = 0;
  let norm2 = 0;
 
  for (let i = 0; i < embedding1.length; i++) {
    dotProduct += embedding1[i] * embedding2[i];
    norm1 += embedding1[i] ** 2;
    norm2 += embedding2[i] ** 2;
  }
 
  return dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2));
}
 
async function semanticSimilarity(
  prediction: string,
  reference: string,
  embeddingFn: (text: string) => Promise<number[]>
): Promise<number> {
  const [predEmbedding, refEmbedding] = await Promise.all([
    embeddingFn(prediction),
    embeddingFn(reference),
  ]);
 
  return cosineSimilarity(predEmbedding, refEmbedding);
}

interface BehavioralAssertion {
  name: string;
  check: (output: string, input: string) => boolean;
}
 
const summarizationAssertions: BehavioralAssertion[] = [
  {
    name: "shorter-than-input",
    check: (output, input) => output.length < input.length * 0.5,
  },
  {
    name: "preserves-named-entities",
    check: (output, input) => {
      const entities = extractNamedEntities(input);
      return entities.every((entity) =>
        output.toLowerCase().includes(entity.toLowerCase())
      );
    },
  },
  {
    name: "preserves-numbers",
    check: (output, input) => {
      const inputNumbers = input.match(/\d+\.?\d*/g) || [];
      const criticalNumbers = inputNumbers.filter(
        (n) => parseFloat(n) > 0
      );
      return criticalNumbers.every((n) => output.includes(n));
    },
  },
  {
    name: "no-hallucinated-quotes",
    check: (output, input) => {
      const outputQuotes = output.match(/"[^"]+"/g) || [];
      return outputQuotes.every((quote) => input.includes(quote));
    },
  },
];
 
function runBehavioralTests(
  output: string,
  input: string,
  assertions: BehavioralAssertion[]
): { passed: string[]; failed: string[] } {
  const passed: string[] = [];
  const failed: string[] = [];
 
  for (const assertion of assertions) {
    if (assertion.check(output, input)) {
      passed.push(assertion.name);
    } else {
      failed.push(assertion.name);
    }
  }
 
  return { passed, failed };
}

interface EvalTestCase {
  id: string;
  input: string;
  expectedOutput: string;
  category: string;
  difficulty: "easy" | "medium" | "hard";
  edgeCaseType?: string;
}
 
const evaluationDataset: EvalTestCase[] = [
  {
    id: "sum-001",
    input:
      "The company reported Q3 revenue of $4.2 billion, up 15% year-over-year. CEO Jane Smith attributed growth to the cloud division.",
    expectedOutput:
      "Company Q3 revenue reached $4.2B (+15% YoY), driven by cloud growth per CEO Jane Smith.",
    category: "financial-summary",
    difficulty: "easy",
  },
  {
    id: "sum-002",
    input:
      "Despite the 23% increase in active users to 150 million, the platform reported a net loss of $89 million due to increased infrastructure spending. However, the company expects profitability by Q2 next year.",
    expectedOutput:
      "Active users grew 23% to 150M, but infrastructure costs drove $89M net loss. Profitability expected by Q2.",
    category: "financial-summary",
    difficulty: "medium",
    edgeCaseType: "contradictory-signals",
  },
  {
    id: "sum-003",
    input: "",
    expectedOutput: "",
    category: "edge-case",
    difficulty: "easy",
    edgeCaseType: "empty-input",
  },
];
 
function validateDatasetCoverage(dataset: EvalTestCase[]): {
  categoryDistribution: Record<string, number>;
  difficultyDistribution: Record<string, number>;
  edgeCaseCoverage: string[];
  gaps: string[];
} {
  const categories: Record<string, number> = {};
  const difficulties: Record<string, number> = {};
  const edgeCases: Set<string> = new Set();
 
  for (const tc of dataset) {
    categories[tc.category] = (categories[tc.category] || 0) + 1;
    difficulties[tc.difficulty] = (difficulties[tc.difficulty] || 0) + 1;
    if (tc.edgeCaseType) edgeCases.add(tc.edgeCaseType);
  }
 
  const expectedEdgeCases = [
    "empty-input",
    "very-long-input",
    "special-characters",
    "multilingual",
    "contradictory-signals",
  ];
  const gaps = expectedEdgeCases.filter((ec) => !edgeCases.has(ec));
 
  return {
    categoryDistribution: categories,
    difficultyDistribution: difficulties,
    edgeCaseCoverage: [...edgeCases],
    gaps,
  };
}

interface RegressionReport {
  baselineVersion: string;
  candidateVersion: string;
  totalTests: number;
  improved: number;
  regressed: number;
  unchanged: number;
  averageScoreChange: number;
  recommendation: "promote" | "investigate" | "reject";
}
 
async function detectRegressions(
  baseline: EvalResult[],
  candidate: EvalResult[],
  regressionThreshold: number = 0.05
): Promise<RegressionReport> {
  let improved = 0;
  let regressed = 0;
  let unchanged = 0;
  let totalScoreChange = 0;
 
  for (let i = 0; i < baseline.length; i++) {
    const diff =
      candidate[i].weightedScore - baseline[i].weightedScore;
    totalScoreChange += diff;
 
    if (diff > regressionThreshold) {
      improved++;
    } else if (diff < -regressionThreshold) {
      regressed++;
    } else {
      unchanged++;
    }
  }
 
  const avgChange = totalScoreChange / baseline.length;
  const regressionRate = regressed / baseline.length;
 
  let recommendation: "promote" | "investigate" | "reject";
  if (regressionRate > 0.1) {
    recommendation = "reject";
  } else if (regressionRate > 0.05 || avgChange < 0) {
    recommendation = "investigate";
  } else {
    recommendation = "promote";
  }
 
  return {
    baselineVersion: "v1.0",
    candidateVersion: "v1.1",
    totalTests: baseline.length,
    improved,
    regressed,
    unchanged,
    averageScoreChange: avgChange,
    recommendation,
  };
}

interface QualityGateConfig {
  minPassRate: number;
  minAverageScore: number;
  maxRegressionRate: number;
  criticalTestCases: string[];
}
 
async function runQualityGate(
  results: EvalResult[],
  config: QualityGateConfig
): Promise<{ passed: boolean; reasons: string[] }> {
  const reasons: string[] = [];
 
  // Check overall pass rate
  const passRate =
    results.filter((r) => r.passed).length / results.length;
  if (passRate < config.minPassRate) {
    reasons.push(
      `Pass rate ${(passRate * 100).toFixed(1)}% below threshold ${config.minPassRate * 100}%`
    );
  }
 
  // Check average score
  const avgScore =
    results.reduce((sum, r) => sum + r.weightedScore, 0) /
    results.length;
  if (avgScore < config.minAverageScore) {
    reasons.push(
      `Average score ${avgScore.toFixed(3)} below threshold ${config.minAverageScore}`
    );
  }
 
  // Check critical test cases
  for (const criticalId of config.criticalTestCases) {
    const result = results.find((r) => r.testCase === criticalId);
    if (result && !result.passed) {
      reasons.push(`Critical test case failed: ${criticalId}`);
    }
  }
 
  return { passed: reasons.length === 0, reasons };
}