Generative AI for Code Review Automation

import { Octokit } from "@octokit/rest";
 
interface ReviewComment {
  path: string;
  line: number;
  body: string;
  severity: "critical" | "warning" | "suggestion";
}
 
interface DiffFile {
  filename: string;
  patch: string;
  status: "added" | "modified" | "removed";
}
 
async function getChangedFiles(
  octokit: Octokit,
  owner: string,
  repo: string,
  pullNumber: number
): Promise<DiffFile[]> {
  const { data: files } = await octokit.pulls.listFiles({
    owner,
    repo,
    pull_number: pullNumber,
    per_page: 100,
  });
 
  return files
    .filter((f) => f.status !== "removed")
    .map((f) => ({
      filename: f.filename,
      patch: f.patch ?? "",
      status: f.status as DiffFile["status"],
    }));
}
 
async function postReviewComments(
  octokit: Octokit,
  owner: string,
  repo: string,
  pullNumber: number,
  commitId: string,
  comments: ReviewComment[]
): Promise<void> {
  if (comments.length === 0) return;
 
  await octokit.pulls.createReview({
    owner,
    repo,
    pull_number: pullNumber,
    commit_id: commitId,
    event: "COMMENT",
    comments: comments.map((c) => ({
      path: c.path,
      line: c.line,
      body: formatComment(c),
    })),
  });
}
 
function formatComment(comment: ReviewComment): string {
  const icons = {
    critical: "🔴",
    warning: "🟡",
    suggestion: "💡",
  };
  return `${icons[comment.severity]} **AI Review** (${comment.severity})\n\n${comment.body}`;
}

function buildReviewPrompt(
  file: DiffFile,
  context: { language: string; framework: string }
): string {
  return `You are reviewing a code diff in a ${context.language} ${context.framework} project.
 
Review ONLY the changed lines (prefixed with +) for these specific issues:
1. **Bugs**: Logic errors, off-by-one errors, null/undefined access
2. **Security**: SQL injection, XSS, hardcoded secrets, path traversal
3. **Error handling**: Missing try/catch, unhandled promise rejections, swallowed errors
4. **Resource leaks**: Unclosed connections, missing cleanup, event listener leaks
5. **Race conditions**: Shared mutable state, missing locks, TOCTOU
 
Do NOT comment on:
- Style preferences (formatting, naming conventions)
- Obvious code that is correct
- Things already handled by linters or formatters
 
For each issue found, respond in JSON:
{
  "comments": [
    {
      "line": <line number in the NEW file>,
      "severity": "critical" | "warning" | "suggestion",
      "issue": "<what is wrong>",
      "suggestion": "<how to fix it with a code example>"
    }
  ]
}
 
If no issues are found, return: { "comments": [] }
 
File: ${file.filename}
Diff:
${file.patch}`;
}

// ❌ Bad prompt — produces noisy, generic comments
const badPrompt = `
  Review this code and suggest improvements:
  ${diff}
`;
// Result: "Consider adding comments to explain this function"
//         "This variable name could be more descriptive"
//         "You might want to add error handling here"
// Noise that wastes reviewer time
 
// ✅ Good prompt — focused on high-value findings
const goodPrompt = `
  Review this TypeScript diff for bugs and security issues only.
  Ignore style, naming, and formatting.
  Only comment if you are confident the issue is real.
  For each issue, show the fix as a code block.
 
  Context: This is a payment processing service handling Stripe webhooks.
  The code must be idempotent and handle duplicate webhook deliveries.
 
  ${diff}
`;
// Result: "Line 45: webhook signature is not verified before
//          processing the event body. An attacker could forge events."
// Actionable, high-confidence finding

interface ReviewPass {
  name: string;
  fileFilter: (filename: string) => boolean;
  promptTemplate: string;
  severity: "critical" | "warning" | "suggestion";
}
 
const reviewPasses: ReviewPass[] = [
  {
    name: "security",
    fileFilter: () => true,
    severity: "critical",
    promptTemplate: `Analyze this diff for security vulnerabilities:
- SQL injection (string concatenation in queries)
- XSS (unescaped user input in HTML/JSX)
- Hardcoded secrets (API keys, passwords, tokens)
- Path traversal (user input in file paths)
- SSRF (user input in URLs for server-side requests)
- Insecure deserialization
Only report issues you are highly confident about.`,
  },
  {
    name: "error-handling",
    fileFilter: (f) => /\.(ts|js|tsx|jsx)$/.test(f),
    severity: "warning",
    promptTemplate: `Check this diff for error handling issues:
- Promises without .catch() or try/catch in async functions
- Empty catch blocks that swallow errors
- Missing null/undefined checks on optional values
- Errors thrown without useful messages
- Missing finally blocks for resource cleanup`,
  },
  {
    name: "database",
    fileFilter: (f) => /\.(sql|ts|js)$/.test(f),
    severity: "warning",
    promptTemplate: `Check this diff for database-related issues:
- N+1 query patterns (queries inside loops)
- Missing transactions for multi-step operations
- Missing indexes for query patterns
- Unbounded queries (no LIMIT clause)
- Hardcoded connection parameters`,
  },
];
 
async function runAllPasses(
  files: DiffFile[],
  passes: ReviewPass[]
): Promise<ReviewComment[]> {
  const allComments: ReviewComment[] = [];
 
  for (const pass of passes) {
    const relevantFiles = files.filter((f) =>
      pass.fileFilter(f.filename)
    );
 
    for (const file of relevantFiles) {
      const prompt = `${pass.promptTemplate}\n\nFile: ${file.filename}\nDiff:\n${file.patch}`;
      const comments = await queryLLM(prompt);
      allComments.push(
        ...comments.map((c) => ({ ...c, severity: pass.severity }))
      );
    }
  }
 
  return deduplicateComments(allComments);
}

interface FeedbackLoop {
  commentId: string;
  reaction: "helpful" | "not-helpful" | "false-positive";
  reviewerNote?: string;
}
 
class ReviewQualityTracker {
  private feedback: FeedbackLoop[] = [];
 
  recordFeedback(entry: FeedbackLoop): void {
    this.feedback.push(entry);
  }
 
  getAccuracyRate(): number {
    if (this.feedback.length === 0) return 0;
    const helpful = this.feedback.filter(
      (f) => f.reaction === "helpful"
    ).length;
    return helpful / this.feedback.length;
  }
 
  shouldPostComment(confidence: number): boolean {
    const accuracyRate = this.getAccuracyRate();
 
    // Adaptive threshold: if bot accuracy is low,
    // only post high-confidence comments
    if (accuracyRate < 0.5) return confidence > 0.9;
    if (accuracyRate < 0.7) return confidence > 0.75;
    return confidence > 0.6;
  }
}
 
// Require confidence scores from the LLM
const promptWithConfidence = `
For each issue, include a confidence score (0.0 to 1.0):
- 0.9+: Certain this is a bug or security issue
- 0.7-0.9: Likely an issue, worth investigating
- 0.5-0.7: Possible issue, might be intentional
- Below 0.5: Do not report
`;

// ❌ Bot that comments on everything
// "Consider using const instead of let" (on a variable that IS reassigned)
// "This function could be shorter" (opinion, not a bug)
// "Missing JSDoc on exported function" (that's a linter's job)
 
// ✅ Bot that only speaks when it matters
// Posts 1-2 comments per PR on average
// Each comment is a real bug, security issue, or resource leak
// Developers learn to pay attention because signal-to-noise is high
 
const botGuidelines = {
  maxCommentsPerPR: 5,
  minConfidence: 0.75,
  neverCommentOn: [
    "formatting",
    "naming conventions",
    "missing documentation",
    "import ordering",
    "preference-based patterns",
  ],
  alwaysCommentOn: [
    "security vulnerabilities (high confidence)",
    "data loss risks",
    "unhandled error paths in critical flows",
    "resource leaks (connections, file handles)",
  ],
};