Building AI-Powered Code Review Assistants with LLM APIs

interface PRFile {
  filename: string;
  status: "added" | "modified" | "removed" | "renamed";
  patch: string;
  additions: number;
  deletions: number;
}
 
async function getFilteredPRFiles(
  owner: string,
  repo: string,
  prNumber: number
): Promise<PRFile[]> {
  const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
  const { data: files } = await octokit.pulls.listFiles({
    owner,
    repo,
    pull_number: prNumber,
    per_page: 100,
  });
 
  const ignoredPatterns = [
    /\.lock$/,
    /package-lock\.json$/,
    /\.min\.(js|css)$/,
    /dist\//,
    /\.generated\./,
    /\.snap$/,
  ];
 
  return files
    .filter((file) => !ignoredPatterns.some((p) => p.test(file.filename)))
    .filter((file) => file.patch) // Has actual diff content
    .map((file) => ({
      filename: file.filename,
      status: file.status as PRFile["status"],
      patch: file.patch!,
      additions: file.additions,
      deletions: file.deletions,
    }));
}

// ❌ Vague prompt — produces generic, unhelpful reviews
const badPrompt = `Review this code: ${diff}`;
 
// ✅ Structured prompt with specific criteria and output format
function buildReviewPrompt(file: PRFile, context: string): string {
  return `You are a senior software engineer reviewing a pull request.
Analyze the following code diff and identify specific issues.
 
## Review Criteria
1. **Bugs**: Logic errors, off-by-one, null/undefined risks, race conditions
2. **Security**: Injection vulnerabilities, missing auth checks, exposed secrets
3. **Performance**: N+1 queries, unnecessary re-renders, missing indexes
4. **Error Handling**: Swallowed errors, missing try/catch, unclear error messages
 
## File Context
Filename: ${file.filename}
Change type: ${file.status}
${context ? `Related context:\n${context}` : ""}
 
## Diff
\`\`\`
${file.patch}
\`\`\`
 
## Output Format
Respond with a JSON array of issues found. If no issues, return an empty array.
Each issue must include:
- line: the line number in the new file
- severity: "error" | "warning" | "suggestion"
- category: one of the review criteria above
- message: specific, actionable feedback (1-2 sentences)
- suggestion: optional code fix
 
Return ONLY the JSON array, no other text.`;
}

interface ReviewComment {
  line: number;
  severity: "error" | "warning" | "suggestion";
  category: string;
  message: string;
  suggestion?: string;
}
 
async function analyzeFile(
  file: PRFile,
  context: string
): Promise<ReviewComment[]> {
  const prompt = buildReviewPrompt(file, context);
 
  const response = await fetch("https://api.openai.com/v1/chat/completions", {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
    },
    body: JSON.stringify({
      model: "gpt-4o",
      messages: [{ role: "user", content: prompt }],
      temperature: 0.1, // Low temperature for consistent analysis
      response_format: { type: "json_object" },
    }),
  });
 
  const data = await response.json();
  const content = data.choices[0]?.message?.content;
 
  if (!content) return [];
 
  try {
    const parsed = JSON.parse(content);
    const comments = Array.isArray(parsed) ? parsed : parsed.issues ?? [];
    return comments.filter(isValidComment);
  } catch {
    console.error(`Failed to parse LLM response for ${file.filename}`);
    return [];
  }
}
 
function isValidComment(comment: unknown): comment is ReviewComment {
  if (typeof comment !== "object" || comment === null) return false;
  const c = comment as Record<string, unknown>;
  return (
    typeof c.line === "number" &&
    typeof c.message === "string" &&
    ["error", "warning", "suggestion"].includes(c.severity as string)
  );
}

async function postReviewComments(
  owner: string,
  repo: string,
  prNumber: number,
  commitSha: string,
  fileComments: Map<string, ReviewComment[]>
): Promise<void> {
  const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
 
  const comments: Array<{
    path: string;
    position?: number;
    line: number;
    side: "RIGHT";
    body: string;
  }> = [];
 
  for (const [filename, fileReviewComments] of fileComments) {
    for (const comment of fileReviewComments) {
      const icon =
        comment.severity === "error" ? "🔴" :
        comment.severity === "warning" ? "🟡" : "🔵";
 
      let body = `${icon} **${comment.category}** (${comment.severity})\n\n${comment.message}`;
 
      if (comment.suggestion) {
        body += `\n\n\`\`\`suggestion\n${comment.suggestion}\n\`\`\``;
      }
 
      comments.push({
        path: filename,
        line: comment.line,
        side: "RIGHT",
        body,
      });
    }
  }
 
  if (comments.length === 0) {
    await octokit.pulls.createReview({
      owner,
      repo,
      pull_number: prNumber,
      commit_id: commitSha,
      body: "✅ AI review found no issues in the changed files.",
      event: "COMMENT",
    });
    return;
  }
 
  await octokit.pulls.createReview({
    owner,
    repo,
    pull_number: prNumber,
    commit_id: commitSha,
    body: `🤖 AI Code Review — Found ${comments.length} item(s) to review.`,
    event: "COMMENT",
    comments,
  });
}

# .github/workflows/ai-review.yml
name: AI Code Review
on:
  pull_request:
    types: [opened, synchronize]
 
jobs:
  review:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
      contents: read
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
 
      - uses: actions/setup-node@v4
        with:
          node-version: "20"
 
      - name: Install dependencies
        run: npm ci
 
      - name: Run AI Review
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          node scripts/ai-review.mjs \
            --owner=${{ github.repository_owner }} \
            --repo=${{ github.event.repository.name }} \
            --pr=${{ github.event.pull_request.number }} \
            --sha=${{ github.event.pull_request.head.sha }} \
            --max-files=20 \
            --max-tokens=50000