Building Production-Ready RAG Pipelines with LangChain
Design retrieval-augmented generation pipelines that go beyond demos: chunking strategies, embedding optimization, reranking and evaluation metrics.
Every RAG tutorial shows the same thing: load a PDF, split it into chunks, embed them, and ask questions. The demo works beautifully. Then you try it with real documents and the answers are wrong, hallucinated, or irrelevant. The gap between demo RAG and production RAG is enormous, and it lives in the details of chunking, retrieval, and evaluation.
Chunking Strategy Matters More Than Model Choice
The most impactful decision in a RAG pipeline isn't which LLM you use—it's how you split your documents. Bad chunking produces bad retrieval, and no amount of model sophistication can fix garbage input.
// ❌ Naive fixed-size chunking breaks semantic boundaries
interface NaiveChunkConfig {
chunkSize: number;
overlap: number;
}
function naiveChunk(text: string, config: NaiveChunkConfig): string[] {
const chunks: string[] = [];
for (let i = 0; i < text.length; i += config.chunkSize - config.overlap) {
chunks.push(text.slice(i, i + config.chunkSize));
}
return chunks;
// Problem: cuts mid-sentence, splits code blocks,
// separates headers from their content
}// ✅ Semantic-aware chunking preserves meaning
interface SemanticChunkConfig {
maxChunkTokens: number;
minChunkTokens: number;
overlapSentences: number;
preserveStructure: boolean;
}
interface DocumentChunk {
content: string;
metadata: {
source: string;
section: string;
pageNumber?: number;
chunkIndex: number;
tokenCount: number;
};
}
function semanticChunk(
document: string,
config: SemanticChunkConfig
): DocumentChunk[] {
const sections = splitBySections(document);
const chunks: DocumentChunk[] = [];
let chunkIndex = 0;
for (const section of sections) {
const sentences = splitSentences(section.content);
let currentChunk: string[] = [];
let currentTokens = 0;
for (const sentence of sentences) {
const sentenceTokens = estimateTokens(sentence);
if (
currentTokens + sentenceTokens > config.maxChunkTokens &&
currentTokens >= config.minChunkTokens
) {
chunks.push({
content: currentChunk.join(" "),
metadata: {
source: document.slice(0, 50),
section: section.heading,
chunkIndex: chunkIndex++,
tokenCount: currentTokens,
},
});
// Keep overlap sentences for context continuity
const overlapStart = Math.max(
0,
currentChunk.length - config.overlapSentences
);
currentChunk = currentChunk.slice(overlapStart);
currentTokens = currentChunk
.reduce((s, sent) => s + estimateTokens(sent), 0);
}
currentChunk.push(sentence);
currentTokens += sentenceTokens;
}
// Flush remaining content
if (currentChunk.length > 0) {
chunks.push({
content: currentChunk.join(" "),
metadata: {
source: document.slice(0, 50),
section: section.heading,
chunkIndex: chunkIndex++,
tokenCount: currentTokens,
},
});
}
}
return chunks;
}
function splitBySections(text: string): { heading: string; content: string }[] {
const sectionPattern = /^(#{1,3})\s+(.+)$/gm;
const sections: { heading: string; content: string }[] = [];
let lastIndex = 0;
let lastHeading = "Introduction";
let match: RegExpExecArray | null;
while ((match = sectionPattern.exec(text)) !== null) {
if (match.index > lastIndex) {
sections.push({
heading: lastHeading,
content: text.slice(lastIndex, match.index).trim(),
});
}
lastHeading = match[2];
lastIndex = match.index + match[0].length;
}
if (lastIndex < text.length) {
sections.push({
heading: lastHeading,
content: text.slice(lastIndex).trim(),
});
}
return sections;
}
function splitSentences(text: string): string[] {
return text.split(/(?<=[.!?])\s+/).filter(s => s.length > 0);
}
function estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
}Section-aware chunking preserves the relationship between headings and their content. The metadata attached to each chunk enables filtered retrieval: "find chunks from the 'Authentication' section."
Hybrid Retrieval: Dense + Sparse
Vector similarity search (dense retrieval) excels at semantic matching but misses exact keyword matches. BM25 (sparse retrieval) finds exact terms but misses semantic equivalents. Combining both outperforms either alone.
interface RetrievalResult {
chunk: DocumentChunk;
score: number;
source: "dense" | "sparse" | "hybrid";
}
interface HybridRetrieverConfig {
denseWeight: number; // 0-1, weight for vector search
sparseWeight: number; // 0-1, weight for BM25
topK: number;
rerankEnabled: boolean;
}
class HybridRetriever {
private config: HybridRetrieverConfig;
constructor(config: HybridRetrieverConfig) {
this.config = config;
}
async retrieve(
query: string,
denseResults: RetrievalResult[],
sparseResults: RetrievalResult[]
): Promise<RetrievalResult[]> {
// Normalize scores to 0-1 range
const normalizedDense = this.normalizeScores(denseResults);
const normalizedSparse = this.normalizeScores(sparseResults);
// Reciprocal Rank Fusion for combining results
const fusedScores = new Map<string, number>();
const chunkMap = new Map<string, DocumentChunk>();
const k = 60; // RRF constant
normalizedDense.forEach((result, rank) => {
const key = result.chunk.metadata.chunkIndex.toString();
const rrfScore = this.config.denseWeight / (k + rank + 1);
fusedScores.set(key, (fusedScores.get(key) ?? 0) + rrfScore);
chunkMap.set(key, result.chunk);
});
normalizedSparse.forEach((result, rank) => {
const key = result.chunk.metadata.chunkIndex.toString();
const rrfScore = this.config.sparseWeight / (k + rank + 1);
fusedScores.set(key, (fusedScores.get(key) ?? 0) + rrfScore);
chunkMap.set(key, result.chunk);
});
// Sort by fused score
const results: RetrievalResult[] = Array.from(fusedScores.entries())
.sort(([, a], [, b]) => b - a)
.slice(0, this.config.topK)
.map(([key, score]) => ({
chunk: chunkMap.get(key)!,
score,
source: "hybrid" as const,
}));
return results;
}
private normalizeScores(
results: RetrievalResult[]
): RetrievalResult[] {
if (results.length === 0) return [];
const maxScore = Math.max(...results.map(r => r.score));
const minScore = Math.min(...results.map(r => r.score));
const range = maxScore - minScore || 1;
return results.map(r => ({
...r,
score: (r.score - minScore) / range,
}));
}
}Reciprocal Rank Fusion (RRF) is particularly effective because it doesn't require the scores from different systems to be on the same scale—it only uses rank positions.
Context Window Assembly
Retrieved chunks need careful assembly before being sent to the LLM. Simply concatenating top-K results often includes redundant information and wastes context window tokens.
interface ContextWindow {
systemPrompt: string;
retrievedContext: string;
userQuery: string;
totalTokens: number;
maxTokens: number;
}
function assembleContext(
query: string,
results: RetrievalResult[],
maxContextTokens: number
): ContextWindow {
const systemPrompt =
"Answer the question based on the provided context. " +
"If the context doesn't contain enough information, say so. " +
"Cite the source section when possible.";
// Deduplicate overlapping chunks
const deduplicated = deduplicateChunks(results);
// Group by section for coherent reading
const grouped = groupBySection(deduplicated);
// Build context within token budget
let contextParts: string[] = [];
let currentTokens = 0;
for (const [section, chunks] of grouped) {
const sectionHeader = `[Section: ${section}]`;
const sectionTokens = estimateTokens(sectionHeader);
if (currentTokens + sectionTokens > maxContextTokens) break;
contextParts.push(sectionHeader);
currentTokens += sectionTokens;
for (const chunk of chunks) {
const chunkTokens = chunk.chunk.metadata.tokenCount;
if (currentTokens + chunkTokens > maxContextTokens) break;
contextParts.push(chunk.chunk.content);
currentTokens += chunkTokens;
}
}
const retrievedContext = contextParts.join("\n\n");
const totalTokens =
estimateTokens(systemPrompt) +
currentTokens +
estimateTokens(query);
return {
systemPrompt,
retrievedContext,
userQuery: query,
totalTokens,
maxTokens: maxContextTokens,
};
}
function deduplicateChunks(
results: RetrievalResult[]
): RetrievalResult[] {
const seen = new Set<number>();
return results.filter(r => {
if (seen.has(r.chunk.metadata.chunkIndex)) return false;
seen.add(r.chunk.metadata.chunkIndex);
return true;
});
}
function groupBySection(
results: RetrievalResult[]
): Map<string, RetrievalResult[]> {
const groups = new Map<string, RetrievalResult[]>();
for (const result of results) {
const section = result.chunk.metadata.section;
const existing = groups.get(section) ?? [];
existing.push(result);
groups.set(section, existing);
}
return groups;
}Evaluating RAG Quality
Without evaluation, you're guessing whether your RAG pipeline actually works. Three metrics matter most: retrieval relevance, answer correctness, and faithfulness (does the answer stick to the retrieved context?).
interface RAGEvaluation {
query: string;
expectedAnswer: string;
retrievedChunks: DocumentChunk[];
generatedAnswer: string;
metrics: {
retrievalRelevance: number; // 0-1: are retrieved chunks relevant?
answerCorrectness: number; // 0-1: is the answer correct?
faithfulness: number; // 0-1: does answer follow from context?
contextUtilization: number; // 0-1: how much context was used?
};
}
function calculateRetrievalRelevance(
chunks: DocumentChunk[],
relevantChunkIds: Set<number>
): number {
if (chunks.length === 0) return 0;
const relevant = chunks.filter(c =>
relevantChunkIds.has(c.metadata.chunkIndex)
);
return relevant.length / chunks.length;
}
interface EvalDataset {
queries: {
query: string;
expectedAnswer: string;
relevantChunkIds: number[];
}[];
}
function runEvaluation(
pipeline: RAGPipeline,
dataset: EvalDataset
): { avgRelevance: number; avgCorrectness: number } {
let totalRelevance = 0;
let totalCorrectness = 0;
for (const item of dataset.queries) {
const result = pipeline.query(item.query);
const relevance = calculateRetrievalRelevance(
result.chunks,
new Set(item.relevantChunkIds)
);
totalRelevance += relevance;
// Correctness typically requires LLM-as-judge evaluation
}
return {
avgRelevance: totalRelevance / dataset.queries.length,
avgCorrectness: totalCorrectness / dataset.queries.length,
};
}Build evaluation datasets from real user questions. Start with 50-100 query-answer pairs, manually verify the expected answers, and run evaluations after every pipeline change. Without this feedback loop, you're optimizing blindly.
Key Takeaways
Production RAG pipelines succeed or fail on three axes: chunking quality, retrieval precision, and systematic evaluation. Chunk documents along semantic boundaries—section headers, paragraph breaks, and logical units—rather than fixed character counts. Combine dense vector search with sparse BM25 retrieval using Reciprocal Rank Fusion to catch both semantic matches and exact keyword hits. Assemble context windows carefully by deduplicating overlapping chunks and grouping by section for coherent reading. Most importantly, build an evaluation dataset from real queries and measure retrieval relevance and answer correctness after every change. The teams that build reliable RAG systems treat it as an information retrieval engineering problem, not a prompt engineering problem.
