RAG Explained for Developers

Without RAG:
User: "What's our refund policy for enterprise customers?"
LLM: [makes something up based on general training data]

With RAG:
User: "What's our refund policy for enterprise customers?"
System: [retrieves actual policy document from knowledge base]
LLM: [generates answer grounded in the real policy document]
     "Enterprise customers can request a full refund within 30 days..."
     Source: internal-docs/refund-policy-v3.md

RAG Pipeline:

┌─────────────────────────────────────────────────────────┐
│  1. QUERY         User asks a question                  │
│     ↓                                                    │
│  2. EMBED         Convert question to vector            │
│     ↓                                                    │
│  3. SEARCH        Find similar vectors in database      │
│     ↓                                                    │
│  4. RETRIEVE      Pull matching document chunks         │
│     ↓                                                    │
│  5. GENERATE      LLM answers using retrieved context   │
└─────────────────────────────────────────────────────────┘

Indexing Phase (offline, done once):
Documents → Chunk → Embed → Store in Vector DB

Query Phase (real-time, per request):
Question → Embed → Search → Retrieve → Augment Prompt → Generate

// Conceptual view of embeddings
embed("How do I reset my password?")  → [0.12, -0.34, 0.78, ...] // 1536 dimensions
embed("Password recovery process")    → [0.11, -0.32, 0.76, ...] // Very similar!
embed("What's for lunch today?")      → [-0.45, 0.67, -0.21, ...] // Very different

// Similarity is measured by cosine distance between vectors
cosineSimilarity(query, doc1) = 0.94  // Highly relevant
cosineSimilarity(query, doc2) = 0.23  // Not relevant

import OpenAI from 'openai';
import { Pinecone } from '@pinecone-database/pinecone';
import { readFileSync, readdirSync } from 'fs';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
const pinecone = new Pinecone({ apiKey: process.env.PINECONE_API_KEY });
const index = pinecone.index('knowledge-base');

// Chunk documents into smaller pieces
function chunkDocument(text, metadata, chunkSize = 500, overlap = 50) {
  const sentences = text.split(/(?<=[.!?])\s+/);
  const chunks = [];
  let current = [];
  let currentLength = 0;

  for (const sentence of sentences) {
    const words = sentence.split(/\s+/).length;
    if (currentLength + words > chunkSize && current.length > 0) {
      chunks.push({
        text: current.join(' '),
        metadata: { ...metadata, chunkIndex: chunks.length }
      });
      // Keep last few sentences for overlap
      const overlapSentences = current.slice(-2);
      current = [...overlapSentences, sentence];
      currentLength = current.join(' ').split(/\s+/).length;
    } else {
      current.push(sentence);
      currentLength += words;
    }
  }
  if (current.length > 0) {
    chunks.push({ text: current.join(' '), metadata: { ...metadata, chunkIndex: chunks.length } });
  }
  return chunks;
}

// Embed and store chunks
async function indexDocuments(docsDir) {
  const files = readdirSync(docsDir).filter(f => f.endsWith('.md'));
  
  for (const file of files) {
    const content = readFileSync(`${docsDir}/${file}`, 'utf-8');
    const chunks = chunkDocument(content, { source: file, indexed: new Date().toISOString() });
    
    // Batch embed (max 2048 inputs per request)
    const embedResponse = await openai.embeddings.create({
      model: "text-embedding-3-small",
      input: chunks.map(c => c.text)
    });
    
    // Upsert to Pinecone
    const vectors = chunks.map((chunk, i) => ({
      id: `${file}-chunk-${i}`,
      values: embedResponse.data[i].embedding,
      metadata: { ...chunk.metadata, text: chunk.text }
    }));
    
    await index.upsert(vectors);
    console.log(`Indexed ${chunks.length} chunks from ${file}`);
  }
}

async function retrieveContext(question, topK = 5) {
  // Embed the question
  const queryEmbedding = await openai.embeddings.create({
    model: "text-embedding-3-small",
    input: question
  });
  
  // Search vector database
  const results = await index.query({
    vector: queryEmbedding.data[0].embedding,
    topK,
    includeMetadata: true
  });
  
  // Return chunks with relevance scores
  return results.matches.map(match => ({
    text: match.metadata.text,
    source: match.metadata.source,
    score: match.score
  }));
}

async function ragQuery(question) {
  // Retrieve relevant context
  const context = await retrieveContext(question);
  
  // Build augmented prompt
  const contextText = context
    .map((c, i) => `[Source ${i + 1}: ${c.source}]\n${c.text}`)
    .join('\n\n---\n\n');
  
  // Generate answer grounded in retrieved documents
  const response = await openai.chat.completions.create({
    model: "gpt-4o",
    temperature: 0.2,
    messages: [
      {
        role: "system",
        content: `You are a helpful assistant that answers questions based on the provided context documents.

Rules:
- Answer ONLY based on the provided context
- If the context doesn't contain the answer, say "I don't have information about that in my knowledge base"
- Cite sources using [Source N] format
- Be concise and direct
- Never make up information not present in the context`
      },
      {
        role: "user",
        content: `Context Documents:\n\n${contextText}\n\n---\n\nQuestion: ${question}`
      }
    ]
  });
  
  return {
    answer: response.choices[0].message.content,
    sources: context.map(c => c.source),
    relevanceScores: context.map(c => c.score)
  };
}

// Usage
const result = await ragQuery("How do I configure SSO for our application?");
console.log(result.answer);    // Grounded answer citing specific docs
console.log(result.sources);   // ["sso-setup-guide.md", "auth-config.md"]

-- Setup pgvector extension
CREATE EXTENSION IF NOT EXISTS vector;

-- Create table for document chunks
CREATE TABLE document_chunks (
  id SERIAL PRIMARY KEY,
  content TEXT NOT NULL,
  source VARCHAR(255),
  chunk_index INTEGER,
  embedding vector(1536), -- OpenAI embedding dimensions
  created_at TIMESTAMP DEFAULT NOW()
);

-- Create index for fast similarity search
CREATE INDEX ON document_chunks 
  USING ivfflat (embedding vector_cosine_ops)
  WITH (lists = 100);

-- Query: find top 5 most similar chunks
SELECT content, source, 
  1 - (embedding <=> $1::vector) as similarity
FROM document_chunks
WHERE 1 - (embedding <=> $1::vector) > 0.7  -- minimum threshold
ORDER BY embedding <=> $1::vector
LIMIT 5;

// Fixed-size chunking with overlap
function fixedChunk(text, size = 400, overlap = 50) {
  const words = text.split(/\s+/);
  const chunks = [];
  for (let i = 0; i < words.length; i += size - overlap) {
    chunks.push(words.slice(i, i + size).join(' '));
  }
  return chunks;
}
// Pro: Simple, predictable chunk sizes
// Con: May split mid-sentence or mid-concept

// Semantic chunking: split by markdown headers
function semanticChunk(markdown) {
  const sections = markdown.split(/^#{1,3}\s+/m);
  return sections
    .filter(s => s.trim().length > 50) // Skip tiny fragments
    .map(section => {
      // Include the header in the chunk for context
      const lines = section.split('\n');
      return lines.join('\n').trim();
    });
}
// Pro: Preserves complete ideas and context
// Con: Variable chunk sizes, some chunks may be too large

// Recursive chunking: progressively split large chunks
function recursiveChunk(text, maxSize = 500, separators = ['\n\n', '\n', '. ', ' ']) {
  if (text.split(/\s+/).length <= maxSize) return [text];
  
  const separator = separators[0];
  const parts = text.split(separator);
  const chunks = [];
  let current = '';
  
  for (const part of parts) {
    if ((current + part).split(/\s+/).length > maxSize) {
      if (current) chunks.push(current.trim());
      // Recursively split if still too large
      if (part.split(/\s+/).length > maxSize && separators.length > 1) {
        chunks.push(...recursiveChunk(part, maxSize, separators.slice(1)));
      } else {
        current = part;
      }
    } else {
      current += separator + part;
    }
  }
  if (current.trim()) chunks.push(current.trim());
  return chunks;
}
// Pro: Best balance of preserving context and controlling size
// Con: More complex implementation