RAG: scrape, chunk, embed

URLs ──▶ Reader ──▶ markdown ──▶ chunks ──▶ embeddings ──▶ vector DB
                                                              │
                                                  query ──▶ retrieve ──▶ LLM

import { ReaderClient } from "@vakra-dev/reader-js";

const reader = new ReaderClient({ apiKey: process.env.READER_KEY! });

const result = await reader.read({
  urls: loadCorpusUrls(), // sitemap, RSS, database, etc.
});

if (result.kind !== "job") throw new Error("expected batch job");
const pages = result.data.results.filter((p) => !p.error && p.markdown);

function chunkMarkdown(text: string, size = 500, overlap = 50): string[] {
  // Rough: split on paragraph boundaries first, then by character count.
  const paragraphs = text.split(/\n\n+/);
  const chunks: string[] = [];
  let current = "";

  for (const p of paragraphs) {
    if ((current + "\n\n" + p).length > size * 4) {
      // ~4 chars per token
      chunks.push(current.trim());
      // Keep the tail of the previous chunk as overlap
      current = current.slice(-overlap * 4) + "\n\n" + p;
    } else {
      current = current ? current + "\n\n" + p : p;
    }
  }
  if (current.trim()) chunks.push(current.trim());

  return chunks;
}

import OpenAI from "openai";
const openai = new OpenAI();

async function indexPage(page: { url: string; markdown: string; metadata: any }) {
  const chunks = chunkMarkdown(page.markdown);

  const embeddings = await openai.embeddings.create({
    model: "text-embedding-3-small",
    input: chunks,
  });

  const rows = chunks.map((text, i) => ({
    id: `${page.url}#${i}`,
    embedding: embeddings.data[i].embedding,
    text,
    sourceUrl: page.url,
    sourceTitle: page.metadata?.title ?? null,
  }));

  await vectorDB.upsert(rows);
}

for (const page of pages) {
  await indexPage(page);
}

async function answer(question: string) {
  const [queryEmbedding] = (
    await openai.embeddings.create({
      model: "text-embedding-3-small",
      input: [question],
    })
  ).data;

  const results = await vectorDB.query(queryEmbedding.embedding, { k: 5 });

  const context = results
    .map(
      (r, i) =>
        `[${i + 1}] Source: ${r.sourceTitle} (${r.sourceUrl})\n${r.text}`,
    )
    .join("\n\n---\n\n");

  // Pass `context` to your LLM alongside the question
  return callLLM({
    system: "Answer using only the provided sources. Cite source numbers.",
    user: `${context}\n\nQuestion: ${question}`,
  });
}