Crawl + scrape in one call

const result = await reader.read({
  url: "https://docs.example.com",
  maxDepth: 3,
  maxPages: 100,
});

if (result.kind === "job") {
  for (const page of result.data.results) {
    console.log(page.url);
    console.log(page.markdown?.slice(0, 200));
    console.log("---");
  }
}

await reader.read({
  url: "https://docs.example.com",
  maxDepth: 3,
  maxPages: 100,
  onlyMainContent: true,       // default, strips nav/footer from every page
  formats: ["markdown", "html"], // both formats for every page
  excludeTags: [".edit-on-github", ".feedback-widget"],
});

const crawled = await reader.read({
  url: "https://blog.example.com",
  maxPages: 200,
});

if (crawled.kind === "job") {
  for (const page of crawled.data.results) {
    if (page.error) continue;
    await vectorStore.upsert({
      id: page.url,
      text: page.markdown!,
      metadata: { title: page.metadata?.title, scrapedAt: page.metadata?.scrapedAt },
    });
  }
}