Batch scraping

const result = await reader.read({
  urls: [
    "https://example.com/article/1",
    "https://example.com/article/2",
    "https://example.com/article/3",
  ],
});

if (result.kind === "job") {
  for (const page of result.data.results) {
    if (page.error) {
      console.warn("failed:", page.url, page.error);
    } else {
      console.log("ok:", page.url, page.markdown?.length);
    }
  }
}

// One request per URL: don't do this
for (const url of urls) {
  await reader.read({ url });
}

await reader.read({
  urls: manyUrls,
  batchConcurrency: 5, // Reader runs up to 5 scrapes in parallel
});

const failed = result.data.results.filter((r) => r.error);
const succeeded = result.data.results.filter((r) => !r.error);

console.log(`${succeeded.length} succeeded, ${failed.length} failed`);

// Retry just the failed subset if needed
if (failed.length > 0) {
  await reader.retryJob(result.data.id);
  // Reader re-queues the failed URLs and you get a fresh completion
}

const sitemapRes = await fetch("https://example.com/sitemap.xml");
const sitemapXml = await sitemapRes.text();
const urls = Array.from(sitemapXml.matchAll(/<loc>([^<]+)<\/loc>/g)).map(
  (m) => m[1],
);

// Submit in chunks of 1,000 (the max per request)
for (let i = 0; i < urls.length; i += 1000) {
  const chunk = urls.slice(i, i + 1000);
  const result = await reader.read({
    urls: chunk,
    webhook: {
      url: "https://your-app.example.com/hooks/reader",
      events: ["job.completed"],
      secret: process.env.READER_WEBHOOK_SECRET,
    },
  });
  console.log(`submitted batch ${i / 1000 + 1}, job=${result.data.id}`);
}