Webhook-driven workflows

[Your service] ──POST /v1/read──▶ [Reader]
                                      │
                                      ▼  (runs scrape / crawl)
                                      │
[Your endpoint] ◀──webhook POST──[Reader]
        │
        ▼
  Insert results into DB
  Enqueue embed job
  Mark source as "fetched"

await reader.request("POST", "/v1/webhooks", {
  url: "https://your-app.example.com/hooks/reader",
  name: "Production batch completion",
  events: ["job.completed", "job.failed"],
  secret: process.env.READER_WEBHOOK_SECRET,
});

const { data: job } = await reader.request("POST", "/v1/read", {
  urls: loadUrlsFromDatabase(),
});
await saveJobToTrackingTable(job.id, "pending");

// Express-style handler
app.post(
  "/hooks/reader",
  express.raw({ type: "application/json" }), // raw bytes for signature verification
  async (req, res) => {
    // 1. Verify signature (see Verifying webhooks guide)
    verify(req, process.env.READER_WEBHOOK_SECRET!);

    // 2. Idempotency: dedupe by delivery ID
    const deliveryId = req.headers["x-reader-delivery"] as string;
    if (await alreadyProcessed(deliveryId)) {
      res.status(200).end();
      return;
    }

    // 3. Handle the event
    const event = req.headers["x-reader-event"];
    const payload = JSON.parse(req.body.toString());

    if (event === "job.completed") {
      await handleJobComplete(payload);
    } else if (event === "job.failed") {
      await handleJobFailed(payload);
    }

    await markProcessed(deliveryId);
    res.status(200).end();
  },
);

async function handleJobComplete(payload) {
  await jobQueue.push({ type: "process-scrape-results", jobId: payload.jobId });
  // Return 200 quickly; a background worker fetches the results and processes them
}