Scrape, batch, crawl, or discover

Scrape, batch, or crawl

curl --request POST \
  --url https://api.reader.dev/v1/read \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <api-key>' \
  --data '
{
  "url": "https://example.com",
  "urls": [
    "<string>"
  ],
  "formats": [
    "markdown"
  ],
  "onlyMainContent": true,
  "includeTags": [
    "<string>"
  ],
  "excludeTags": [
    "<string>"
  ],
  "waitForSelector": "<string>",
  "timeoutMs": 60500,
  "proxyMode": "standard",
  "batchConcurrency": 10,
  "maxConcurrency": 2,
  "maxDepth": 5,
  "maxPages": 5000,
  "scrape": true,
  "cache": true
}
'

import requests

url = "https://api.reader.dev/v1/read"

payload = {
    "url": "https://example.com",
    "urls": ["<string>"],
    "formats": ["markdown"],
    "onlyMainContent": True,
    "includeTags": ["<string>"],
    "excludeTags": ["<string>"],
    "waitForSelector": "<string>",
    "timeoutMs": 60500,
    "proxyMode": "standard",
    "batchConcurrency": 10,
    "maxConcurrency": 2,
    "maxDepth": 5,
    "maxPages": 5000,
    "scrape": True,
    "cache": True
}
headers = {
    "x-api-key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: 'https://example.com',
    urls: ['<string>'],
    formats: ['markdown'],
    onlyMainContent: true,
    includeTags: ['<string>'],
    excludeTags: ['<string>'],
    waitForSelector: '<string>',
    timeoutMs: 60500,
    proxyMode: 'standard',
    batchConcurrency: 10,
    maxConcurrency: 2,
    maxDepth: 5,
    maxPages: 5000,
    scrape: true,
    cache: true
  })
};

fetch('https://api.reader.dev/v1/read', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.reader.dev/v1/read",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'https://example.com',
    'urls' => [
        '<string>'
    ],
    'formats' => [
        'markdown'
    ],
    'onlyMainContent' => true,
    'includeTags' => [
        '<string>'
    ],
    'excludeTags' => [
        '<string>'
    ],
    'waitForSelector' => '<string>',
    'timeoutMs' => 60500,
    'proxyMode' => 'standard',
    'batchConcurrency' => 10,
    'maxConcurrency' => 2,
    'maxDepth' => 5,
    'maxPages' => 5000,
    'scrape' => true,
    'cache' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.reader.dev/v1/read"

	payload := strings.NewReader("{\n  \"url\": \"https://example.com\",\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"waitForSelector\": \"<string>\",\n  \"timeoutMs\": 60500,\n  \"proxyMode\": \"standard\",\n  \"batchConcurrency\": 10,\n  \"maxConcurrency\": 2,\n  \"maxDepth\": 5,\n  \"maxPages\": 5000,\n  \"scrape\": true,\n  \"cache\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("x-api-key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.reader.dev/v1/read")
  .header("x-api-key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"https://example.com\",\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"waitForSelector\": \"<string>\",\n  \"timeoutMs\": 60500,\n  \"proxyMode\": \"standard\",\n  \"batchConcurrency\": 10,\n  \"maxConcurrency\": 2,\n  \"maxDepth\": 5,\n  \"maxPages\": 5000,\n  \"scrape\": true,\n  \"cache\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.reader.dev/v1/read")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"https://example.com\",\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"waitForSelector\": \"<string>\",\n  \"timeoutMs\": 60500,\n  \"proxyMode\": \"standard\",\n  \"batchConcurrency\": 10,\n  \"maxConcurrency\": 2,\n  \"maxDepth\": 5,\n  \"maxPages\": 5000,\n  \"scrape\": true,\n  \"cache\": true\n}"

response = http.request(request)
puts response.read_body

{
  "success": true,
  "data": {
    "url": "https://example.com",
    "markdown": "# Example Domain\n\nThis domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.\n\n[More information...](https://www.iana.org/domains/example)",
    "metadata": {
      "title": "Example Domain",
      "description": null,
      "statusCode": 200,
      "duration": 487,
      "cached": false,
      "proxyMode": "standard",
      "scrapedAt": "2026-04-04T12:00:00Z"
    }
  }
}

{
  "success": true,
  "data": {
    "id": "job_9fba2",
    "status": "queued",
    "mode": "batch",
    "total": 3,
    "completed": 0,
    "creditsUsed": 0,
    "createdAt": "2026-04-04T12:00:00Z"
  }
}

{
  "success": false,
  "error": {
    "code": "invalid_request",
    "message": "Invalid request body",
    "details": {
      "issues": [
        {
          "path": "url",
          "code": "invalid_string",
          "message": "Invalid url"
        }
      ]
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#invalid-request"
  }
}

{
  "success": false,
  "error": {
    "code": "unauthenticated",
    "message": "Missing or invalid API key",
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#unauthenticated"
  }
}

{
  "success": false,
  "error": {
    "code": "insufficient_credits",
    "message": "You need 50 credits but only 10 are available. Wait for your credits to reset at the next billing cycle.",
    "details": {
      "required": 50,
      "available": 10,
      "resetAt": "2026-05-01T00:00:00Z"
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#insufficient-credits"
  }
}

{
  "success": false,
  "error": {
    "code": "url_blocked",
    "message": "URL blocked: Resolves to private IP",
    "details": {
      "url": "https://10.0.0.1",
      "reason": "Resolves to private IP"
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#url-blocked"
  }
}

{
  "success": false,
  "error": {
    "code": "rate_limited",
    "message": "Rate limit of 60 requests per 60s exceeded. Retry after 12s.",
    "details": {
      "limit": 60,
      "windowSeconds": 60,
      "retryAfterSeconds": 12
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#rate-limited"
  }
}

{
  "success": false,
  "error": {
    "code": "upstream_unavailable",
    "message": "Upstream service is unavailable",
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#upstream-unavailable"
  }
}

{
  "success": false,
  "error": {
    "code": "scrape_timeout",
    "message": "Scrape exceeded 30000ms timeout",
    "details": {
      "timeoutMs": 30000
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#scrape-timeout"
  }
}

POST

read

Scrape, batch, or crawl

curl --request POST \
  --url https://api.reader.dev/v1/read \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <api-key>' \
  --data '
{
  "url": "https://example.com",
  "urls": [
    "<string>"
  ],
  "formats": [
    "markdown"
  ],
  "onlyMainContent": true,
  "includeTags": [
    "<string>"
  ],
  "excludeTags": [
    "<string>"
  ],
  "waitForSelector": "<string>",
  "timeoutMs": 60500,
  "proxyMode": "standard",
  "batchConcurrency": 10,
  "maxConcurrency": 2,
  "maxDepth": 5,
  "maxPages": 5000,
  "scrape": true,
  "cache": true
}
'

import requests

url = "https://api.reader.dev/v1/read"

payload = {
    "url": "https://example.com",
    "urls": ["<string>"],
    "formats": ["markdown"],
    "onlyMainContent": True,
    "includeTags": ["<string>"],
    "excludeTags": ["<string>"],
    "waitForSelector": "<string>",
    "timeoutMs": 60500,
    "proxyMode": "standard",
    "batchConcurrency": 10,
    "maxConcurrency": 2,
    "maxDepth": 5,
    "maxPages": 5000,
    "scrape": True,
    "cache": True
}
headers = {
    "x-api-key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: 'https://example.com',
    urls: ['<string>'],
    formats: ['markdown'],
    onlyMainContent: true,
    includeTags: ['<string>'],
    excludeTags: ['<string>'],
    waitForSelector: '<string>',
    timeoutMs: 60500,
    proxyMode: 'standard',
    batchConcurrency: 10,
    maxConcurrency: 2,
    maxDepth: 5,
    maxPages: 5000,
    scrape: true,
    cache: true
  })
};

fetch('https://api.reader.dev/v1/read', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.reader.dev/v1/read",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'https://example.com',
    'urls' => [
        '<string>'
    ],
    'formats' => [
        'markdown'
    ],
    'onlyMainContent' => true,
    'includeTags' => [
        '<string>'
    ],
    'excludeTags' => [
        '<string>'
    ],
    'waitForSelector' => '<string>',
    'timeoutMs' => 60500,
    'proxyMode' => 'standard',
    'batchConcurrency' => 10,
    'maxConcurrency' => 2,
    'maxDepth' => 5,
    'maxPages' => 5000,
    'scrape' => true,
    'cache' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.reader.dev/v1/read"

	payload := strings.NewReader("{\n  \"url\": \"https://example.com\",\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"waitForSelector\": \"<string>\",\n  \"timeoutMs\": 60500,\n  \"proxyMode\": \"standard\",\n  \"batchConcurrency\": 10,\n  \"maxConcurrency\": 2,\n  \"maxDepth\": 5,\n  \"maxPages\": 5000,\n  \"scrape\": true,\n  \"cache\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("x-api-key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.reader.dev/v1/read")
  .header("x-api-key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"https://example.com\",\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"waitForSelector\": \"<string>\",\n  \"timeoutMs\": 60500,\n  \"proxyMode\": \"standard\",\n  \"batchConcurrency\": 10,\n  \"maxConcurrency\": 2,\n  \"maxDepth\": 5,\n  \"maxPages\": 5000,\n  \"scrape\": true,\n  \"cache\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.reader.dev/v1/read")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"https://example.com\",\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"waitForSelector\": \"<string>\",\n  \"timeoutMs\": 60500,\n  \"proxyMode\": \"standard\",\n  \"batchConcurrency\": 10,\n  \"maxConcurrency\": 2,\n  \"maxDepth\": 5,\n  \"maxPages\": 5000,\n  \"scrape\": true,\n  \"cache\": true\n}"

response = http.request(request)
puts response.read_body

{
  "success": true,
  "data": {
    "url": "https://example.com",
    "markdown": "# Example Domain\n\nThis domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.\n\n[More information...](https://www.iana.org/domains/example)",
    "metadata": {
      "title": "Example Domain",
      "description": null,
      "statusCode": 200,
      "duration": 487,
      "cached": false,
      "proxyMode": "standard",
      "scrapedAt": "2026-04-04T12:00:00Z"
    }
  }
}

{
  "success": true,
  "data": {
    "id": "job_9fba2",
    "status": "queued",
    "mode": "batch",
    "total": 3,
    "completed": 0,
    "creditsUsed": 0,
    "createdAt": "2026-04-04T12:00:00Z"
  }
}

{
  "success": false,
  "error": {
    "code": "invalid_request",
    "message": "Invalid request body",
    "details": {
      "issues": [
        {
          "path": "url",
          "code": "invalid_string",
          "message": "Invalid url"
        }
      ]
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#invalid-request"
  }
}

{
  "success": false,
  "error": {
    "code": "unauthenticated",
    "message": "Missing or invalid API key",
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#unauthenticated"
  }
}

{
  "success": false,
  "error": {
    "code": "insufficient_credits",
    "message": "You need 50 credits but only 10 are available. Wait for your credits to reset at the next billing cycle.",
    "details": {
      "required": 50,
      "available": 10,
      "resetAt": "2026-05-01T00:00:00Z"
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#insufficient-credits"
  }
}

{
  "success": false,
  "error": {
    "code": "url_blocked",
    "message": "URL blocked: Resolves to private IP",
    "details": {
      "url": "https://10.0.0.1",
      "reason": "Resolves to private IP"
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#url-blocked"
  }
}

{
  "success": false,
  "error": {
    "code": "rate_limited",
    "message": "Rate limit of 60 requests per 60s exceeded. Retry after 12s.",
    "details": {
      "limit": 60,
      "windowSeconds": 60,
      "retryAfterSeconds": 12
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#rate-limited"
  }
}

{
  "success": false,
  "error": {
    "code": "upstream_unavailable",
    "message": "Upstream service is unavailable",
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#upstream-unavailable"
  }
}

{
  "success": false,
  "error": {
    "code": "scrape_timeout",
    "message": "Scrape exceeded 30000ms timeout",
    "details": {
      "timeoutMs": 30000
    },
    "docsUrl": "https://reader.dev/docs/home/concepts/errors#scrape-timeout"
  }
}

POST /v1/read is the unified content-extraction endpoint. It auto-detects the operation from the body:

Single url - synchronous scrape, returned immediately in the response
Multiple urls - async batch job
url + maxDepth or maxPages - async crawl job (discover + scrape)
url + maxDepth or maxPages + scrape: false - async discover job (URLs only, no content)

See The read primitive for a narrative overview.

Proxy mode

Set proxyMode to "standard" (1 credit, fast, default) or "premium" (3 credits, residential proxies, bypasses bot walls). The response metadata tells you which mode ran. See Proxy modes.

Concurrency

Set maxConcurrency to limit how many browser slots a batch or crawl job uses. Prevents one large job from consuming all your plan’s concurrent browser slots. Capped to your plan’s limit. Check your current usage with Queue status.

Extract

Add the extract parameter to pull structured data from the page alongside markdown. Provide a JSON Schema, a shorthand schema, a natural language prompt, or both.

curl -X POST https://api.reader.dev/v1/read \
  -H "x-api-key: $READER_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com/product",
    "extract": {
      "schema": { "title": "string", "price": "number", "in_stock": "boolean" }
    }
  }'

The response includes an extracted field with the structured data and extraction metadata. Adds 2 credits to the scrape cost. Only supported for single-URL scrapes (not batch or crawl). See Extract for details.

Idempotency

Pass an x-idempotency-key header to deduplicate retried POSTs. Reader caches the original response for 24 hours and returns it verbatim on any subsequent request with the same key.

curl -X POST https://api.reader.dev/v1/read \
  -H "x-api-key: $READER_KEY" \
  -H "x-idempotency-key: batch-2026-04-04-run-1" \
  -H "Content-Type: application/json" \
  -d '{ "urls": ["https://example.com/a", "https://example.com/b"] }'

Discover mode

Set scrape to false on a crawl request to discover URLs without extracting content. Results contain URL, title, and description only. Costs 1 credit per discover job regardless of how many URLs are found.

curl -X POST https://api.reader.dev/v1/read \
  -H "x-api-key: $READER_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://docs.example.com",
    "maxDepth": 3,
    "maxPages": 500,
    "scrape": false
  }'

See Scrape, crawl, and discover for usage patterns.

Authorizations

x-api-key

string

header

required

Body

application/json

url

string<uri>

Example:

"https://example.com"

urls

string<uri>[]

Required array length: 1 - 1000 elements

formats

enum<string>[]

Content formats to include in the response.

Available options:

markdown,

html,

screenshot

Example:

["markdown"]

onlyMainContent

boolean

Strip navigation, footers, and boilerplate. Default: true.

includeTags

string[]

excludeTags

string[]

waitForSelector

string

timeoutMs

integer

Required range: 1000 <= x <= 120000

proxyMode

enum<string>

Proxy mode for the scrape. standard uses datacenter proxies (default, 1 credit); premium uses residential proxies for better bot bypass (3 credits).

Available options:

standard,

premium

Example:

"standard"

batchConcurrency

integer

Required range: 1 <= x <= 20

maxConcurrency

integer

Max concurrent browser slots for this request. Capped to your plan's limit.

Required range: x >= 1

maxDepth

integer

Crawl depth (when crawling). Omit for single-URL scrape.

Required range: 1 <= x <= 10

maxPages

integer

Maximum pages to discover during crawl.

Required range: 1 <= x <= 10000

scrape

boolean

When false, crawl discovers URLs without scraping their content (discover mode). Only applies when maxDepth or maxPages is set. Default: true.

cache

boolean

Reuse cached content within TTL. Default: true.

webhook

object

Show child attributes

Response

Sync scrape completed

success

enum<boolean>

required

Available options:

true

data

object

required

Show child attributes

Getting Started

Jobs

Webhooks

Sessions

Account

Scrape, batch, crawl, or discover

Proxy mode

Concurrency

Extract

Idempotency

Discover mode

Authorizations

Body

Response

​Proxy mode

​Concurrency

​Extract

​Idempotency

​Discover mode

Authorizations

Body

Response

Proxy mode

Concurrency

Extract

Idempotency

Discover mode