CLI Usage

npm install -g @vakra-dev/reader

npx @vakra-dev/reader scrape https://example.com

# Scrape a single URL
reader scrape https://example.com

# Scrape multiple URLs
reader scrape https://example.com https://example.org

# Short alias
reader s https://example.com

# Markdown only (default)
reader scrape https://example.com

# Multiple formats
reader scrape https://example.com -f markdown,html

# Save to file
reader scrape https://example.com -o output.json

# Scrape multiple URLs concurrently
reader scrape url1 url2 url3 url4 url5 -c 3

# Set per-page timeout
reader scrape https://example.com -t 60000

# Set batch timeout
reader scrape url1 url2 url3 --batch-timeout 300000

# Disable main content extraction (full page)
reader scrape https://example.com --no-main-content

# Include specific elements
reader scrape https://example.com --include-tags ".article,.content"

# Exclude specific elements
reader scrape https://example.com --exclude-tags ".comments,.sidebar"

reader scrape https://example.com --proxy http://user:[email protected]:8080

# Verbose logging
reader scrape https://example.com -v

# Show browser window
reader scrape https://example.com --show-chrome

# Crawl a website
reader crawl https://example.com

# Short alias
reader c https://example.com

# Set crawl depth
reader crawl https://example.com -d 3

# Limit pages
reader crawl https://example.com -m 100

# Both
reader crawl https://example.com -d 3 -m 100

# Crawl and scrape content
reader crawl https://example.com -d 2 --scrape

# With format
reader crawl https://example.com --scrape -f markdown

# Include patterns
reader crawl https://example.com --include "blog/*,docs/*"

# Exclude patterns
reader crawl https://example.com --exclude "admin/*,api/*"

# Set delay between requests
reader crawl https://example.com --delay 2000

# Start with default settings
reader start

# Custom pool size
reader start --pool-size 5

# Custom port
reader start -p 4000

reader status

reader stop

# Start daemon
reader start --pool-size 5

# These commands use the daemon's browser pool
reader scrape https://example.com
reader scrape https://example.org
reader crawl https://example.net

# Bypass daemon (standalone mode)
reader scrape https://example.com --standalone

# Stop daemon when done
reader stop

{
  "data": [
    {
      "markdown": "# Page Title\n\nContent...",
      "html": "<h1>Page Title</h1>...",
      "metadata": {
        "baseUrl": "https://example.com",
        "scrapedAt": "2024-01-15T10:30:00Z",
        "duration": 1234,
        "website": {
          "title": "Page Title",
          "description": "Page description"
        }
      }
    }
  ],
  "batchMetadata": {
    "totalUrls": 1,
    "successfulUrls": 1,
    "failedUrls": 0,
    "totalDuration": 1234
  }
}

{
  "urls": [
    { "url": "https://example.com/", "title": "Home" },
    { "url": "https://example.com/about", "title": "About" }
  ],
  "metadata": {
    "totalUrls": 2,
    "maxDepth": 1,
    "totalDuration": 2345,
    "seedUrl": "https://example.com"
  }
}

# Extract just the markdown
reader scrape https://example.com | jq -r '.data[0].markdown'

# Get all titles from batch
reader scrape url1 url2 url3 | jq -r '.data[].metadata.website.title'

reader crawl https://docs.example.com -d 3 --scrape -o docs.json

cat urls.txt | xargs reader scrape -c 5 -o results.json