> ## Documentation Index
> Fetch the complete documentation index at: https://docs.context.dev/llms.txt
> Use this file to discover all available pages before exploring further.
# Submit a Batch
> Scrape 25K URLs or crawl large websites asynchronously.
1 Credit per successful URL
## OpenAPI
````yaml POST /batch/submit
openapi: 3.1.0
info:
title: Context API
description: API for retrieving context data from any website
version: 1.0.0
servers:
- url: https://api.context.dev/v1
security: []
tags:
- name: Batch
description: Scrape many pages or crawl a site asynchronously.
- name: Monitors
description: >-
Monitor pages, sitemaps, and extracted website data for exact or semantic
changes. Webhook payloads are documented by the
MonitorsChangeDetectedWebhookPayload and
MonitorsRunCompletedWebhookPayload schemas.
- name: WebDBs
description: Create structured tables from web pages and keep them up to date.
- name: News
description: >-
Search live first-party RSS and free historical news data by company
identity.
paths:
/batch/submit:
post:
tags:
- Batch
summary: Submit a batch
description: 'Scrape 25K URLs or crawl large websites asynchronously. '
operationId: submitBatch
parameters:
- schema:
type: string
maxLength: 200
description: >-
Any string unique to this submission. Retries with the same key
return the original batch.
required: false
description: >-
Any string unique to this submission. Retries with the same key
return the original batch.
name: Idempotency-Key
in: header
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/BatchSubmitRequest'
examples:
scrapeMarkdown:
summary: Scrape URLs as Markdown
value:
input:
type: scrape
data:
type: markdown
urls:
- url: https://example.com/products/anvil
itemId: sku-1
meta:
category: tools
- url: https://example.com/products/hammer
itemId: sku-2
options:
useMainContentOnly: true
scrapeHtml:
summary: Scrape URLs as HTML
value:
input:
type: scrape
data:
type: html
urls:
- url: https://example.com/about
- url: https://example.com/pricing
crawlStartUrlMarkdown:
summary: Crawl from a URL as Markdown
value:
input:
type: crawl
data:
type: markdown
source:
type: start_url
url: https://example.com/docs
controls:
maxUrls: 500
maxDepth: 3
followSubdomains: false
regex: ^https://example\.com/docs/
options:
includeLinks: true
crawlStartUrlHtml:
summary: Crawl from a URL as HTML
value:
input:
type: crawl
data:
type: html
source:
type: start_url
url: https://example.com/blog
controls:
maxUrls: 100
crawlSitemapMarkdown:
summary: Crawl a sitemap as Markdown
value:
input:
type: crawl
data:
type: markdown
source:
type: sitemap
domain: example.com
controls:
maxUrls: 500
regex: ^https://example\.com/docs/
crawlSitemapHtml:
summary: Crawl a sitemap as HTML
value:
input:
type: crawl
data:
type: html
source:
type: sitemap
domain: example.com
controls:
maxUrls: 500
maxDepth: 0
responses:
'202':
description: >-
Batch accepted. Read progress and results from `GET
/batch/{batch_id}`.
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
allOf:
- $ref: '#/components/schemas/BatchAccepted'
- type: object
properties:
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
description: API key usage for this request.
'400':
$ref: '#/components/responses/BadRequest'
'401':
$ref: '#/components/responses/Unauthorized'
'403':
description: Concurrent batch limit reached (error_code BATCH_LIMIT_EXCEEDED).
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'409':
description: >-
Idempotency-Key reused with a different body (error_code
IDEMPOTENCY_KEY_CONFLICT).
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
description: Batch input could not be staged or queued.
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
security:
- bearerAuth: []
x-codeSamples:
- lang: JavaScript
source: |-
import ContextDev from 'context.dev';
const client = new ContextDev({
apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted
});
const response = await client.batch.submit({
input: {
data: {
urls: [
{
url: 'https://example.com/products/anvil',
itemId: 'sku-1',
meta: { category: 'tools' },
},
{ url: 'https://example.com/products/hammer', itemId: 'sku-2' },
],
options: { useMainContentOnly: true },
},
},
});
console.log(response.id);
- lang: Python
source: |-
import os
from context.dev import ContextDev
client = ContextDev(
api_key=os.environ.get("CONTEXT_DEV_API_KEY"), # This is the default and can be omitted
)
response = client.batch.submit(
input={
"data": {
"urls": [{
"url": "https://example.com/products/anvil",
"item_id": "sku-1",
"meta": {
"category": "tools"
},
}, {
"url": "https://example.com/products/hammer",
"item_id": "sku-2",
}],
"options": {
"use_main_content_only": True
},
}
},
)
print(response.id)
- lang: Go
source: "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/context-dot-dev/context-go-sdk\"\n\t\"github.com/context-dot-dev/context-go-sdk/option\"\n)\n\nfunc main() {\n\tclient := contextdev.NewClient(\n\t\toption.WithAPIKey(\"My API Key\"),\n\t)\n\tresponse, err := client.Batch.Submit(context.TODO(), contextdev.BatchSubmitParams{\n\t\tInput: contextdev.BatchSubmitParamsInputUnion{\n\t\t\tOfScrape: &contextdev.BatchSubmitParamsInputScrape{\n\t\t\t\tData: contextdev.BatchSubmitParamsInputScrapeDataUnion{\n\t\t\t\t\tOfMarkdown: &contextdev.BatchSubmitParamsInputScrapeDataMarkdown{\n\t\t\t\t\t\tURLs: []contextdev.BatchSubmitParamsInputScrapeDataMarkdownURL{{\n\t\t\t\t\t\t\tURL: \"https://example.com/products/anvil\",\n\t\t\t\t\t\t\tItemID: contextdev.String(\"sku-1\"),\n\t\t\t\t\t\t\tMeta: map[string]any{\n\t\t\t\t\t\t\t\t\"category\": \"tools\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t}, {\n\t\t\t\t\t\t\tURL: \"https://example.com/products/hammer\",\n\t\t\t\t\t\t\tItemID: contextdev.String(\"sku-2\"),\n\t\t\t\t\t\t}},\n\t\t\t\t\t\tOptions: contextdev.BatchSubmitParamsInputScrapeDataMarkdownOptions{\n\t\t\t\t\t\t\tUseMainContentOnly: contextdev.Bool(true),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tpanic(err.Error())\n\t}\n\tfmt.Printf(\"%+v\\n\", response.ID)\n}\n"
- lang: Ruby
source: |-
require "context_dev"
context_dev = ContextDev::Client.new(api_key: "My API Key")
response = context_dev.batch.submit(
input: {
data: {
format: :markdown,
urls: [{url: "https://example.com/products/anvil"}, {url: "https://example.com/products/hammer"}]
},
mode: :scrape
}
)
puts(response)
- lang: PHP
source: >-
batch->submit(
input: [
'data' => [
'format' => 'markdown',
'urls' => [
[
'url' => 'https://example.com/products/anvil',
'itemID' => 'sku-1',
'meta' => ['category' => 'bar'],
],
[
'url' => 'https://example.com/products/hammer',
'itemID' => 'sku-2',
'meta' => ['foo' => 'bar'],
],
],
'options' => [
'country' => 'de',
'excludeSelectors' => ['x'],
'includeHTML' => true,
'includeImages' => true,
'includeLinks' => true,
'includeSelectors' => ['x'],
'maxAgeMs' => 0,
'pdf' => [
'end' => 1, 'ocr' => 'true', 'shouldParse' => 'true', 'start' => 1
],
'settleAnimations' => true,
'shortenBase64Images' => true,
'useMainContentOnly' => true,
'waitForMs' => 0,
],
],
'mode' => 'scrape',
],
tags: ['docs', 'competitor'],
webhookURL: 'webhookUrl',
idempotencyKey: 'Idempotency-Key',
);
var_dump($response);
} catch (APIException $e) {
echo $e->getMessage();
}
- lang: CLI
source: |-
context-dev batch submit \
--api-key 'My API Key' \
--input '{data: {format: markdown, urls: [{url: https://example.com/products/anvil}, {url: https://example.com/products/hammer}]}, mode: scrape}'
components:
schemas:
BatchSubmitRequest:
type: object
properties:
input:
oneOf:
- type: object
properties:
mode:
type: string
enum:
- scrape
description: Scrape the pages in `data.urls`.
data:
oneOf:
- type: object
properties:
format:
type: string
enum:
- markdown
description: Return page content as Markdown.
urls:
type: array
items:
type: object
properties:
url:
type: string
minLength: 1
description: Page URL to scrape.
example: https://example.com/products/anvil
itemId:
type: string
minLength: 1
maxLength: 200
description: >-
Your ID for this page, returned with its
result. The same URL can use different IDs.
example: sku-1
meta:
type: object
additionalProperties: {}
description: >-
Custom JSON returned unchanged with this page
result.
required:
- url
additionalProperties: false
description: >-
A page to scrape, with optional data for matching
results.
minItems: 1
maxItems: 25000
description: Pages to scrape. Maximum 25000.
options:
type: object
properties:
includeLinks:
type: boolean
default: true
description: Include links in the Markdown.
includeImages:
type: boolean
default: false
description: Include image references in the Markdown.
shortenBase64Images:
type: boolean
default: true
description: Shorten inline base64 image data.
useMainContentOnly:
type: boolean
default: false
description: >-
Return the main content without navigation or
footers.
includeHTML:
type: boolean
default: false
description: >-
Also include each page's HTML in its result
record, as an `html` field alongside the
Markdown.
pdf:
type: object
properties:
shouldParse:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: true
description: >-
When true, PDF URLs are fetched and parsed.
When false, PDF URLs are skipped and a 400
PDF_SKIPPED is returned.
start:
type: integer
minimum: 1
description: >-
First 1-based PDF page to parse. When
omitted, parsing starts at the first page.
end:
type: integer
minimum: 1
description: >-
Last 1-based PDF page to parse. When
omitted, parsing ends at the last page. Must
be greater than or equal to start when both
are provided.
ocr:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: false
description: >-
When true, OCR the selected PDF pages that
have no usable text layer (scans), replacing
each recovered page's text with the OCR
result while pages with a real text layer
keep it. Billed at 1 credit per page OCR
actually recovered, on top of the base
request cost. When false, no OCR runs.
default:
shouldParse: true
ocr: false
description: >-
PDF parsing controls. Use start/end to limit
text extraction and embedded-image detection/OCR
to an inclusive 1-based page range.
includeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Keep only the subtrees matching these CSS
selectors. Filtered pages are always fetched
fresh, ignoring `maxAgeMs`.
excludeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Remove elements matching these CSS selectors.
Applied after `includeSelectors`, so an element
matching both is removed.
waitForMs:
type: integer
minimum: 0
maximum: 15000
description: >-
How long to wait after initial page load, in
milliseconds. `0` waits 500 ms.
settleAnimations:
type: boolean
default: false
description: >-
Wait briefly for CSS and transition animations
to settle before extraction, on pages that
render in a browser.
country:
$ref: '#/components/schemas/BrowserCountryCode'
maxAgeMs:
type:
- integer
- 'null'
minimum: 0
maximum: 2592000000
default: 86400000
description: >-
Return a cached result if a prior scrape for the
same parameters exists and is younger than this
many milliseconds. Defaults to 1 day (86400000
ms) when omitted. Max is 30 days (2592000000
ms). Set to 0 to always scrape fresh.
default:
includeLinks: true
includeImages: false
shortenBase64Images: true
useMainContentOnly: false
includeHTML: false
pdf:
shouldParse: true
ocr: false
settleAnimations: false
maxAgeMs: 86400000
additionalProperties: false
description: Options for Markdown output.
required:
- format
- urls
additionalProperties: false
description: Scrape the listed pages as Markdown.
title: Markdown
- type: object
properties:
format:
type: string
enum:
- html
description: Return page content as HTML.
urls:
type: array
items:
type: object
properties:
url:
type: string
minLength: 1
description: Page URL to scrape.
example: https://example.com/products/anvil
itemId:
type: string
minLength: 1
maxLength: 200
description: >-
Your ID for this page, returned with its
result. The same URL can use different IDs.
example: sku-1
meta:
type: object
additionalProperties: {}
description: >-
Custom JSON returned unchanged with this page
result.
required:
- url
additionalProperties: false
description: >-
A page to scrape, with optional data for matching
results.
minItems: 1
maxItems: 25000
description: Pages to scrape. Maximum 25000.
options:
type: object
properties:
useMainContentOnly:
type: boolean
default: false
description: >-
Return the main content without navigation or
footers.
pdf:
type: object
properties:
shouldParse:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: true
description: >-
When true, PDF URLs are fetched and parsed.
When false, PDF URLs are skipped and a 400
PDF_SKIPPED is returned.
start:
type: integer
minimum: 1
description: >-
First 1-based PDF page to parse. When
omitted, parsing starts at the first page.
end:
type: integer
minimum: 1
description: >-
Last 1-based PDF page to parse. When
omitted, parsing ends at the last page. Must
be greater than or equal to start when both
are provided.
ocr:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: false
description: >-
When true, OCR the selected PDF pages that
have no usable text layer (scans), replacing
each recovered page's text with the OCR
result while pages with a real text layer
keep it. Billed at 1 credit per page OCR
actually recovered, on top of the base
request cost. When false, no OCR runs.
default:
shouldParse: true
ocr: false
description: >-
PDF parsing controls. Use start/end to limit
text extraction and embedded-image detection/OCR
to an inclusive 1-based page range.
includeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Keep only the subtrees matching these CSS
selectors. Filtered pages are always fetched
fresh, ignoring `maxAgeMs`.
excludeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Remove elements matching these CSS selectors.
Applied after `includeSelectors`, so an element
matching both is removed.
waitForMs:
type: integer
minimum: 0
maximum: 15000
description: >-
How long to wait after initial page load, in
milliseconds. `0` waits 500 ms.
settleAnimations:
type: boolean
default: false
description: >-
Wait briefly for CSS and transition animations
to settle before extraction, on pages that
render in a browser.
country:
$ref: '#/components/schemas/BrowserCountryCode'
maxAgeMs:
type:
- integer
- 'null'
minimum: 0
maximum: 2592000000
default: 86400000
description: >-
Return a cached result if a prior scrape for the
same parameters exists and is younger than this
many milliseconds. Defaults to 1 day (86400000
ms) when omitted. Max is 30 days (2592000000
ms). Set to 0 to always scrape fresh.
default:
useMainContentOnly: false
pdf:
shouldParse: true
ocr: false
settleAnimations: false
maxAgeMs: 86400000
additionalProperties: false
description: Options for HTML output.
required:
- format
- urls
additionalProperties: false
description: Scrape the listed pages as HTML.
title: HTML
discriminator:
propertyName: format
title: Scrape data
description: Pages to scrape and their output format.
required:
- mode
- data
additionalProperties: false
title: Scrape
description: Scrape up to 25K URLs in one batch.
- type: object
properties:
mode:
type: string
enum:
- crawl
description: Discover and scrape pages from `data.source`.
data:
oneOf:
- type: object
properties:
format:
type: string
enum:
- markdown
description: Return page content as Markdown.
source:
oneOf:
- type: object
properties:
type:
type: string
enum:
- start_url
description: Start from one page.
url:
type: string
minLength: 1
description: >-
Page where crawling begins. A URL without a
scheme is read as https://.
example: https://example.com/docs
controls:
type: object
properties:
maxUrls:
type: integer
minimum: 1
maximum: 25000
default: 100
description: >-
Maximum pages to fetch. Unused reserved
credits are refunded. Maximum 25000.
maxDepth:
type: integer
minimum: 0
maximum: 50
description: >-
Maximum link depth. Source pages are
depth 0. No limit when omitted.
followSubdomains:
type: boolean
default: false
description: Follow links to subdomains.
regex:
type: string
maxLength: 256
description: >-
RE2 pattern for URLs to include. The
`start_url` itself is always included.
example: ^https://example\.com/docs/
default:
maxUrls: 100
followSubdomains: false
additionalProperties: false
description: Limits and filters for page discovery.
required:
- type
- url
additionalProperties: false
description: Discover pages by following links from one URL.
title: Start URL
- type: object
properties:
type:
type: string
enum:
- sitemap
description: Scrape the URLs in the domain's sitemap.
domain:
type: string
minLength: 3
description: >-
Domain whose sitemap lists the pages to
scrape. A full URL is reduced to its domain.
example: example.com
controls:
type: object
properties:
maxUrls:
type: integer
minimum: 1
maximum: 25000
default: 100
description: >-
Maximum pages to fetch. Unused reserved
credits are refunded. Maximum 25000.
regex:
type: string
maxLength: 256
description: >-
RE2 pattern; only sitemap URLs matching
it are scraped.
example: ^https://example\.com/docs/
default:
maxUrls: 100
additionalProperties: false
description: >-
Limits and filters for the sitemap URLs. A
sitemap batch scrapes exactly those URLs and
never follows links off them, so there is no
crawl depth here.
required:
- type
- domain
additionalProperties: false
description: >-
Scrape the pages listed in a domain's sitemap.
Links on those pages are not followed.
title: Sitemap
discriminator:
propertyName: type
title: Crawl source
description: How to find pages to crawl.
options:
type: object
properties:
includeLinks:
type: boolean
default: true
description: Include links in the Markdown.
includeImages:
type: boolean
default: false
description: Include image references in the Markdown.
shortenBase64Images:
type: boolean
default: true
description: Shorten inline base64 image data.
useMainContentOnly:
type: boolean
default: false
description: >-
Return the main content without navigation or
footers.
includeHTML:
type: boolean
default: false
description: >-
Also include each page's HTML in its result
record, as an `html` field alongside the
Markdown.
pdf:
type: object
properties:
shouldParse:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: true
description: >-
When true, PDF URLs are fetched and parsed.
When false, PDF URLs are skipped and a 400
PDF_SKIPPED is returned.
start:
type: integer
minimum: 1
description: >-
First 1-based PDF page to parse. When
omitted, parsing starts at the first page.
end:
type: integer
minimum: 1
description: >-
Last 1-based PDF page to parse. When
omitted, parsing ends at the last page. Must
be greater than or equal to start when both
are provided.
ocr:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: false
description: >-
When true, OCR the selected PDF pages that
have no usable text layer (scans), replacing
each recovered page's text with the OCR
result while pages with a real text layer
keep it. Billed at 1 credit per page OCR
actually recovered, on top of the base
request cost. When false, no OCR runs.
default:
shouldParse: true
ocr: false
description: >-
PDF parsing controls. Use start/end to limit
text extraction and embedded-image detection/OCR
to an inclusive 1-based page range.
includeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Keep only the subtrees matching these CSS
selectors. Filtered pages are always fetched
fresh, ignoring `maxAgeMs`.
excludeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Remove elements matching these CSS selectors.
Applied after `includeSelectors`, so an element
matching both is removed.
waitForMs:
type: integer
minimum: 0
maximum: 15000
description: >-
How long to wait after initial page load, in
milliseconds. `0` waits 500 ms.
settleAnimations:
type: boolean
default: false
description: >-
Wait briefly for CSS and transition animations
to settle before extraction, on pages that
render in a browser.
country:
$ref: '#/components/schemas/BrowserCountryCode'
maxAgeMs:
type:
- integer
- 'null'
minimum: 0
maximum: 2592000000
default: 86400000
description: >-
Return a cached result if a prior scrape for the
same parameters exists and is younger than this
many milliseconds. Defaults to 1 day (86400000
ms) when omitted. Max is 30 days (2592000000
ms). Set to 0 to always scrape fresh.
default:
includeLinks: true
includeImages: false
shortenBase64Images: true
useMainContentOnly: false
includeHTML: false
pdf:
shouldParse: true
ocr: false
settleAnimations: false
maxAgeMs: 86400000
additionalProperties: false
description: Options for Markdown output.
required:
- format
- source
additionalProperties: false
title: Markdown
description: Crawl pages and return Markdown.
- type: object
properties:
format:
type: string
enum:
- html
description: Return page content as HTML.
source:
oneOf:
- type: object
properties:
type:
type: string
enum:
- start_url
description: Start from one page.
url:
type: string
minLength: 1
description: >-
Page where crawling begins. A URL without a
scheme is read as https://.
example: https://example.com/docs
controls:
type: object
properties:
maxUrls:
type: integer
minimum: 1
maximum: 25000
default: 100
description: >-
Maximum pages to fetch. Unused reserved
credits are refunded. Maximum 25000.
maxDepth:
type: integer
minimum: 0
maximum: 50
description: >-
Maximum link depth. Source pages are
depth 0. No limit when omitted.
followSubdomains:
type: boolean
default: false
description: Follow links to subdomains.
regex:
type: string
maxLength: 256
description: >-
RE2 pattern for URLs to include. The
`start_url` itself is always included.
example: ^https://example\.com/docs/
default:
maxUrls: 100
followSubdomains: false
additionalProperties: false
description: Limits and filters for page discovery.
required:
- type
- url
additionalProperties: false
description: Discover pages by following links from one URL.
title: Start URL
- type: object
properties:
type:
type: string
enum:
- sitemap
description: Scrape the URLs in the domain's sitemap.
domain:
type: string
minLength: 3
description: >-
Domain whose sitemap lists the pages to
scrape. A full URL is reduced to its domain.
example: example.com
controls:
type: object
properties:
maxUrls:
type: integer
minimum: 1
maximum: 25000
default: 100
description: >-
Maximum pages to fetch. Unused reserved
credits are refunded. Maximum 25000.
regex:
type: string
maxLength: 256
description: >-
RE2 pattern; only sitemap URLs matching
it are scraped.
example: ^https://example\.com/docs/
default:
maxUrls: 100
additionalProperties: false
description: >-
Limits and filters for the sitemap URLs. A
sitemap batch scrapes exactly those URLs and
never follows links off them, so there is no
crawl depth here.
required:
- type
- domain
additionalProperties: false
description: >-
Scrape the pages listed in a domain's sitemap.
Links on those pages are not followed.
title: Sitemap
discriminator:
propertyName: type
title: Crawl source
description: How to find pages to crawl.
options:
type: object
properties:
useMainContentOnly:
type: boolean
default: false
description: >-
Return the main content without navigation or
footers.
pdf:
type: object
properties:
shouldParse:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: true
description: >-
When true, PDF URLs are fetched and parsed.
When false, PDF URLs are skipped and a 400
PDF_SKIPPED is returned.
start:
type: integer
minimum: 1
description: >-
First 1-based PDF page to parse. When
omitted, parsing starts at the first page.
end:
type: integer
minimum: 1
description: >-
Last 1-based PDF page to parse. When
omitted, parsing ends at the last page. Must
be greater than or equal to start when both
are provided.
ocr:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: false
description: >-
When true, OCR the selected PDF pages that
have no usable text layer (scans), replacing
each recovered page's text with the OCR
result while pages with a real text layer
keep it. Billed at 1 credit per page OCR
actually recovered, on top of the base
request cost. When false, no OCR runs.
default:
shouldParse: true
ocr: false
description: >-
PDF parsing controls. Use start/end to limit
text extraction and embedded-image detection/OCR
to an inclusive 1-based page range.
includeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Keep only the subtrees matching these CSS
selectors. Filtered pages are always fetched
fresh, ignoring `maxAgeMs`.
excludeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Remove elements matching these CSS selectors.
Applied after `includeSelectors`, so an element
matching both is removed.
waitForMs:
type: integer
minimum: 0
maximum: 15000
description: >-
How long to wait after initial page load, in
milliseconds. `0` waits 500 ms.
settleAnimations:
type: boolean
default: false
description: >-
Wait briefly for CSS and transition animations
to settle before extraction, on pages that
render in a browser.
country:
$ref: '#/components/schemas/BrowserCountryCode'
maxAgeMs:
type:
- integer
- 'null'
minimum: 0
maximum: 2592000000
default: 86400000
description: >-
Return a cached result if a prior scrape for the
same parameters exists and is younger than this
many milliseconds. Defaults to 1 day (86400000
ms) when omitted. Max is 30 days (2592000000
ms). Set to 0 to always scrape fresh.
default:
useMainContentOnly: false
pdf:
shouldParse: true
ocr: false
settleAnimations: false
maxAgeMs: 86400000
additionalProperties: false
description: Options for HTML output.
required:
- format
- source
additionalProperties: false
title: HTML
description: Crawl pages and return HTML.
discriminator:
propertyName: format
title: Crawl data
description: Crawl source and output format.
required:
- mode
- data
additionalProperties: false
title: Crawl
description: Crawl pages starting from a URL or from a domain's sitemap.
discriminator:
propertyName: mode
title: Batch input
description: Choose a URL list or a site crawl.
webhookUrl:
type: string
description: URL notified when the batch finishes.
tags:
type: array
items:
type: string
minLength: 1
maxLength: 50
maxItems: 20
description: Tags stored on the batch. Filter the batch list by them later.
example:
- docs
- competitor
required:
- input
title: Submit batch request
description: Create an asynchronous scraping job.
BatchAccepted:
type: object
properties:
id:
type: string
example: batch_9f2c8a
description: Batch ID. Poll GET /batch/{batch_id} with it.
status:
type: string
enum:
- queued
description: Always `queued`. An accepted batch has not started yet.
mode:
type: string
enum:
- scrape
- crawl
description: How pages will be selected.
format:
type: string
enum:
- markdown
- html
description: What each page will be returned as.
tags:
type: array
items:
type: string
description: Tags stored on the batch.
example:
- docs
crawl:
$ref: '#/components/schemas/BatchNullableCrawlControls'
input:
$ref: '#/components/schemas/BatchIntake'
credits:
type: object
properties:
reserved:
type: integer
example: 24817
description: >-
Credits just debited from your balance. Whatever the batch does
not spend is refunded when it settles.
required:
- reserved
description: What accepting this batch cost.
created_at:
type: string
description: When the batch was created.
invalid_urls:
type: array
items:
type: object
properties:
url:
type: string
description: Rejected URL.
reason:
type: string
description: Why it was rejected.
example: Must be a public http:// or https:// URL
required:
- url
- reason
description: Rejected URLs, up to 100. These are not charged.
webhook_secret:
type: string
description: >-
Signing secret for the completion webhook, returned only here and
never again. Store it now; it is not repeated by GET
/batch/{batch_id}.
required:
- id
- status
- mode
- format
- tags
- crawl
- input
- credits
- created_at
- invalid_urls
title: Accepted batch
KeyMetadata:
type: object
properties:
credits_consumed:
type: integer
description: The number of credits consumed by this request.
credits_remaining:
type: integer
description: >-
The number of credits remaining for your organization after this
request.
required:
- credits_consumed
- credits_remaining
description: >-
Metadata about the API key used for the request. Included in every
response whenever a valid API key is provided, even when the response
status is not 200.
ErrorResponse:
type: object
properties:
message:
type: string
description: Error message
error_code:
type: string
enum:
- INTERNAL_ERROR
- VALID
- NOT_FOUND
- FORBIDDEN
- USAGE_EXCEEDED
- RATE_LIMITED
- UNAUTHORIZED
- DISABLED
- PAID_PLAN_REQUIRED
- INSUFFICIENT_PERMISSIONS
- TIMEOUT_EXCEEDS_MAXIMUM
- WEBSITE_ACCESS_ERROR
- WEBSITE_BLOCKED
- WEBSITE_NOT_FOUND
- PDF_SKIPPED
- PDF_IMAGES_ONLY
- EXTERNAL_PROVIDER_ERROR
- INPUT_VALIDATION_ERROR
- ZDR_NOT_SUPPORTED
- ZDR_NOT_ENABLED
- FREE_EMAIL_DETECTED
- DISPOSABLE_EMAIL_DETECTED
- REQUEST_TIMEOUT
- COLD_DOMAIN_TIMEOUT_TOO_LOW
- UNSUPPORTED_CONTENT
- CONTENT_TOO_LARGE
- MONITOR_PAUSED
- MONITOR_NO_WEBHOOK
- COLLECTION_PAUSED
- MONITOR_LIMIT_EXCEEDED
- SEARCH_UNAVAILABLE
- BATCH_LIMIT_EXCEEDED
- BATCH_NOT_CANCELLABLE
- BATCH_NOT_COMPLETED
- IDEMPOTENCY_KEY_CONFLICT
description: Error code indicating the type of error
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
BrowserCountryCode:
type: string
enum:
- ad
- ae
- af
- ag
- ai
- al
- am
- ao
- ar
- at
- au
- aw
- az
- ba
- bb
- bd
- be
- bf
- bg
- bh
- bi
- bj
- bm
- bn
- bo
- bq
- br
- bs
- bw
- by
- bz
- ca
- cd
- cf
- cg
- ch
- ci
- cl
- cm
- cn
- co
- cr
- cv
- cw
- cy
- cz
- de
- dj
- dk
- dm
- do
- dz
- ec
- ee
- eg
- es
- et
- fi
- fj
- fr
- ga
- gb
- gd
- ge
- gf
- gg
- gh
- gm
- gn
- gp
- gq
- gr
- gt
- gu
- gw
- gy
- hk
- hn
- hr
- ht
- hu
- id
- ie
- il
- im
- in
- iq
- ir
- is
- it
- je
- jm
- jo
- jp
- ke
- kg
- kh
- kn
- kr
- kw
- ky
- kz
- la
- lb
- lc
- lk
- lr
- ls
- lt
- lu
- lv
- ly
- ma
- mc
- md
- me
- mf
- mg
- mk
- ml
- mm
- mn
- mo
- mq
- mr
- mt
- mu
- mv
- mw
- mx
- my
- mz
- na
- nc
- ne
- ng
- ni
- nl
- 'no'
- np
- nz
- om
- pa
- pe
- pf
- pg
- ph
- pk
- pl
- pr
- ps
- pt
- py
- qa
- re
- ro
- rs
- ru
- rw
- sa
- sc
- sd
- se
- sg
- si
- sk
- sl
- sm
- sn
- so
- sr
- ss
- st
- sv
- sx
- sy
- sz
- tc
- td
- tg
- th
- tj
- tl
- tm
- tn
- tr
- tt
- tw
- tz
- ua
- ug
- us
- uy
- uz
- vc
- ve
- vg
- vi
- vn
- ye
- yt
- za
- zm
- zw
example: de
description: >-
Fetch the target page through a residential proxy in this country (ISO
3166-1 alpha-2).
BatchNullableCrawlControls:
anyOf:
- allOf:
- $ref: '#/components/schemas/BatchCrawlControls'
- type: 'null'
description: How the crawl was configured. Null for scrape batches.
BatchIntake:
type: object
properties:
reserved:
type: integer
example: 24817
description: >-
Pages credits were reserved for. Everything else — progress, the
refund, the completion percentage — is measured against this.
reserved_is_ceiling:
type: boolean
example: false
description: >-
Whether `reserved` is an upper bound the batch may finish under.
True only for a crawl that follows links, whose reachable page count
is unknowable until it runs. False for a scrape and for a sitemap
crawl, where `reserved` is an exact page count.
submitted:
type:
- integer
- 'null'
example: 25000
description: >-
URLs in the list you sent, before validation and de-duplication.
Null for a crawl, which is given a source rather than a list.
duplicates:
type: integer
example: 183
description: >-
URLs dropped before reserving because another entry resolved to the
same page. Non-zero for sitemap crawls too, whose sitemaps routinely
list a page more than once.
invalid:
type:
- integer
- 'null'
example: 0
description: >-
URLs from your list rejected as unusable; the same ones are itemised
in `invalid_urls` at submission. Null for a crawl — a crawl that
resolves no usable page is rejected outright with a 400 rather than
accepted with an empty list.
required:
- reserved
- reserved_is_ceiling
- submitted
- duplicates
- invalid
x-stainless-model: batch.intake
description: What submission took in, and what it charged for.
BatchCrawlControls:
type: object
properties:
source:
anyOf:
- type: object
properties:
type:
type: string
enum:
- start_url
url:
type: string
example: https://example.com/docs
description: Page the crawl started from.
required:
- type
- url
title: Start URL
description: The crawl discovered pages by following links from one URL.
- type: object
properties:
type:
type: string
enum:
- sitemap
domain:
type: string
example: example.com
description: Domain whose sitemap supplied the pages.
required:
- type
- domain
title: Sitemap
description: The crawl scraped the pages listed in the domain's sitemap.
description: Where the crawl started.
max_pages:
type: integer
example: 500
description: >-
The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only
the URLs its sitemap actually lists, up to this many, so
`input.reserved` is often lower.
max_depth:
type:
- integer
- 'null'
example: 0
description: >-
Link depth limit. Always 0 for a sitemap crawl, which never follows
links off its URLs; null when a `start_url` crawl set no limit.
follow_subdomains:
type: boolean
description: >-
Whether links to subdomains were followed. Always false for a
sitemap crawl.
url_pattern:
type:
- string
- 'null'
example: ^https://example\.com/docs/
description: >-
RE2 pattern URLs had to match to be crawled. Null when the crawl set
none.
required:
- source
- max_pages
- max_depth
- follow_subdomains
- url_pattern
x-stainless-model: batch.crawl_controls
title: Crawl controls
description: >-
The crawl controls as submitted, so the limits requested can be compared
against what the crawl reached.
headers:
RateLimitLimit:
description: >-
Maximum requests allowed in the current fixed one-minute window.
Returned when the authenticated API key has a per-minute rate limit.
schema:
type: integer
minimum: 1
RateLimitRemaining:
description: >-
Requests remaining in the current fixed one-minute window. Returned when
the authenticated API key has a per-minute rate limit.
schema:
type: integer
minimum: 0
RateLimitReset:
description: >-
Unix timestamp in seconds when the current rate-limit window resets.
Returned when the authenticated API key has a per-minute rate limit.
schema:
type: integer
responses:
BadRequest:
description: Bad request
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
Unauthorized:
description: Unauthorized
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
securitySchemes:
bearerAuth:
type: http
scheme: bearer
description: >-
Bearer authentication header of the form `Bearer `, where
`` is your api key.
````