web.scrape

POST

ops

web.scrape

Fetch one web page and return its readable text (markdown or plain text) with title and basic metadata.

curl --request POST \
  --url https://api.agentkey.app/v2/ops/web.scrape \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "url": "<string>",
  "fields": [
    "<string>"
  ],
  "format": "markdown",
  "include_raw": false,
  "provider": "<string>",
  "routing": {
    "allow_fallbacks": true,
    "only": [
      "<string>"
    ],
    "order": [
      "<string>"
    ],
    "strategy": "auto"
  }
}
'

import requests

url = "https://api.agentkey.app/v2/ops/web.scrape"

payload = {
    "url": "<string>",
    "fields": ["<string>"],
    "format": "markdown",
    "include_raw": False,
    "provider": "<string>",
    "routing": {
        "allow_fallbacks": True,
        "only": ["<string>"],
        "order": ["<string>"],
        "strategy": "auto"
    }
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: '<string>',
    fields: ['<string>'],
    format: 'markdown',
    include_raw: false,
    provider: '<string>',
    routing: {
      allow_fallbacks: true,
      only: ['<string>'],
      order: ['<string>'],
      strategy: 'auto'
    }
  })
};

fetch('https://api.agentkey.app/v2/ops/web.scrape', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.agentkey.app/v2/ops/web.scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '<string>',
    'fields' => [
        '<string>'
    ],
    'format' => 'markdown',
    'include_raw' => false,
    'provider' => '<string>',
    'routing' => [
        'allow_fallbacks' => true,
        'only' => [
                '<string>'
        ],
        'order' => [
                '<string>'
        ],
        'strategy' => 'auto'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.agentkey.app/v2/ops/web.scrape"

	payload := strings.NewReader("{\n  \"url\": \"<string>\",\n  \"fields\": [\n    \"<string>\"\n  ],\n  \"format\": \"markdown\",\n  \"include_raw\": false,\n  \"provider\": \"<string>\",\n  \"routing\": {\n    \"allow_fallbacks\": true,\n    \"only\": [\n      \"<string>\"\n    ],\n    \"order\": [\n      \"<string>\"\n    ],\n    \"strategy\": \"auto\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.agentkey.app/v2/ops/web.scrape")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"<string>\",\n  \"fields\": [\n    \"<string>\"\n  ],\n  \"format\": \"markdown\",\n  \"include_raw\": false,\n  \"provider\": \"<string>\",\n  \"routing\": {\n    \"allow_fallbacks\": true,\n    \"only\": [\n      \"<string>\"\n    ],\n    \"order\": [\n      \"<string>\"\n    ],\n    \"strategy\": \"auto\"\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.agentkey.app/v2/ops/web.scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"<string>\",\n  \"fields\": [\n    \"<string>\"\n  ],\n  \"format\": \"markdown\",\n  \"include_raw\": false,\n  \"provider\": \"<string>\",\n  \"routing\": {\n    \"allow_fallbacks\": true,\n    \"only\": [\n      \"<string>\"\n    ],\n    \"order\": [\n      \"<string>\"\n    ],\n    \"strategy\": \"auto\"\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "credits_charged": 123,
  "data": {
    "content_type": "<string>",
    "description": "<string>",
    "final_url": "<string>",
    "format": "markdown",
    "lang": "<string>",
    "published_at": "2023-11-07T05:31:56Z",
    "status_code": 123,
    "text": "<string>",
    "title": "<string>",
    "url": "<string>"
  },
  "op": "<string>",
  "op_version": 123,
  "provider": "<string>",
  "request_id": "<string>",
  "took_ms": 123,
  "unsupported_fields": [
    "<string>"
  ],
  "raw": "<unknown>",
  "raw_truncated": true
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

Authorizations

Authorization

string

header

required

AgentKey API key (ak_…).

Body

application/json

url

string

required

Absolute http(s) URL to fetch.

fields

string[]

Projection: keep only these dotted paths of data, e.g. ["items.id", "items.text"].

format

enum<string>

default:markdown

Format of the returned text. markdown keeps headings, lists and links; text is plain text.

Available options:

markdown,

text

include_raw

boolean

default:false

Attach the raw upstream payload as raw (not a stable contract; truncated above 256 KB).

provider

string

Shorthand for routing.only = [provider].

routing

object

Provider routing override. order: try these providers first; only: restrict to these (equivalent to pinning); allow_fallbacks: false tries only the first eligible provider.

Show child attributes

Response

Success envelope

Success envelope whose data is a WebPage.

attempts

object[]

required

Show child attributes

credits_charged

number

required

Credits charged for this call (the method's fixed price).

data

object

required

One fetched web page reduced to readable text, normalized across scrapers.

Show child attributes

string

required

Method name.

op_version

integer

required

Method version that served the call.

provider

string

required

Provider that actually served the call.

request_id

string

required

Correlation id; quote it in support requests.

took_ms

integer

required

unsupported_fields

string[]

required

Output paths this provider structurally does not supply; such fields are null for a different reason than 'missing on this record'.

raw

any

Upstream payload, only with include_raw=true. Not a stable contract.

raw_truncated

boolean

Fetch one web page and return its readable text (markdown or plain text) with title and basic metadata.

curl --request POST \
  --url https://api.agentkey.app/v2/ops/web.scrape \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "url": "<string>",
  "fields": [
    "<string>"
  ],
  "format": "markdown",
  "include_raw": false,
  "provider": "<string>",
  "routing": {
    "allow_fallbacks": true,
    "only": [
      "<string>"
    ],
    "order": [
      "<string>"
    ],
    "strategy": "auto"
  }
}
'

import requests

url = "https://api.agentkey.app/v2/ops/web.scrape"

payload = {
    "url": "<string>",
    "fields": ["<string>"],
    "format": "markdown",
    "include_raw": False,
    "provider": "<string>",
    "routing": {
        "allow_fallbacks": True,
        "only": ["<string>"],
        "order": ["<string>"],
        "strategy": "auto"
    }
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: '<string>',
    fields: ['<string>'],
    format: 'markdown',
    include_raw: false,
    provider: '<string>',
    routing: {
      allow_fallbacks: true,
      only: ['<string>'],
      order: ['<string>'],
      strategy: 'auto'
    }
  })
};

fetch('https://api.agentkey.app/v2/ops/web.scrape', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.agentkey.app/v2/ops/web.scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '<string>',
    'fields' => [
        '<string>'
    ],
    'format' => 'markdown',
    'include_raw' => false,
    'provider' => '<string>',
    'routing' => [
        'allow_fallbacks' => true,
        'only' => [
                '<string>'
        ],
        'order' => [
                '<string>'
        ],
        'strategy' => 'auto'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.agentkey.app/v2/ops/web.scrape"

	payload := strings.NewReader("{\n  \"url\": \"<string>\",\n  \"fields\": [\n    \"<string>\"\n  ],\n  \"format\": \"markdown\",\n  \"include_raw\": false,\n  \"provider\": \"<string>\",\n  \"routing\": {\n    \"allow_fallbacks\": true,\n    \"only\": [\n      \"<string>\"\n    ],\n    \"order\": [\n      \"<string>\"\n    ],\n    \"strategy\": \"auto\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.agentkey.app/v2/ops/web.scrape")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"<string>\",\n  \"fields\": [\n    \"<string>\"\n  ],\n  \"format\": \"markdown\",\n  \"include_raw\": false,\n  \"provider\": \"<string>\",\n  \"routing\": {\n    \"allow_fallbacks\": true,\n    \"only\": [\n      \"<string>\"\n    ],\n    \"order\": [\n      \"<string>\"\n    ],\n    \"strategy\": \"auto\"\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.agentkey.app/v2/ops/web.scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"<string>\",\n  \"fields\": [\n    \"<string>\"\n  ],\n  \"format\": \"markdown\",\n  \"include_raw\": false,\n  \"provider\": \"<string>\",\n  \"routing\": {\n    \"allow_fallbacks\": true,\n    \"only\": [\n      \"<string>\"\n    ],\n    \"order\": [\n      \"<string>\"\n    ],\n    \"strategy\": \"auto\"\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "credits_charged": 123,
  "data": {
    "content_type": "<string>",
    "description": "<string>",
    "final_url": "<string>",
    "format": "markdown",
    "lang": "<string>",
    "published_at": "2023-11-07T05:31:56Z",
    "status_code": 123,
    "text": "<string>",
    "title": "<string>",
    "url": "<string>"
  },
  "op": "<string>",
  "op_version": 123,
  "provider": "<string>",
  "request_id": "<string>",
  "took_ms": 123,
  "unsupported_fields": [
    "<string>"
  ],
  "raw": "<unknown>",
  "raw_truncated": true
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

{
  "code": 123,
  "error": "<string>",
  "error_type": "<string>",
  "attempts": [
    {
      "provider": "<string>",
      "status": 123,
      "took_ms": 123,
      "tool": "<string>",
      "error_type": "<string>",
      "refunded": true,
      "upstream_status": 123
    }
  ],
  "op": "<string>",
  "request_id": "<string>"
}

Overview

Scenario Methods

Catalog & Describe

Methods

Tool Catalog (MCP)

Authorizations

Body

Response