Start a crawl
curl --request POST \
--url https://api.scrinly.com/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"maxPages": 50,
"maxDepth": 3,
"maxConcurrentPages": 3,
"maxDurationMs": 1800000,
"scope": "origin",
"includePaths": [
"<string>"
],
"excludePaths": [
"<string>"
],
"followBinary": false,
"robots": "obey",
"markdown": true,
"html": false,
"structure": false,
"store": true,
"storage_path": "crawls",
"query": "normalise",
"webhookUrl": "<string>",
"webhookSecret": "<string>"
}
'import requests
url = "https://api.scrinly.com/crawl"
payload = {
"url": "<string>",
"maxPages": 50,
"maxDepth": 3,
"maxConcurrentPages": 3,
"maxDurationMs": 1800000,
"scope": "origin",
"includePaths": ["<string>"],
"excludePaths": ["<string>"],
"followBinary": False,
"robots": "obey",
"markdown": True,
"html": False,
"structure": False,
"store": True,
"storage_path": "crawls",
"query": "normalise",
"webhookUrl": "<string>",
"webhookSecret": "<string>"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
maxPages: 50,
maxDepth: 3,
maxConcurrentPages: 3,
maxDurationMs: 1800000,
scope: 'origin',
includePaths: ['<string>'],
excludePaths: ['<string>'],
followBinary: false,
robots: 'obey',
markdown: true,
html: false,
structure: false,
store: true,
storage_path: 'crawls',
query: 'normalise',
webhookUrl: '<string>',
webhookSecret: '<string>'
})
};
fetch('https://api.scrinly.com/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.scrinly.com/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'maxPages' => 50,
'maxDepth' => 3,
'maxConcurrentPages' => 3,
'maxDurationMs' => 1800000,
'scope' => 'origin',
'includePaths' => [
'<string>'
],
'excludePaths' => [
'<string>'
],
'followBinary' => false,
'robots' => 'obey',
'markdown' => true,
'html' => false,
'structure' => false,
'store' => true,
'storage_path' => 'crawls',
'query' => 'normalise',
'webhookUrl' => '<string>',
'webhookSecret' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.scrinly.com/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"maxPages\": 50,\n \"maxDepth\": 3,\n \"maxConcurrentPages\": 3,\n \"maxDurationMs\": 1800000,\n \"scope\": \"origin\",\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"followBinary\": false,\n \"robots\": \"obey\",\n \"markdown\": true,\n \"html\": false,\n \"structure\": false,\n \"store\": true,\n \"storage_path\": \"crawls\",\n \"query\": \"normalise\",\n \"webhookUrl\": \"<string>\",\n \"webhookSecret\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.scrinly.com/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"maxPages\": 50,\n \"maxDepth\": 3,\n \"maxConcurrentPages\": 3,\n \"maxDurationMs\": 1800000,\n \"scope\": \"origin\",\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"followBinary\": false,\n \"robots\": \"obey\",\n \"markdown\": true,\n \"html\": false,\n \"structure\": false,\n \"store\": true,\n \"storage_path\": \"crawls\",\n \"query\": \"normalise\",\n \"webhookUrl\": \"<string>\",\n \"webhookSecret\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.scrinly.com/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"maxPages\": 50,\n \"maxDepth\": 3,\n \"maxConcurrentPages\": 3,\n \"maxDurationMs\": 1800000,\n \"scope\": \"origin\",\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"followBinary\": false,\n \"robots\": \"obey\",\n \"markdown\": true,\n \"html\": false,\n \"structure\": false,\n \"store\": true,\n \"storage_path\": \"crawls\",\n \"query\": \"normalise\",\n \"webhookUrl\": \"<string>\",\n \"webhookSecret\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"success": "<unknown>",
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"statusUrl": "<string>",
"pagesUrl": "<string>",
"resultsUrl": "<string>"
}{
"success": true,
"error": "<string>",
"type": "<string>",
"message": "<string>",
"fields": [
{
"field": "<string>",
"message": "<string>"
}
]
}{
"success": true,
"error": "<string>",
"type": "<string>",
"message": "<string>",
"fields": [
{
"field": "<string>",
"message": "<string>"
}
]
}{
"success": true,
"error": "<string>",
"type": "<string>",
"message": "<string>",
"fields": [
{
"field": "<string>",
"message": "<string>"
}
]
}Crawls
Start a crawl
POST
/
crawl
Start a crawl
curl --request POST \
--url https://api.scrinly.com/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"maxPages": 50,
"maxDepth": 3,
"maxConcurrentPages": 3,
"maxDurationMs": 1800000,
"scope": "origin",
"includePaths": [
"<string>"
],
"excludePaths": [
"<string>"
],
"followBinary": false,
"robots": "obey",
"markdown": true,
"html": false,
"structure": false,
"store": true,
"storage_path": "crawls",
"query": "normalise",
"webhookUrl": "<string>",
"webhookSecret": "<string>"
}
'import requests
url = "https://api.scrinly.com/crawl"
payload = {
"url": "<string>",
"maxPages": 50,
"maxDepth": 3,
"maxConcurrentPages": 3,
"maxDurationMs": 1800000,
"scope": "origin",
"includePaths": ["<string>"],
"excludePaths": ["<string>"],
"followBinary": False,
"robots": "obey",
"markdown": True,
"html": False,
"structure": False,
"store": True,
"storage_path": "crawls",
"query": "normalise",
"webhookUrl": "<string>",
"webhookSecret": "<string>"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
maxPages: 50,
maxDepth: 3,
maxConcurrentPages: 3,
maxDurationMs: 1800000,
scope: 'origin',
includePaths: ['<string>'],
excludePaths: ['<string>'],
followBinary: false,
robots: 'obey',
markdown: true,
html: false,
structure: false,
store: true,
storage_path: 'crawls',
query: 'normalise',
webhookUrl: '<string>',
webhookSecret: '<string>'
})
};
fetch('https://api.scrinly.com/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.scrinly.com/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'maxPages' => 50,
'maxDepth' => 3,
'maxConcurrentPages' => 3,
'maxDurationMs' => 1800000,
'scope' => 'origin',
'includePaths' => [
'<string>'
],
'excludePaths' => [
'<string>'
],
'followBinary' => false,
'robots' => 'obey',
'markdown' => true,
'html' => false,
'structure' => false,
'store' => true,
'storage_path' => 'crawls',
'query' => 'normalise',
'webhookUrl' => '<string>',
'webhookSecret' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.scrinly.com/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"maxPages\": 50,\n \"maxDepth\": 3,\n \"maxConcurrentPages\": 3,\n \"maxDurationMs\": 1800000,\n \"scope\": \"origin\",\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"followBinary\": false,\n \"robots\": \"obey\",\n \"markdown\": true,\n \"html\": false,\n \"structure\": false,\n \"store\": true,\n \"storage_path\": \"crawls\",\n \"query\": \"normalise\",\n \"webhookUrl\": \"<string>\",\n \"webhookSecret\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.scrinly.com/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"maxPages\": 50,\n \"maxDepth\": 3,\n \"maxConcurrentPages\": 3,\n \"maxDurationMs\": 1800000,\n \"scope\": \"origin\",\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"followBinary\": false,\n \"robots\": \"obey\",\n \"markdown\": true,\n \"html\": false,\n \"structure\": false,\n \"store\": true,\n \"storage_path\": \"crawls\",\n \"query\": \"normalise\",\n \"webhookUrl\": \"<string>\",\n \"webhookSecret\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.scrinly.com/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"maxPages\": 50,\n \"maxDepth\": 3,\n \"maxConcurrentPages\": 3,\n \"maxDurationMs\": 1800000,\n \"scope\": \"origin\",\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"followBinary\": false,\n \"robots\": \"obey\",\n \"markdown\": true,\n \"html\": false,\n \"structure\": false,\n \"store\": true,\n \"storage_path\": \"crawls\",\n \"query\": \"normalise\",\n \"webhookUrl\": \"<string>\",\n \"webhookSecret\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"success": "<unknown>",
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"statusUrl": "<string>",
"pagesUrl": "<string>",
"resultsUrl": "<string>"
}{
"success": true,
"error": "<string>",
"type": "<string>",
"message": "<string>",
"fields": [
{
"field": "<string>",
"message": "<string>"
}
]
}{
"success": true,
"error": "<string>",
"type": "<string>",
"message": "<string>",
"fields": [
{
"field": "<string>",
"message": "<string>"
}
]
}{
"success": true,
"error": "<string>",
"type": "<string>",
"message": "<string>",
"fields": [
{
"field": "<string>",
"message": "<string>"
}
]
}Authorizations
Customer API key from the Scrinly dashboard.
Body
application/json
Maximum string length:
2048Pattern:
^https?://Required range:
1 <= x <= 5000Required range:
0 <= x <= 10Required range:
1 <= x <= 10Required range:
1000 <= x <= 21600000Available options:
origin, host, domain, any Available options:
obey, ignore Available options:
normalise, strip, keep Maximum string length:
2048Pattern:
^https?://⌘I
