Preview Website Crawl
Crawls a website source and stages the pages it finds for you to choose from, without importing anything. Returns the run. Poll Get Source Run with its id until state is preview_ready, zero_results, failed, or canceled.
Then list the pages with List Crawl Preview Pages and import the ones you want with Import Selected Pages.
The source must be a web source. The number of pages crawled follows the source’s crawl settings. A crawl that returns more than 1,000 pages fails, so narrow those settings first.
A preview counts as the source’s active run until you import from it or cancel it with Cancel Source Run, so cancel one you won’t import. Wait for any other run on the source to end before starting a preview.
For the preview-and-import flow, see Import Website Pages.
curl --request POST \
--url https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview \
--header 'Authorization: Bearer <token>' \
--header 'Idempotency-Key: <idempotency-key>'import requests
url = "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview"
headers = {
"Idempotency-Key": "<idempotency-key>",
"Authorization": "Bearer <token>"
}
response = requests.post(url, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'Idempotency-Key': '<idempotency-key>', Authorization: 'Bearer <token>'}
};
fetch('https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Idempotency-Key: <idempotency-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview"
req, _ := http.NewRequest("POST", url, nil)
req.Header.Add("Idempotency-Key", "<idempotency-key>")
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview")
.header("Idempotency-Key", "<idempotency-key>")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"canceled_at": null,
"checkpoint": "queued",
"completed_at": null,
"created_at": "2026-10-20T10:00:00Z",
"cursor": null,
"execution_attempts": 0,
"failure": null,
"id": "e1f2a3b4-c5d6-4e7f-8a9b-0c1d2e3f4a5b",
"kind": "crawl-preview",
"knowledge_space_id": "0a9d3c2e-5b6f-4e1a-8c7d-2f3e4a5b6c7d",
"last_error_code": null,
"max_execution_attempts": 5,
"progress_completed": 0,
"progress_failed": 0,
"progress_skipped": 0,
"progress_total": null,
"source_id": "6f1c2b0e-3d0b-4a5e-9f4c-1c0e6a7b8d90",
"state": "queued",
"updated_at": "2026-10-20T10:00:00Z"
}Authorizations
Every request authenticates with an API key: Authorization: Bearer {API_KEY}. App endpoints take an app API key; knowledge endpoints take a knowledge base API key (Get Started).
Keep keys server-side; never embed them in client code. Requests with a missing or invalid key fail with HTTP 401 (unauthorized).
Headers
Repeating the same key for the same source from the same API key returns the original run, in its current state, instead of starting another. Use a new key for each preview you mean to start. Use printable ASCII characters without leading or trailing whitespace.
8 - 255Response
The run, with kind: crawl-preview and state: queued.
Last step the run reached: queued, provider-read, preview-staged, selection-frozen, materialized, cleanup-staging, or source-committed. Retry Source Run starts the run over whatever its checkpoint.
Times a worker has picked up the run, including after Retry Source Run, which does not reset it.
Run ID. Pass it to Get Source Run, Cancel Source Run, or Retry Source Run.
What the run does:
sync: started by Sync Source, Update Source, a schedule, or Dify.crawl-preview: started by Preview Website Crawl. It keeps this kind while Import Selected Pages imports from it.crawl-import: started by Import Website Pages.online-document-import,online-drive-import: started by Import Source Items.
Dify also starts previews and imports when a source is added or edited there.
Limit on execution_attempts. Once it is reached, Retry Source Run answers 409.
Items the run expects to process: the pages or items an import requested, the pages a crawl preview has staged so far, or the source's items once a sync has listed them. null until known.
While an Import Website Pages run crawls its URLs, progress_completed and progress_total both count the pages crawled so far. Once every URL is crawled, they restart at 0 of the number of URLs.
Progress of the run.
Live states:
queued: waiting for a workercrawling: a crawl preview is fetching pagespreview_ready: a crawl preview is waiting for Import Selected Pages or Cancel Source Runimporting: importing and indexing itemssyncing: refreshing the source's documents
Terminal states:
completed: every item was imported or refreshed, indexed, and made activezero_results: a crawl preview found no pagesfailed: seefailurecanceled
After failed or canceled, Retry Source Run runs it again, and Sync Source starts a new sync.
Was this page helpful?
curl --request POST \
--url https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview \
--header 'Authorization: Bearer <token>' \
--header 'Idempotency-Key: <idempotency-key>'import requests
url = "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview"
headers = {
"Idempotency-Key": "<idempotency-key>",
"Authorization": "Bearer <token>"
}
response = requests.post(url, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'Idempotency-Key': '<idempotency-key>', Authorization: 'Bearer <token>'}
};
fetch('https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Idempotency-Key: <idempotency-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview"
req, _ := http.NewRequest("POST", url, nil)
req.Header.Add("Idempotency-Key", "<idempotency-key>")
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview")
.header("Idempotency-Key", "<idempotency-key>")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-preview")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"canceled_at": null,
"checkpoint": "queued",
"completed_at": null,
"created_at": "2026-10-20T10:00:00Z",
"cursor": null,
"execution_attempts": 0,
"failure": null,
"id": "e1f2a3b4-c5d6-4e7f-8a9b-0c1d2e3f4a5b",
"kind": "crawl-preview",
"knowledge_space_id": "0a9d3c2e-5b6f-4e1a-8c7d-2f3e4a5b6c7d",
"last_error_code": null,
"max_execution_attempts": 5,
"progress_completed": 0,
"progress_failed": 0,
"progress_skipped": 0,
"progress_total": null,
"source_id": "6f1c2b0e-3d0b-4a5e-9f4c-1c0e6a7b8d90",
"state": "queued",
"updated_at": "2026-10-20T10:00:00Z"
}