Import Website Pages
Imports known URLs into a website source and returns the run. Poll Get Source Run with its id until state is completed, failed, or canceled.
Each URL is crawled on its own, and importing starts once every URL has been crawled. A URL that returns no page fails the run before any page is imported, and the source’s documents don’t change.
When the run completes, sourceUrls becomes the source’s page list. Each import sets the whole list, so send every URL you want the source to keep.
The source must be a web source. A source runs one sync, preview, or import at a time. Wait for its active run to end before starting another, and cancel a crawl preview you won’t import.
For both ways to import website pages, see Import Website Pages.
curl --request POST \
--url https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--header 'Idempotency-Key: <idempotency-key>' \
--data '
{
"replaceExistingSelection": true,
"sourceUrls": [
"https://docs.example.com/guides/billing",
"https://docs.example.com/pricing"
]
}
'import requests
url = "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import"
payload = {
"replaceExistingSelection": True,
"sourceUrls": ["https://docs.example.com/guides/billing", "https://docs.example.com/pricing"]
}
headers = {
"Idempotency-Key": "<idempotency-key>",
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {
'Idempotency-Key': '<idempotency-key>',
Authorization: 'Bearer <token>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
replaceExistingSelection: true,
sourceUrls: ['https://docs.example.com/guides/billing', 'https://docs.example.com/pricing']
})
};
fetch('https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'replaceExistingSelection' => true,
'sourceUrls' => [
'https://docs.example.com/guides/billing',
'https://docs.example.com/pricing'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json",
"Idempotency-Key: <idempotency-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import"
payload := strings.NewReader("{\n \"replaceExistingSelection\": true,\n \"sourceUrls\": [\n \"https://docs.example.com/guides/billing\",\n \"https://docs.example.com/pricing\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Idempotency-Key", "<idempotency-key>")
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import")
.header("Idempotency-Key", "<idempotency-key>")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"replaceExistingSelection\": true,\n \"sourceUrls\": [\n \"https://docs.example.com/guides/billing\",\n \"https://docs.example.com/pricing\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"replaceExistingSelection\": true,\n \"sourceUrls\": [\n \"https://docs.example.com/guides/billing\",\n \"https://docs.example.com/pricing\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"canceled_at": null,
"checkpoint": "queued",
"completed_at": null,
"created_at": "2026-10-20T10:00:00Z",
"cursor": null,
"execution_attempts": 0,
"failure": null,
"id": "e1f2a3b4-c5d6-4e7f-8a9b-0c1d2e3f4a5b",
"kind": "crawl-import",
"knowledge_space_id": "0a9d3c2e-5b6f-4e1a-8c7d-2f3e4a5b6c7d",
"last_error_code": null,
"max_execution_attempts": 5,
"progress_completed": 0,
"progress_failed": 0,
"progress_skipped": 0,
"progress_total": 2,
"source_id": "6f1c2b0e-3d0b-4a5e-9f4c-1c0e6a7b8d90",
"state": "queued",
"updated_at": "2026-10-20T10:00:00Z"
}Authorizations
Every request authenticates with an API key: Authorization: Bearer {API_KEY}. App endpoints take an app API key; knowledge endpoints take a knowledge base API key (Get Started).
Keep keys server-side; never embed them in client code. Requests with a missing or invalid key fail with HTTP 401 (unauthorized).
Headers
Repeating the same key and request from the same API key returns the original run in its current state. Use a new key for each import you start. Reusing the key on this source with a different body returns 409. Use printable ASCII characters without leading or trailing whitespace.
8 - 255Body
The normalized request must fit within 1,048,576 UTF-8 bytes. See Request Size Limits.
Response
The run, with kind: crawl-import, state: queued, and progress_total set to the number of URLs.
Last step the run reached: queued, provider-read, preview-staged, selection-frozen, materialized, cleanup-staging, or source-committed. Retry Source Run starts the run over whatever its checkpoint.
Times a worker has picked up the run, including after Retry Source Run, which does not reset it.
Run ID. Pass it to Get Source Run, Cancel Source Run, or Retry Source Run.
What the run does:
sync: started by Sync Source, Update Source, a schedule, or Dify.crawl-preview: started by Preview Website Crawl. It keeps this kind while Import Selected Pages imports from it.crawl-import: started by Import Website Pages.online-document-import,online-drive-import: started by Import Source Items.
Dify also starts previews and imports when a source is added or edited there.
Limit on execution_attempts. Once it is reached, Retry Source Run answers 409.
Items the run expects to process: the pages or items an import requested, the pages a crawl preview has staged so far, or the source's items once a sync has listed them. null until known.
While an Import Website Pages run crawls its URLs, progress_completed and progress_total both count the pages crawled so far. Once every URL is crawled, they restart at 0 of the number of URLs.
Progress of the run.
Live states:
queued: waiting for a workercrawling: a crawl preview is fetching pagespreview_ready: a crawl preview is waiting for Import Selected Pages or Cancel Source Runimporting: importing and indexing itemssyncing: refreshing the source's documents
Terminal states:
completed: every item was imported or refreshed, indexed, and made activezero_results: a crawl preview found no pagesfailed: seefailurecanceled
After failed or canceled, Retry Source Run runs it again, and Sync Source starts a new sync.
Was this page helpful?
curl --request POST \
--url https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--header 'Idempotency-Key: <idempotency-key>' \
--data '
{
"replaceExistingSelection": true,
"sourceUrls": [
"https://docs.example.com/guides/billing",
"https://docs.example.com/pricing"
]
}
'import requests
url = "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import"
payload = {
"replaceExistingSelection": True,
"sourceUrls": ["https://docs.example.com/guides/billing", "https://docs.example.com/pricing"]
}
headers = {
"Idempotency-Key": "<idempotency-key>",
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {
'Idempotency-Key': '<idempotency-key>',
Authorization: 'Bearer <token>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
replaceExistingSelection: true,
sourceUrls: ['https://docs.example.com/guides/billing', 'https://docs.example.com/pricing']
})
};
fetch('https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'replaceExistingSelection' => true,
'sourceUrls' => [
'https://docs.example.com/guides/billing',
'https://docs.example.com/pricing'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json",
"Idempotency-Key: <idempotency-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import"
payload := strings.NewReader("{\n \"replaceExistingSelection\": true,\n \"sourceUrls\": [\n \"https://docs.example.com/guides/billing\",\n \"https://docs.example.com/pricing\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Idempotency-Key", "<idempotency-key>")
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import")
.header("Idempotency-Key", "<idempotency-key>")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"replaceExistingSelection\": true,\n \"sourceUrls\": [\n \"https://docs.example.com/guides/billing\",\n \"https://docs.example.com/pricing\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://{api_base_url}/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"replaceExistingSelection\": true,\n \"sourceUrls\": [\n \"https://docs.example.com/guides/billing\",\n \"https://docs.example.com/pricing\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"canceled_at": null,
"checkpoint": "queued",
"completed_at": null,
"created_at": "2026-10-20T10:00:00Z",
"cursor": null,
"execution_attempts": 0,
"failure": null,
"id": "e1f2a3b4-c5d6-4e7f-8a9b-0c1d2e3f4a5b",
"kind": "crawl-import",
"knowledge_space_id": "0a9d3c2e-5b6f-4e1a-8c7d-2f3e4a5b6c7d",
"last_error_code": null,
"max_execution_attempts": 5,
"progress_completed": 0,
"progress_failed": 0,
"progress_skipped": 0,
"progress_total": 2,
"source_id": "6f1c2b0e-3d0b-4a5e-9f4c-1c0e6a7b8d90",
"state": "queued",
"updated_at": "2026-10-20T10:00:00Z"
}