Struct CrawlerConfig

pub struct CrawlerConfig {

    pub url: String,
    pub page_limit: Option<u32>,
    pub max_depth: Option<u32>,
    pub max_duration: Option<u32>,
    pub max_api_credit: Option<u32>,
    pub exclude_paths: Vec<String>,
    pub include_only_paths: Vec<String>,
    pub ignore_base_path_restriction: bool,
    pub follow_external_links: bool,
    pub allowed_external_domains: Vec<String>,
    pub follow_internal_subdomains: Option<bool>,
    pub allowed_internal_subdomains: Vec<String>,
    pub headers: BTreeMap<String, String>,
    pub delay: Option<u32>,
    pub user_agent: Option<String>,
    pub max_concurrency: Option<u32>,
    pub rendering_delay: Option<u32>,
    pub use_sitemaps: bool,
    pub ignore_no_follow: bool,
    pub respect_robots_txt: Option<bool>,
    pub cache: bool,
    pub cache_ttl: Option<u32>,
    pub cache_clear: bool,
    pub content_formats: Vec<CrawlerContentFormat>,
    pub extraction_rules: Option<Value>,
    pub asp: bool,
    pub proxy_pool: Option<String>,
    pub country: Option<String>,
    pub webhook_name: Option<String>,
    pub webhook_events: Vec<CrawlerWebhookEvent>,
}

Configuration for a POST /crawl request.

Every field except url is optional; unset fields are NOT serialized so the server applies its own documented defaults.

Fields

url: String

Seed URL (required, must be HTTP/HTTPS).

page_limit: Option<u32>

Max pages to crawl.

max_depth: Option<u32>

Max link-follow depth.

max_duration: Option<u32>

Max duration (seconds, 15..=10800).

max_api_credit: Option<u32>

Max API credit to spend (0 = no limit).

exclude_paths: Vec<String>

Exclude these URL paths (≤100 entries).

include_only_paths: Vec<String>

Restrict crawl to these paths (≤100 entries).

ignore_base_path_restriction: bool

Ignore the seed URL’s base-path restriction.

follow_external_links: bool

Follow links to external domains.

allowed_external_domains: Vec<String>

Whitelist of external domains (≤250 entries).

follow_internal_subdomains: Option<bool>

Tri-state: None = unset (server default true), Some(v) = explicit.

allowed_internal_subdomains: Vec<String>

Whitelist of internal subdomains (≤250 entries).

headers: BTreeMap<String, String>

Request headers sent for every crawled page.

delay: Option<u32>

Delay between requests (ms, 0..=15000).

user_agent: Option<String>

Override User-Agent.

max_concurrency: Option<u32>

Max concurrent workers.

rendering_delay: Option<u32>

Rendering delay (ms, 0..=25000).

use_sitemaps: bool

Honor sitemaps.

ignore_no_follow: bool

Follow nofollow links anyway.

respect_robots_txt: Option<bool>

Tri-state: None = server default (true).

cache: bool

Enable cache.

cache_ttl: Option<u32>

Cache TTL seconds (0..=604800).

cache_clear: bool

Force cache refresh.

content_formats: Vec<CrawlerContentFormat>

Desired content formats.

extraction_rules: Option<Value>

Inline extraction rules.

asp: bool

Enable ASP bypass.

proxy_pool: Option<String>

Proxy pool name.

country: Option<String>

Proxy country.

webhook_name: Option<String>

Webhook name.

webhook_events: Vec<CrawlerWebhookEvent>

Webhook events.

Implementations

impl CrawlerConfig

pub fn builder(url: impl Into<String>) -> CrawlerConfigBuilder

Start a builder for url.

pub fn validate(&self) -> Result<(), ScrapflyError>

Validate numeric bounds + list sizes. Ported from sdk/go/config_crawler.go::validateBounds.

pub fn to_json_body(&self) -> Result<Vec<u8>, ScrapflyError>

Serialize into the JSON body the crawler endpoint expects.

Trait Implementations

impl Clone for CrawlerConfig

fn clone(&self) -> CrawlerConfig

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CrawlerConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for CrawlerConfig

fn default() -> CrawlerConfig

Returns the “default value” for a type.

impl Serialize for CrawlerConfig

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.