Struct CrawlerConfigBuilder

pub struct CrawlerConfigBuilder { /* private fields */ }

Builder for CrawlerConfig.

Implementations

impl CrawlerConfigBuilder

pub fn page_limit(self, v: u32) -> Self

Set page limit.

pub fn max_depth(self, v: u32) -> Self

Set max depth.

pub fn max_duration(self, v: u32) -> Self

Set max duration (seconds).

pub fn max_api_credit(self, v: u32) -> Self

Set max API credit.

pub fn exclude_paths(self, v: Vec<String>) -> Self

Set exclude paths.

pub fn include_only_paths(self, v: Vec<String>) -> Self

Set include-only paths.

pub fn ignore_base_path_restriction(self, v: bool) -> Self

Ignore base-path restriction.

pub fn follow_external_links(self, v: bool) -> Self

Follow external links.

pub fn allowed_external_domains(self, v: Vec<String>) -> Self

Set allowed external domains.

pub fn follow_internal_subdomains(self, v: bool) -> Self

Tri-state follow-internal-subdomains.

pub fn allowed_internal_subdomains(self, v: Vec<String>) -> Self

Set allowed internal subdomains.

pub fn header(self, k: impl Into<String>, v: impl Into<String>) -> Self

Add header.

pub fn delay(self, v: u32) -> Self

Set delay (ms).

pub fn user_agent(self, v: impl Into<String>) -> Self

Set User-Agent.

pub fn max_concurrency(self, v: u32) -> Self

Set max concurrency.

pub fn rendering_delay(self, v: u32) -> Self

Set rendering delay (ms).

pub fn use_sitemaps(self, v: bool) -> Self

Honor sitemaps.

pub fn ignore_no_follow(self, v: bool) -> Self

Ignore nofollow.

pub fn respect_robots_txt(self, v: bool) -> Self

Tri-state respect-robots-txt.

pub fn cache(self, v: bool) -> Self

Enable cache.

pub fn cache_ttl(self, v: u32) -> Self

Cache TTL.

pub fn cache_clear(self, v: bool) -> Self

Force cache refresh.

pub fn content_format(self, v: CrawlerContentFormat) -> Self

Add content format.

pub fn extraction_rules(self, v: Value) -> Self

Set extraction rules.

pub fn asp(self, v: bool) -> Self

Enable ASP.

pub fn proxy_pool(self, v: impl Into<String>) -> Self

Set proxy pool name.

pub fn country(self, v: impl Into<String>) -> Self

Set country.

pub fn webhook_name(self, v: impl Into<String>) -> Self

Set webhook name.

pub fn webhook_event(self, v: CrawlerWebhookEvent) -> Self

Add webhook event.

pub fn build(self) -> Result<CrawlerConfig, ScrapflyError>

Finalize the builder.

Trait Implementations

impl Clone for CrawlerConfigBuilder

fn clone(&self) -> CrawlerConfigBuilder

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CrawlerConfigBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.