scrapfly_sdk/config/
crawler.rs

1//! Crawler endpoint configuration — ported from `sdk/go/config_crawler.go`.
2
3use std::collections::BTreeMap;
4
5use serde::Serialize;
6
7use crate::enums::{CrawlerContentFormat, CrawlerWebhookEvent};
8use crate::error::ScrapflyError;
9
10/// Configuration for a `POST /crawl` request.
11///
12/// Every field except `url` is optional; unset fields are NOT serialized so
13/// the server applies its own documented defaults.
14#[derive(Debug, Clone, Default, Serialize)]
15pub struct CrawlerConfig {
16    /// Seed URL (required, must be HTTP/HTTPS).
17    pub url: String,
18
19    /// Max pages to crawl.
20    #[serde(skip_serializing_if = "Option::is_none")]
21    pub page_limit: Option<u32>,
22    /// Max link-follow depth.
23    #[serde(skip_serializing_if = "Option::is_none")]
24    pub max_depth: Option<u32>,
25    /// Max duration (seconds, 15..=10800).
26    #[serde(skip_serializing_if = "Option::is_none")]
27    pub max_duration: Option<u32>,
28    /// Max API credit to spend (0 = no limit).
29    #[serde(skip_serializing_if = "Option::is_none")]
30    pub max_api_credit: Option<u32>,
31
32    /// Exclude these URL paths (≤100 entries).
33    #[serde(skip_serializing_if = "Vec::is_empty")]
34    pub exclude_paths: Vec<String>,
35    /// Restrict crawl to these paths (≤100 entries).
36    #[serde(skip_serializing_if = "Vec::is_empty")]
37    pub include_only_paths: Vec<String>,
38
39    /// Ignore the seed URL's base-path restriction.
40    #[serde(skip_serializing_if = "is_false")]
41    pub ignore_base_path_restriction: bool,
42    /// Follow links to external domains.
43    #[serde(skip_serializing_if = "is_false")]
44    pub follow_external_links: bool,
45    /// Whitelist of external domains (≤250 entries).
46    #[serde(skip_serializing_if = "Vec::is_empty")]
47    pub allowed_external_domains: Vec<String>,
48
49    /// Tri-state: None = unset (server default true), Some(v) = explicit.
50    #[serde(skip_serializing_if = "Option::is_none")]
51    pub follow_internal_subdomains: Option<bool>,
52    /// Whitelist of internal subdomains (≤250 entries).
53    #[serde(skip_serializing_if = "Vec::is_empty")]
54    pub allowed_internal_subdomains: Vec<String>,
55
56    /// Request headers sent for every crawled page.
57    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
58    pub headers: BTreeMap<String, String>,
59    /// Delay between requests (ms, 0..=15000).
60    #[serde(skip_serializing_if = "Option::is_none")]
61    pub delay: Option<u32>,
62    /// Override User-Agent.
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub user_agent: Option<String>,
65    /// Max concurrent workers.
66    #[serde(skip_serializing_if = "Option::is_none")]
67    pub max_concurrency: Option<u32>,
68    /// Rendering delay (ms, 0..=25000).
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub rendering_delay: Option<u32>,
71
72    /// Honor sitemaps.
73    #[serde(skip_serializing_if = "is_false")]
74    pub use_sitemaps: bool,
75    /// Follow `nofollow` links anyway.
76    #[serde(skip_serializing_if = "is_false")]
77    pub ignore_no_follow: bool,
78
79    /// Tri-state: None = server default (true).
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub respect_robots_txt: Option<bool>,
82
83    /// Enable cache.
84    #[serde(skip_serializing_if = "is_false")]
85    pub cache: bool,
86    /// Cache TTL seconds (0..=604800).
87    #[serde(skip_serializing_if = "Option::is_none")]
88    pub cache_ttl: Option<u32>,
89    /// Force cache refresh.
90    #[serde(skip_serializing_if = "is_false")]
91    pub cache_clear: bool,
92
93    /// Desired content formats.
94    #[serde(skip_serializing_if = "Vec::is_empty")]
95    pub content_formats: Vec<CrawlerContentFormat>,
96    /// Inline extraction rules.
97    #[serde(skip_serializing_if = "Option::is_none")]
98    pub extraction_rules: Option<serde_json::Value>,
99
100    /// Enable ASP bypass.
101    #[serde(skip_serializing_if = "is_false")]
102    pub asp: bool,
103    /// Proxy pool name.
104    #[serde(skip_serializing_if = "Option::is_none")]
105    pub proxy_pool: Option<String>,
106    /// Proxy country.
107    #[serde(skip_serializing_if = "Option::is_none")]
108    pub country: Option<String>,
109
110    /// Webhook name.
111    #[serde(skip_serializing_if = "Option::is_none")]
112    pub webhook_name: Option<String>,
113    /// Webhook events.
114    #[serde(skip_serializing_if = "Vec::is_empty")]
115    pub webhook_events: Vec<CrawlerWebhookEvent>,
116}
117
118fn is_false(v: &bool) -> bool {
119    !*v
120}
121
122impl CrawlerConfig {
123    /// Start a builder for `url`.
124    pub fn builder(url: impl Into<String>) -> CrawlerConfigBuilder {
125        CrawlerConfigBuilder {
126            cfg: CrawlerConfig {
127                url: url.into(),
128                ..Default::default()
129            },
130        }
131    }
132
133    /// Validate numeric bounds + list sizes. Ported from
134    /// `sdk/go/config_crawler.go::validateBounds`.
135    pub fn validate(&self) -> Result<(), ScrapflyError> {
136        if self.url.is_empty() {
137            return Err(ScrapflyError::Config("url is required".into()));
138        }
139        if let Some(d) = self.max_duration {
140            if !(15..=10800).contains(&d) {
141                return Err(ScrapflyError::Config(format!(
142                    "max_duration must be between 15 and 10800 seconds, got {}",
143                    d
144                )));
145            }
146        }
147        if let Some(rd) = self.rendering_delay {
148            if rd > 25000 {
149                return Err(ScrapflyError::Config(format!(
150                    "rendering_delay must be between 0 and 25000 ms, got {}",
151                    rd
152                )));
153            }
154        }
155        if let Some(delay) = self.delay {
156            if delay > 15000 {
157                return Err(ScrapflyError::Config(format!(
158                    "delay must be between 0 and 15000 ms, got {}",
159                    delay
160                )));
161            }
162        }
163        if let Some(ttl) = self.cache_ttl {
164            if ttl > 604800 {
165                return Err(ScrapflyError::Config(format!(
166                    "cache_ttl must be between 0 and 604800 seconds, got {}",
167                    ttl
168                )));
169            }
170        }
171        if self.exclude_paths.len() > 100 {
172            return Err(ScrapflyError::Config(format!(
173                "exclude_paths is limited to 100 entries, got {}",
174                self.exclude_paths.len()
175            )));
176        }
177        if self.include_only_paths.len() > 100 {
178            return Err(ScrapflyError::Config(format!(
179                "include_only_paths is limited to 100 entries, got {}",
180                self.include_only_paths.len()
181            )));
182        }
183        if !self.exclude_paths.is_empty() && !self.include_only_paths.is_empty() {
184            return Err(ScrapflyError::Config(
185                "exclude_paths and include_only_paths are mutually exclusive".into(),
186            ));
187        }
188        if self.allowed_external_domains.len() > 250 {
189            return Err(ScrapflyError::Config(format!(
190                "allowed_external_domains is limited to 250 entries, got {}",
191                self.allowed_external_domains.len()
192            )));
193        }
194        if self.allowed_internal_subdomains.len() > 250 {
195            return Err(ScrapflyError::Config(format!(
196                "allowed_internal_subdomains is limited to 250 entries, got {}",
197                self.allowed_internal_subdomains.len()
198            )));
199        }
200        Ok(())
201    }
202
203    /// Serialize into the JSON body the crawler endpoint expects.
204    pub fn to_json_body(&self) -> Result<Vec<u8>, ScrapflyError> {
205        self.validate()?;
206        Ok(serde_json::to_vec(self)?)
207    }
208}
209
210/// Builder for [`CrawlerConfig`].
211#[derive(Debug, Clone)]
212pub struct CrawlerConfigBuilder {
213    cfg: CrawlerConfig,
214}
215
216impl CrawlerConfigBuilder {
217    /// Set page limit.
218    pub fn page_limit(mut self, v: u32) -> Self {
219        self.cfg.page_limit = Some(v);
220        self
221    }
222    /// Set max depth.
223    pub fn max_depth(mut self, v: u32) -> Self {
224        self.cfg.max_depth = Some(v);
225        self
226    }
227    /// Set max duration (seconds).
228    pub fn max_duration(mut self, v: u32) -> Self {
229        self.cfg.max_duration = Some(v);
230        self
231    }
232    /// Set max API credit.
233    pub fn max_api_credit(mut self, v: u32) -> Self {
234        self.cfg.max_api_credit = Some(v);
235        self
236    }
237    /// Set exclude paths.
238    pub fn exclude_paths(mut self, v: Vec<String>) -> Self {
239        self.cfg.exclude_paths = v;
240        self
241    }
242    /// Set include-only paths.
243    pub fn include_only_paths(mut self, v: Vec<String>) -> Self {
244        self.cfg.include_only_paths = v;
245        self
246    }
247    /// Ignore base-path restriction.
248    pub fn ignore_base_path_restriction(mut self, v: bool) -> Self {
249        self.cfg.ignore_base_path_restriction = v;
250        self
251    }
252    /// Follow external links.
253    pub fn follow_external_links(mut self, v: bool) -> Self {
254        self.cfg.follow_external_links = v;
255        self
256    }
257    /// Set allowed external domains.
258    pub fn allowed_external_domains(mut self, v: Vec<String>) -> Self {
259        self.cfg.allowed_external_domains = v;
260        self
261    }
262    /// Tri-state follow-internal-subdomains.
263    pub fn follow_internal_subdomains(mut self, v: bool) -> Self {
264        self.cfg.follow_internal_subdomains = Some(v);
265        self
266    }
267    /// Set allowed internal subdomains.
268    pub fn allowed_internal_subdomains(mut self, v: Vec<String>) -> Self {
269        self.cfg.allowed_internal_subdomains = v;
270        self
271    }
272    /// Add header.
273    pub fn header(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
274        self.cfg.headers.insert(k.into(), v.into());
275        self
276    }
277    /// Set delay (ms).
278    pub fn delay(mut self, v: u32) -> Self {
279        self.cfg.delay = Some(v);
280        self
281    }
282    /// Set User-Agent.
283    pub fn user_agent(mut self, v: impl Into<String>) -> Self {
284        self.cfg.user_agent = Some(v.into());
285        self
286    }
287    /// Set max concurrency.
288    pub fn max_concurrency(mut self, v: u32) -> Self {
289        self.cfg.max_concurrency = Some(v);
290        self
291    }
292    /// Set rendering delay (ms).
293    pub fn rendering_delay(mut self, v: u32) -> Self {
294        self.cfg.rendering_delay = Some(v);
295        self
296    }
297    /// Honor sitemaps.
298    pub fn use_sitemaps(mut self, v: bool) -> Self {
299        self.cfg.use_sitemaps = v;
300        self
301    }
302    /// Ignore nofollow.
303    pub fn ignore_no_follow(mut self, v: bool) -> Self {
304        self.cfg.ignore_no_follow = v;
305        self
306    }
307    /// Tri-state respect-robots-txt.
308    pub fn respect_robots_txt(mut self, v: bool) -> Self {
309        self.cfg.respect_robots_txt = Some(v);
310        self
311    }
312    /// Enable cache.
313    pub fn cache(mut self, v: bool) -> Self {
314        self.cfg.cache = v;
315        self
316    }
317    /// Cache TTL.
318    pub fn cache_ttl(mut self, v: u32) -> Self {
319        self.cfg.cache_ttl = Some(v);
320        self
321    }
322    /// Force cache refresh.
323    pub fn cache_clear(mut self, v: bool) -> Self {
324        self.cfg.cache_clear = v;
325        self
326    }
327    /// Add content format.
328    pub fn content_format(mut self, v: CrawlerContentFormat) -> Self {
329        self.cfg.content_formats.push(v);
330        self
331    }
332    /// Set extraction rules.
333    pub fn extraction_rules(mut self, v: serde_json::Value) -> Self {
334        self.cfg.extraction_rules = Some(v);
335        self
336    }
337    /// Enable ASP.
338    pub fn asp(mut self, v: bool) -> Self {
339        self.cfg.asp = v;
340        self
341    }
342    /// Set proxy pool name.
343    pub fn proxy_pool(mut self, v: impl Into<String>) -> Self {
344        self.cfg.proxy_pool = Some(v.into());
345        self
346    }
347    /// Set country.
348    pub fn country(mut self, v: impl Into<String>) -> Self {
349        self.cfg.country = Some(v.into());
350        self
351    }
352    /// Set webhook name.
353    pub fn webhook_name(mut self, v: impl Into<String>) -> Self {
354        self.cfg.webhook_name = Some(v.into());
355        self
356    }
357    /// Add webhook event.
358    pub fn webhook_event(mut self, v: CrawlerWebhookEvent) -> Self {
359        self.cfg.webhook_events.push(v);
360        self
361    }
362    /// Finalize the builder.
363    pub fn build(self) -> Result<CrawlerConfig, ScrapflyError> {
364        self.cfg.validate()?;
365        Ok(self.cfg)
366    }
367}
scrapfly_sdk/config/crawler.rs

scrapfly_sdk/config/
crawler.rs
