scrapfly_sdk/config/
crawler.rs

1//! Crawler endpoint configuration — ported from `sdk/go/config_crawler.go`.
2
3use std::collections::BTreeMap;
4
5use serde::Serialize;
6
7use crate::enums::{CrawlerContentFormat, CrawlerWebhookEvent};
8use crate::error::ScrapflyError;
9
10/// Configuration for a `POST /crawl` request.
11///
12/// Exactly one URL source must be provided: `url` (seed crawl with
13/// discovery), `url_list` (in-memory list, no discovery), or
14/// `remote_url_list` (URL of a hosted text file fetched at crawl start, no
15/// discovery). Other fields default to server-side values when zero.
16#[derive(Debug, Clone, Default, Serialize)]
17pub struct CrawlerConfig {
18    /// Seed URL (must be HTTP/HTTPS). Mutually exclusive with `url_list`
19    /// and `remote_url_list`.
20    #[serde(skip_serializing_if = "String::is_empty")]
21    pub url: String,
22    /// Explicit URL list (no discovery). Mutually exclusive with `url`
23    /// and `remote_url_list`. When set, the SDK posts the request as
24    /// multipart/form-data so the URLs are uploaded as a streamed file
25    /// payload rather than inlined into the JSON body.
26    #[serde(skip)]
27    pub url_list: Vec<String>,
28    /// URL of a hosted text file (one URL per line) fetched at crawl
29    /// start. Mutually exclusive with `url` and `url_list`.
30    #[serde(skip_serializing_if = "String::is_empty")]
31    pub remote_url_list: String,
32
33    /// Max pages to crawl.
34    #[serde(skip_serializing_if = "Option::is_none")]
35    pub page_limit: Option<u32>,
36    /// Max link-follow depth.
37    #[serde(skip_serializing_if = "Option::is_none")]
38    pub max_depth: Option<u32>,
39    /// Max duration (seconds, 15..=10800).
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub max_duration: Option<u32>,
42    /// Max API credit to spend (0 = no limit).
43    #[serde(skip_serializing_if = "Option::is_none")]
44    pub max_api_credit: Option<u32>,
45
46    /// Exclude these URL paths (≤100 entries).
47    #[serde(skip_serializing_if = "Vec::is_empty")]
48    pub exclude_paths: Vec<String>,
49    /// Restrict crawl to these paths (≤100 entries).
50    #[serde(skip_serializing_if = "Vec::is_empty")]
51    pub include_only_paths: Vec<String>,
52
53    /// Ignore the seed URL's base-path restriction.
54    #[serde(skip_serializing_if = "is_false")]
55    pub ignore_base_path_restriction: bool,
56    /// Follow links to external domains.
57    #[serde(skip_serializing_if = "is_false")]
58    pub follow_external_links: bool,
59    /// Whitelist of external domains (≤250 entries).
60    #[serde(skip_serializing_if = "Vec::is_empty")]
61    pub allowed_external_domains: Vec<String>,
62
63    /// Tri-state: None = unset (server default true), Some(v) = explicit.
64    #[serde(skip_serializing_if = "Option::is_none")]
65    pub follow_internal_subdomains: Option<bool>,
66    /// Whitelist of internal subdomains (≤250 entries).
67    #[serde(skip_serializing_if = "Vec::is_empty")]
68    pub allowed_internal_subdomains: Vec<String>,
69
70    /// Request headers sent for every crawled page.
71    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
72    pub headers: BTreeMap<String, String>,
73    /// Delay between requests (ms, 0..=15000).
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub delay: Option<u32>,
76    /// Override User-Agent.
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub user_agent: Option<String>,
79    /// Max concurrent workers.
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub max_concurrency: Option<u32>,
82    /// Rendering delay (ms, 0..=25000).
83    #[serde(skip_serializing_if = "Option::is_none")]
84    pub rendering_delay: Option<u32>,
85
86    /// Honor sitemaps.
87    #[serde(skip_serializing_if = "is_false")]
88    pub use_sitemaps: bool,
89    /// Follow `nofollow` links anyway.
90    #[serde(skip_serializing_if = "is_false")]
91    pub ignore_no_follow: bool,
92
93    /// Tri-state: None = server default (true).
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub respect_robots_txt: Option<bool>,
96
97    /// Enable cache.
98    #[serde(skip_serializing_if = "is_false")]
99    pub cache: bool,
100    /// Cache TTL seconds (0..=604800).
101    #[serde(skip_serializing_if = "Option::is_none")]
102    pub cache_ttl: Option<u32>,
103    /// Force cache refresh.
104    #[serde(skip_serializing_if = "is_false")]
105    pub cache_clear: bool,
106
107    /// Desired content formats.
108    #[serde(skip_serializing_if = "Vec::is_empty")]
109    pub content_formats: Vec<CrawlerContentFormat>,
110    /// Inline extraction rules.
111    #[serde(skip_serializing_if = "Option::is_none")]
112    pub extraction_rules: Option<serde_json::Value>,
113
114    /// Enable ASP bypass.
115    #[serde(skip_serializing_if = "is_false")]
116    pub asp: bool,
117    /// Proxy pool name.
118    #[serde(skip_serializing_if = "Option::is_none")]
119    pub proxy_pool: Option<String>,
120    /// Proxy country.
121    #[serde(skip_serializing_if = "Option::is_none")]
122    pub country: Option<String>,
123
124    /// Webhook name.
125    #[serde(skip_serializing_if = "Option::is_none")]
126    pub webhook_name: Option<String>,
127    /// Webhook events.
128    #[serde(skip_serializing_if = "Vec::is_empty")]
129    pub webhook_events: Vec<CrawlerWebhookEvent>,
130}
131
132fn is_false(v: &bool) -> bool {
133    !*v
134}
135
136impl CrawlerConfig {
137    /// Start a builder for `url` (seed-URL crawl with discovery).
138    pub fn builder(url: impl Into<String>) -> CrawlerConfigBuilder {
139        CrawlerConfigBuilder {
140            cfg: CrawlerConfig {
141                url: url.into(),
142                ..Default::default()
143            },
144        }
145    }
146
147    /// Start a builder for an explicit `url_list` (no discovery). The SDK
148    /// will post this configuration as multipart/form-data so the URLs are
149    /// uploaded as a streamed file payload.
150    pub fn builder_url_list(
151        urls: impl IntoIterator<Item = impl Into<String>>,
152    ) -> CrawlerConfigBuilder {
153        CrawlerConfigBuilder {
154            cfg: CrawlerConfig {
155                url_list: urls.into_iter().map(Into::into).collect(),
156                ..Default::default()
157            },
158        }
159    }
160
161    /// Start a builder for `remote_url_list` (no discovery, list fetched
162    /// from the given URL at crawl start).
163    pub fn builder_remote_url_list(url: impl Into<String>) -> CrawlerConfigBuilder {
164        CrawlerConfigBuilder {
165            cfg: CrawlerConfig {
166                remote_url_list: url.into(),
167                ..Default::default()
168            },
169        }
170    }
171
172    /// Validate numeric bounds + list sizes. Ported from
173    /// `sdk/go/config_crawler.go::validateBounds`.
174    pub fn validate(&self) -> Result<(), ScrapflyError> {
175        let has_seed = !self.url.is_empty();
176        let has_list = !self.url_list.is_empty();
177        let has_remote = !self.remote_url_list.is_empty();
178        let count = (has_seed as u8) + (has_list as u8) + (has_remote as u8);
179        if count == 0 {
180            return Err(ScrapflyError::Config(
181                "provide one of url, url_list, or remote_url_list".into(),
182            ));
183        }
184        if count > 1 {
185            return Err(ScrapflyError::Config(
186                "only one of url, url_list, or remote_url_list can be set".into(),
187            ));
188        }
189        if let Some(d) = self.max_duration {
190            if !(15..=10800).contains(&d) {
191                return Err(ScrapflyError::Config(format!(
192                    "max_duration must be between 15 and 10800 seconds, got {}",
193                    d
194                )));
195            }
196        }
197        if let Some(rd) = self.rendering_delay {
198            if rd > 25000 {
199                return Err(ScrapflyError::Config(format!(
200                    "rendering_delay must be between 0 and 25000 ms, got {}",
201                    rd
202                )));
203            }
204        }
205        if let Some(delay) = self.delay {
206            if delay > 15000 {
207                return Err(ScrapflyError::Config(format!(
208                    "delay must be between 0 and 15000 ms, got {}",
209                    delay
210                )));
211            }
212        }
213        if let Some(ttl) = self.cache_ttl {
214            if ttl > 604800 {
215                return Err(ScrapflyError::Config(format!(
216                    "cache_ttl must be between 0 and 604800 seconds, got {}",
217                    ttl
218                )));
219            }
220        }
221        if self.exclude_paths.len() > 100 {
222            return Err(ScrapflyError::Config(format!(
223                "exclude_paths is limited to 100 entries, got {}",
224                self.exclude_paths.len()
225            )));
226        }
227        if self.include_only_paths.len() > 100 {
228            return Err(ScrapflyError::Config(format!(
229                "include_only_paths is limited to 100 entries, got {}",
230                self.include_only_paths.len()
231            )));
232        }
233        if !self.exclude_paths.is_empty() && !self.include_only_paths.is_empty() {
234            return Err(ScrapflyError::Config(
235                "exclude_paths and include_only_paths are mutually exclusive".into(),
236            ));
237        }
238        if self.allowed_external_domains.len() > 250 {
239            return Err(ScrapflyError::Config(format!(
240                "allowed_external_domains is limited to 250 entries, got {}",
241                self.allowed_external_domains.len()
242            )));
243        }
244        if self.allowed_internal_subdomains.len() > 250 {
245            return Err(ScrapflyError::Config(format!(
246                "allowed_internal_subdomains is limited to 250 entries, got {}",
247                self.allowed_internal_subdomains.len()
248            )));
249        }
250        Ok(())
251    }
252
253    /// Serialize into the JSON body the crawler endpoint expects. Used
254    /// for seed-URL crawls and remote_url_list crawls. For in-memory URL
255    /// lists the SDK switches to multipart (see `to_multipart_body`).
256    pub fn to_json_body(&self) -> Result<Vec<u8>, ScrapflyError> {
257        self.validate()?;
258        Ok(serde_json::to_vec(self)?)
259    }
260
261    /// Build a multipart/form-data body for `POST /crawl` when an in-memory
262    /// `url_list` is supplied. The `config` part carries the JSON config
263    /// (without url_list) and the `urls` part carries the URLs as
264    /// text/plain, one per line. Returns the body bytes and the matching
265    /// Content-Type header (with the boundary baked in).
266    pub fn to_multipart_body(&self) -> Result<(Vec<u8>, String), ScrapflyError> {
267        self.validate()?;
268        if self.url_list.is_empty() {
269            return Err(ScrapflyError::Config(
270                "to_multipart_body requires url_list to be set".into(),
271            ));
272        }
273
274        // Boundary derived from the SDK name + a short random tail. The
275        // value is opaque to the server; it just needs to be unique within
276        // the request and absent from the body content (URLs and JSON).
277        let boundary = format!("scrapfly-rs-{:016x}", rand_u64());
278        let config_json = serde_json::to_vec(self)?;
279        let urls_blob = self.url_list.join("\n");
280
281        let mut buf: Vec<u8> = Vec::with_capacity(config_json.len() + urls_blob.len() + 256);
282        buf.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
283        buf.extend_from_slice(
284            b"Content-Disposition: form-data; name=\"config\"; filename=\"config.json\"\r\n",
285        );
286        buf.extend_from_slice(b"Content-Type: application/json\r\n\r\n");
287        buf.extend_from_slice(&config_json);
288        buf.extend_from_slice(b"\r\n");
289        buf.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
290        buf.extend_from_slice(
291            b"Content-Disposition: form-data; name=\"urls\"; filename=\"urls.txt\"\r\n",
292        );
293        buf.extend_from_slice(b"Content-Type: text/plain\r\n\r\n");
294        buf.extend_from_slice(urls_blob.as_bytes());
295        buf.extend_from_slice(b"\r\n");
296        buf.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
297
298        Ok((buf, format!("multipart/form-data; boundary={}", boundary)))
299    }
300}
301
302// Small non-crypto random for the multipart boundary. We use a fast,
303// dependency-free PRNG seeded from the system clock — the boundary just
304// needs to be unique within the request, not cryptographically secure.
305fn rand_u64() -> u64 {
306    use std::time::{SystemTime, UNIX_EPOCH};
307    let nanos = SystemTime::now()
308        .duration_since(UNIX_EPOCH)
309        .map(|d| d.as_nanos() as u64)
310        .unwrap_or(0);
311    // splitmix64 step from the source paper — adequate for boundary tags.
312    let mut z = nanos.wrapping_add(0x9E3779B97F4A7C15);
313    z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9);
314    z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB);
315    z ^ (z >> 31)
316}
317
318/// Builder for [`CrawlerConfig`].
319#[derive(Debug, Clone)]
320pub struct CrawlerConfigBuilder {
321    cfg: CrawlerConfig,
322}
323
324impl CrawlerConfigBuilder {
325    /// Set page limit.
326    pub fn page_limit(mut self, v: u32) -> Self {
327        self.cfg.page_limit = Some(v);
328        self
329    }
330    /// Set max depth.
331    pub fn max_depth(mut self, v: u32) -> Self {
332        self.cfg.max_depth = Some(v);
333        self
334    }
335    /// Set max duration (seconds).
336    pub fn max_duration(mut self, v: u32) -> Self {
337        self.cfg.max_duration = Some(v);
338        self
339    }
340    /// Set max API credit.
341    pub fn max_api_credit(mut self, v: u32) -> Self {
342        self.cfg.max_api_credit = Some(v);
343        self
344    }
345    /// Set exclude paths.
346    pub fn exclude_paths(mut self, v: Vec<String>) -> Self {
347        self.cfg.exclude_paths = v;
348        self
349    }
350    /// Set include-only paths.
351    pub fn include_only_paths(mut self, v: Vec<String>) -> Self {
352        self.cfg.include_only_paths = v;
353        self
354    }
355    /// Ignore base-path restriction.
356    pub fn ignore_base_path_restriction(mut self, v: bool) -> Self {
357        self.cfg.ignore_base_path_restriction = v;
358        self
359    }
360    /// Follow external links.
361    pub fn follow_external_links(mut self, v: bool) -> Self {
362        self.cfg.follow_external_links = v;
363        self
364    }
365    /// Set allowed external domains.
366    pub fn allowed_external_domains(mut self, v: Vec<String>) -> Self {
367        self.cfg.allowed_external_domains = v;
368        self
369    }
370    /// Tri-state follow-internal-subdomains.
371    pub fn follow_internal_subdomains(mut self, v: bool) -> Self {
372        self.cfg.follow_internal_subdomains = Some(v);
373        self
374    }
375    /// Set allowed internal subdomains.
376    pub fn allowed_internal_subdomains(mut self, v: Vec<String>) -> Self {
377        self.cfg.allowed_internal_subdomains = v;
378        self
379    }
380    /// Add header.
381    pub fn header(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
382        self.cfg.headers.insert(k.into(), v.into());
383        self
384    }
385    /// Set delay (ms).
386    pub fn delay(mut self, v: u32) -> Self {
387        self.cfg.delay = Some(v);
388        self
389    }
390    /// Set User-Agent.
391    pub fn user_agent(mut self, v: impl Into<String>) -> Self {
392        self.cfg.user_agent = Some(v.into());
393        self
394    }
395    /// Set max concurrency.
396    pub fn max_concurrency(mut self, v: u32) -> Self {
397        self.cfg.max_concurrency = Some(v);
398        self
399    }
400    /// Set rendering delay (ms).
401    pub fn rendering_delay(mut self, v: u32) -> Self {
402        self.cfg.rendering_delay = Some(v);
403        self
404    }
405    /// Honor sitemaps.
406    pub fn use_sitemaps(mut self, v: bool) -> Self {
407        self.cfg.use_sitemaps = v;
408        self
409    }
410    /// Ignore nofollow.
411    pub fn ignore_no_follow(mut self, v: bool) -> Self {
412        self.cfg.ignore_no_follow = v;
413        self
414    }
415    /// Tri-state respect-robots-txt.
416    pub fn respect_robots_txt(mut self, v: bool) -> Self {
417        self.cfg.respect_robots_txt = Some(v);
418        self
419    }
420    /// Enable cache.
421    pub fn cache(mut self, v: bool) -> Self {
422        self.cfg.cache = v;
423        self
424    }
425    /// Cache TTL.
426    pub fn cache_ttl(mut self, v: u32) -> Self {
427        self.cfg.cache_ttl = Some(v);
428        self
429    }
430    /// Force cache refresh.
431    pub fn cache_clear(mut self, v: bool) -> Self {
432        self.cfg.cache_clear = v;
433        self
434    }
435    /// Add content format.
436    pub fn content_format(mut self, v: CrawlerContentFormat) -> Self {
437        self.cfg.content_formats.push(v);
438        self
439    }
440    /// Set extraction rules.
441    pub fn extraction_rules(mut self, v: serde_json::Value) -> Self {
442        self.cfg.extraction_rules = Some(v);
443        self
444    }
445    /// Enable ASP.
446    pub fn asp(mut self, v: bool) -> Self {
447        self.cfg.asp = v;
448        self
449    }
450    /// Set proxy pool name.
451    pub fn proxy_pool(mut self, v: impl Into<String>) -> Self {
452        self.cfg.proxy_pool = Some(v.into());
453        self
454    }
455    /// Set country.
456    pub fn country(mut self, v: impl Into<String>) -> Self {
457        self.cfg.country = Some(v.into());
458        self
459    }
460    /// Set webhook name.
461    pub fn webhook_name(mut self, v: impl Into<String>) -> Self {
462        self.cfg.webhook_name = Some(v.into());
463        self
464    }
465    /// Add webhook event.
466    pub fn webhook_event(mut self, v: CrawlerWebhookEvent) -> Self {
467        self.cfg.webhook_events.push(v);
468        self
469    }
470    /// Finalize the builder.
471    pub fn build(self) -> Result<CrawlerConfig, ScrapflyError> {
472        self.cfg.validate()?;
473        Ok(self.cfg)
474    }
475}
scrapfly_sdk/config/crawler.rs

scrapfly_sdk/config/
crawler.rs
