scrapfly_sdk/config/
scrape.rs

1//! Scrape endpoint configuration — ported from `sdk/go/config_scrape.go`.
2
3use std::collections::BTreeMap;
4
5use crate::enums::{ExtractionModel, Format, FormatOption, HttpMethod, ProxyPool, ScreenshotFlag};
6use crate::error::ScrapflyError;
7
8use super::url_safe_b64_encode;
9
10/// Configuration for a single `POST /scrape` request.
11///
12/// Construct via [`ScrapeConfig::builder`].
13#[derive(Debug, Clone, Default)]
14pub struct ScrapeConfig {
15    /// Target URL (required).
16    pub url: String,
17    /// HTTP method; defaults to `GET`.
18    pub method: Option<HttpMethod>,
19    /// Raw request body.
20    pub body: Option<String>,
21    /// Request headers (sent as `headers[key]=value`).
22    pub headers: BTreeMap<String, String>,
23    /// Cookies (merged into `headers[cookie]`).
24    pub cookies: BTreeMap<String, String>,
25    /// Proxy country.
26    pub country: Option<String>,
27    /// Proxy pool.
28    pub proxy_pool: Option<ProxyPool>,
29    /// Enable JavaScript rendering.
30    pub render_js: bool,
31    /// Enable Anti-Scraping Protection bypass.
32    pub asp: bool,
33    /// Enable cache.
34    pub cache: bool,
35    /// Cache TTL (seconds).
36    pub cache_ttl: Option<u32>,
37    /// Force cache refresh.
38    pub cache_clear: bool,
39    /// Timeout in milliseconds.
40    pub timeout: Option<u32>,
41    /// Maximum API credit cost the caller is willing to spend on this
42    /// request. If the server's pre-flight estimate exceeds this budget the
43    /// request is rejected before execution.
44    pub cost_budget: Option<u32>,
45    /// Enable automatic retries.
46    pub retry: Option<bool>,
47    /// Session name.
48    pub session: Option<String>,
49    /// Sticky-proxy inside the session.
50    pub session_sticky_proxy: bool,
51    /// Custom tags.
52    pub tags: Vec<String>,
53    /// Webhook name.
54    pub webhook: Option<String>,
55    /// Debug mode.
56    pub debug: bool,
57    /// Capture SSL details.
58    pub ssl: bool,
59    /// Capture DNS details.
60    pub dns: bool,
61    /// Correlation ID.
62    pub correlation_id: Option<String>,
63    /// Output format.
64    pub format: Option<Format>,
65    /// Format options.
66    pub format_options: Vec<FormatOption>,
67    /// Saved extraction template name.
68    pub extraction_template: Option<String>,
69    /// Inline (ephemeral) extraction template as JSON value.
70    pub extraction_ephemeral_template: Option<serde_json::Value>,
71    /// AI extraction prompt.
72    pub extraction_prompt: Option<String>,
73    /// Extraction model.
74    pub extraction_model: Option<ExtractionModel>,
75    /// Wait for CSS selector (requires `render_js`).
76    pub wait_for_selector: Option<String>,
77    /// Extra wait after page load, milliseconds.
78    pub rendering_wait: Option<u32>,
79    /// Auto-scroll to load lazy content.
80    pub auto_scroll: bool,
81    /// Named screenshots (name → selector, or "fullpage").
82    pub screenshots: BTreeMap<String, String>,
83    /// Screenshot flags.
84    pub screenshot_flags: Vec<ScreenshotFlag>,
85    /// Inline JavaScript code (base64url-encoded on the wire).
86    pub js: Option<String>,
87    /// JS scenario (serialized as JSON then base64url-encoded).
88    pub js_scenario: Option<serde_json::Value>,
89    /// OS fingerprint hint.
90    pub os: Option<String>,
91    /// Accept-Language values.
92    pub lang: Vec<String>,
93    /// Browser brand (`chrome` | `edge` | `brave` | `opera`).
94    pub browser_brand: Option<String>,
95    /// Spoof browser geolocation. Format: `"latitude,longitude"`.
96    pub geolocation: Option<String>,
97    /// Page load stage to wait for. `complete` (default) or `domcontentloaded`.
98    pub rendering_stage: Option<String>,
99    /// Return the raw upstream response instead of the JSON envelope.
100    /// When true, callers must use `Client::scrape_proxified()` which
101    /// returns `reqwest::Response` directly.
102    pub proxified_response: bool,
103}
104
105impl ScrapeConfig {
106    /// Start a builder for `url`.
107    pub fn builder(url: impl Into<String>) -> ScrapeConfigBuilder {
108        ScrapeConfigBuilder {
109            cfg: ScrapeConfig {
110                url: url.into(),
111                // Sticky proxy is on by default (matches the API): a session
112                // keeps one exit IP unless the caller opts out.
113                session_sticky_proxy: true,
114                ..Default::default()
115            },
116        }
117    }
118
119    /// Serialize the config into the query-parameter pairs that the
120    /// `/scrape` endpoint expects. Mirrors
121    /// `sdk/go/config_scrape.go::toAPIParamsWithValidation`.
122    pub fn to_query_pairs(&self) -> Result<Vec<(String, String)>, ScrapflyError> {
123        if self.url.is_empty() {
124            return Err(ScrapflyError::Config("url is required".into()));
125        }
126
127        let mut out: Vec<(String, String)> = Vec::new();
128        out.push(("url".into(), self.url.clone()));
129
130        if let Some(country) = &self.country {
131            out.push(("country".into(), country.to_lowercase()));
132        }
133        if let Some(pool) = &self.proxy_pool {
134            out.push(("proxy_pool".into(), pool.as_str().into()));
135        }
136
137        if self.render_js {
138            out.push(("render_js".into(), "true".into()));
139            if let Some(sel) = &self.wait_for_selector {
140                out.push(("wait_for_selector".into(), sel.clone()));
141            }
142            if let Some(wait) = self.rendering_wait {
143                out.push(("rendering_wait".into(), wait.to_string()));
144            }
145            if let Some(ref geo) = self.geolocation {
146                out.push(("geolocation".into(), geo.clone()));
147            }
148            if let Some(ref stage) = self.rendering_stage {
149                if stage != "complete" {
150                    out.push(("rendering_stage".into(), stage.clone()));
151                }
152            }
153            if self.auto_scroll {
154                out.push(("auto_scroll".into(), "true".into()));
155            }
156            if let Some(js) = &self.js {
157                out.push(("js".into(), url_safe_b64_encode(js)));
158            }
159            if let Some(sc) = &self.js_scenario {
160                let as_str = serde_json::to_string(sc)?;
161                out.push(("js_scenario".into(), url_safe_b64_encode(&as_str)));
162            }
163            for (name, value) in &self.screenshots {
164                if value.is_empty() {
165                    return Err(ScrapflyError::Config(format!(
166                        "screenshots[{}] requires either a selector or 'fullpage'",
167                        name
168                    )));
169                }
170                out.push((format!("screenshots[{}]", name), value.clone()));
171            }
172            if !self.screenshot_flags.is_empty() {
173                let joined = self
174                    .screenshot_flags
175                    .iter()
176                    .map(|f| f.as_str())
177                    .collect::<Vec<_>>()
178                    .join(",");
179                out.push(("screenshot_flags".into(), joined));
180            }
181        }
182
183        if self.asp {
184            out.push(("asp".into(), "true".into()));
185        }
186        if self.retry == Some(false) {
187            out.push(("retry".into(), "false".into()));
188        }
189        if self.cache {
190            out.push(("cache".into(), "true".into()));
191            if let Some(ttl) = self.cache_ttl {
192                out.push(("cache_ttl".into(), ttl.to_string()));
193            }
194            if self.cache_clear {
195                out.push(("cache_clear".into(), "true".into()));
196            }
197        }
198        if let Some(timeout) = self.timeout {
199            out.push(("timeout".into(), timeout.to_string()));
200        }
201        if let Some(budget) = self.cost_budget {
202            out.push(("cost_budget".into(), budget.to_string()));
203        }
204        if self.debug {
205            out.push(("debug".into(), "true".into()));
206        }
207        if self.ssl {
208            out.push(("ssl".into(), "true".into()));
209        }
210        if self.dns {
211            out.push(("dns".into(), "true".into()));
212        }
213        if let Some(cid) = &self.correlation_id {
214            out.push(("correlation_id".into(), cid.clone()));
215        }
216        if !self.tags.is_empty() {
217            out.push(("tags".into(), self.tags.join(",")));
218        }
219        if let Some(wh) = &self.webhook {
220            out.push(("webhook_name".into(), wh.clone()));
221        }
222        if let Some(session) = &self.session {
223            out.push(("session".into(), session.clone()));
224            out.push((
225                "session_sticky_proxy".into(),
226                self.session_sticky_proxy.to_string(),
227            ));
228        }
229        if let Some(os) = &self.os {
230            out.push(("os".into(), os.clone()));
231        }
232        if !self.lang.is_empty() {
233            out.push(("lang".into(), self.lang.join(",")));
234        }
235        if let Some(bb) = &self.browser_brand {
236            out.push(("browser_brand".into(), bb.clone()));
237        }
238        if self.proxified_response {
239            out.push(("proxified_response".into(), "true".into()));
240        }
241
242        if let Some(format) = self.format {
243            let mut val = format.as_str().to_string();
244            if !self.format_options.is_empty() {
245                val.push(':');
246                val.push_str(
247                    &self
248                        .format_options
249                        .iter()
250                        .map(|f| f.as_str())
251                        .collect::<Vec<_>>()
252                        .join(","),
253                );
254            }
255            out.push(("format".into(), val));
256        }
257
258        // Extraction — exclusivity enforced by builder.
259        if let Some(tpl) = &self.extraction_template {
260            out.push(("extraction_template".into(), format!("persistent:{}", tpl)));
261        } else if let Some(tpl) = &self.extraction_ephemeral_template {
262            let as_str = serde_json::to_string(tpl)?;
263            out.push((
264                "extraction_template".into(),
265                format!("ephemeral:{}", url_safe_b64_encode(&as_str)),
266            ));
267        } else if let Some(prompt) = &self.extraction_prompt {
268            out.push(("extraction_prompt".into(), prompt.clone()));
269        } else if let Some(model) = self.extraction_model {
270            out.push(("extraction_model".into(), model.as_str().into()));
271        }
272
273        // Headers → `headers[key]=value`.
274        for (k, v) in &self.headers {
275            if k.is_empty() || v.is_empty() {
276                return Err(ScrapflyError::Config(
277                    "headers key and value cannot be empty".into(),
278                ));
279            }
280            out.push((format!("headers[{}]", k.to_lowercase()), v.clone()));
281        }
282
283        // Cookies merged into `headers[cookie]`.
284        if !self.cookies.is_empty() {
285            let cookie_str = self
286                .cookies
287                .iter()
288                .map(|(k, v)| format!("{}={}", k, v))
289                .collect::<Vec<_>>()
290                .join("; ");
291            // Check for any existing `headers[cookie]` from above.
292            let existing = out
293                .iter()
294                .position(|(k, _)| k.eq_ignore_ascii_case("headers[cookie]"));
295            match existing {
296                Some(idx) => {
297                    let existing_val = out[idx].1.clone();
298                    out[idx].1 = format!("{}; {}", existing_val, cookie_str);
299                }
300                None => out.push(("headers[cookie]".into(), cookie_str)),
301            }
302        }
303
304        Ok(out)
305    }
306}
307
308/// Builder for [`ScrapeConfig`].
309#[derive(Debug, Clone)]
310pub struct ScrapeConfigBuilder {
311    cfg: ScrapeConfig,
312}
313
314impl ScrapeConfigBuilder {
315    /// Set HTTP method.
316    pub fn method(mut self, m: HttpMethod) -> Self {
317        self.cfg.method = Some(m);
318        self
319    }
320    /// Set raw request body.
321    pub fn body(mut self, b: impl Into<String>) -> Self {
322        self.cfg.body = Some(b.into());
323        self
324    }
325    /// Set a custom header.
326    pub fn header(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
327        self.cfg.headers.insert(k.into(), v.into());
328        self
329    }
330    /// Replace the full header map.
331    pub fn headers(mut self, headers: BTreeMap<String, String>) -> Self {
332        self.cfg.headers = headers;
333        self
334    }
335    /// Set a cookie.
336    pub fn cookie(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
337        self.cfg.cookies.insert(k.into(), v.into());
338        self
339    }
340    /// Set proxy country.
341    pub fn country(mut self, c: impl Into<String>) -> Self {
342        self.cfg.country = Some(c.into());
343        self
344    }
345    /// Set proxy pool.
346    pub fn proxy_pool(mut self, p: ProxyPool) -> Self {
347        self.cfg.proxy_pool = Some(p);
348        self
349    }
350    /// Enable JS rendering.
351    pub fn render_js(mut self, v: bool) -> Self {
352        self.cfg.render_js = v;
353        self
354    }
355    /// Enable ASP bypass.
356    pub fn asp(mut self, v: bool) -> Self {
357        self.cfg.asp = v;
358        self
359    }
360    /// Enable cache.
361    pub fn cache(mut self, v: bool) -> Self {
362        self.cfg.cache = v;
363        self
364    }
365    /// Set cache TTL.
366    pub fn cache_ttl(mut self, v: u32) -> Self {
367        self.cfg.cache_ttl = Some(v);
368        self
369    }
370    /// Force cache refresh.
371    pub fn cache_clear(mut self, v: bool) -> Self {
372        self.cfg.cache_clear = v;
373        self
374    }
375    /// Set the maximum API credit cost the caller will accept for this
376    /// request. The server rejects the request pre-flight if its estimate
377    /// exceeds the budget, so callers get a fast failure instead of a
378    /// surprise bill.
379    pub fn cost_budget(mut self, v: u32) -> Self {
380        self.cfg.cost_budget = Some(v);
381        self
382    }
383    /// Set request timeout (ms).
384    pub fn timeout(mut self, v: u32) -> Self {
385        self.cfg.timeout = Some(v);
386        self
387    }
388    /// Set automatic retry flag.
389    pub fn retry(mut self, v: bool) -> Self {
390        self.cfg.retry = Some(v);
391        self
392    }
393    /// Set session name.
394    pub fn session(mut self, v: impl Into<String>) -> Self {
395        self.cfg.session = Some(v.into());
396        self
397    }
398    /// Sticky proxy in a session.
399    pub fn session_sticky_proxy(mut self, v: bool) -> Self {
400        self.cfg.session_sticky_proxy = v;
401        self
402    }
403    /// Add a tag.
404    pub fn tag(mut self, v: impl Into<String>) -> Self {
405        self.cfg.tags.push(v.into());
406        self
407    }
408    /// Set all tags.
409    pub fn tags(mut self, v: Vec<String>) -> Self {
410        self.cfg.tags = v;
411        self
412    }
413    /// Set webhook name.
414    pub fn webhook(mut self, v: impl Into<String>) -> Self {
415        self.cfg.webhook = Some(v.into());
416        self
417    }
418    /// Enable debug mode.
419    pub fn debug(mut self, v: bool) -> Self {
420        self.cfg.debug = v;
421        self
422    }
423    /// Capture SSL details.
424    pub fn ssl(mut self, v: bool) -> Self {
425        self.cfg.ssl = v;
426        self
427    }
428    /// Capture DNS details.
429    pub fn dns(mut self, v: bool) -> Self {
430        self.cfg.dns = v;
431        self
432    }
433    /// Set correlation ID.
434    pub fn correlation_id(mut self, v: impl Into<String>) -> Self {
435        self.cfg.correlation_id = Some(v.into());
436        self
437    }
438    /// Set output format.
439    pub fn format(mut self, v: Format) -> Self {
440        self.cfg.format = Some(v);
441        self
442    }
443    /// Add a format option.
444    pub fn format_option(mut self, v: FormatOption) -> Self {
445        self.cfg.format_options.push(v);
446        self
447    }
448    /// Set saved extraction template name.
449    pub fn extraction_template(mut self, v: impl Into<String>) -> Self {
450        self.cfg.extraction_template = Some(v.into());
451        self
452    }
453    /// Set inline extraction template.
454    pub fn extraction_ephemeral_template(mut self, v: serde_json::Value) -> Self {
455        self.cfg.extraction_ephemeral_template = Some(v);
456        self
457    }
458    /// Set AI extraction prompt.
459    pub fn extraction_prompt(mut self, v: impl Into<String>) -> Self {
460        self.cfg.extraction_prompt = Some(v.into());
461        self
462    }
463    /// Set extraction model.
464    pub fn extraction_model(mut self, v: ExtractionModel) -> Self {
465        self.cfg.extraction_model = Some(v);
466        self
467    }
468    /// Set wait-for-selector.
469    pub fn wait_for_selector(mut self, v: impl Into<String>) -> Self {
470        self.cfg.wait_for_selector = Some(v.into());
471        self
472    }
473    /// Set extra rendering wait (ms).
474    pub fn rendering_wait(mut self, v: u32) -> Self {
475        self.cfg.rendering_wait = Some(v);
476        self
477    }
478    /// Enable auto-scroll.
479    pub fn auto_scroll(mut self, v: bool) -> Self {
480        self.cfg.auto_scroll = v;
481        self
482    }
483    /// Add a named screenshot.
484    pub fn screenshot(mut self, name: impl Into<String>, selector: impl Into<String>) -> Self {
485        self.cfg.screenshots.insert(name.into(), selector.into());
486        self
487    }
488    /// Add a screenshot flag.
489    pub fn screenshot_flag(mut self, v: ScreenshotFlag) -> Self {
490        self.cfg.screenshot_flags.push(v);
491        self
492    }
493    /// Set inline JS code.
494    pub fn js(mut self, v: impl Into<String>) -> Self {
495        self.cfg.js = Some(v.into());
496        self
497    }
498    /// Set JS scenario (as serde_json::Value).
499    pub fn js_scenario(mut self, v: serde_json::Value) -> Self {
500        self.cfg.js_scenario = Some(v);
501        self
502    }
503    /// Set OS fingerprint hint.
504    pub fn os(mut self, v: impl Into<String>) -> Self {
505        self.cfg.os = Some(v.into());
506        self
507    }
508    /// Add an Accept-Language value.
509    pub fn lang(mut self, v: impl Into<String>) -> Self {
510        self.cfg.lang.push(v.into());
511        self
512    }
513    /// Set browser brand.
514    pub fn browser_brand(mut self, v: impl Into<String>) -> Self {
515        self.cfg.browser_brand = Some(v.into());
516        self
517    }
518    /// Enable proxified response mode (raw upstream pass-through).
519    /// Spoof browser geolocation. Format: `"latitude,longitude"`.
520    pub fn geolocation(mut self, v: impl Into<String>) -> Self {
521        self.cfg.geolocation = Some(v.into());
522        self
523    }
524    /// Set page load stage: `"complete"` (default) or `"domcontentloaded"`.
525    pub fn rendering_stage(mut self, v: impl Into<String>) -> Self {
526        self.cfg.rendering_stage = Some(v.into());
527        self
528    }
529    /// Enable proxified response mode (raw upstream pass-through).
530    pub fn proxified_response(mut self) -> Self {
531        self.cfg.proxified_response = true;
532        self
533    }
534
535    /// Finalize the builder, enforcing the mutual-exclusion rules for the
536    /// extraction fields.
537    pub fn build(self) -> Result<ScrapeConfig, ScrapflyError> {
538        let cfg = self.cfg;
539        let count = [
540            cfg.extraction_template.is_some(),
541            cfg.extraction_ephemeral_template.is_some(),
542            cfg.extraction_prompt.is_some(),
543            cfg.extraction_model.is_some(),
544        ]
545        .iter()
546        .filter(|x| **x)
547        .count();
548        if count > 1 {
549            return Err(ScrapflyError::Config(
550                "extraction_template, extraction_ephemeral_template, extraction_prompt and extraction_model are mutually exclusive"
551                    .into(),
552            ));
553        }
554        Ok(cfg)
555    }
556}
scrapfly_sdk/config/scrape.rs

scrapfly_sdk/config/
scrape.rs
