scrapfly_sdk/result/
scrape.rs

1//! Scrape result — port of `sdk/go/result_scrape.go`.
2//!
3//! Only the fields a typical caller cares about are strongly typed; the
4//! rest land in `serde_json::Value` to survive API contract drift.
5
6use std::collections::BTreeMap;
7
8use serde::Deserialize;
9
10/// Response envelope from `POST /scrape`.
11#[derive(Debug, Clone, Deserialize)]
12pub struct ScrapeResult {
13    /// UUID of the scrape.
14    #[serde(default)]
15    pub uuid: String,
16    /// Echo of the config the server used.
17    #[serde(default)]
18    pub config: serde_json::Value,
19    /// Execution context (cost, proxy, cache…).
20    #[serde(default)]
21    pub context: serde_json::Value,
22    /// Result data.
23    pub result: ResultData,
24}
25
26/// Body of the `result` field.
27#[derive(Debug, Clone, Deserialize, Default)]
28pub struct ResultData {
29    /// Scraped content (HTML, text, markdown…).
30    #[serde(default)]
31    pub content: String,
32    /// Content encoding.
33    #[serde(default)]
34    pub content_encoding: String,
35    /// Content type.
36    #[serde(default)]
37    pub content_type: String,
38    /// Final URL.
39    #[serde(default)]
40    pub url: String,
41    /// HTTP status code from the target.
42    #[serde(default, deserialize_with = "null_as_zero_u16")]
43    pub status_code: u16,
44    /// Status string (`DONE`, etc.).
45    #[serde(default)]
46    pub status: String,
47    /// Scrape success marker.
48    #[serde(default)]
49    pub success: bool,
50    /// Scrape duration (seconds).
51    #[serde(default)]
52    pub duration: f64,
53    /// Format marker (`raw`, `text`, `clob`, `blob`…).
54    #[serde(default)]
55    pub format: String,
56    /// Error envelope for server-side scrape failures.
57    #[serde(default)]
58    pub error: Option<ScrapeErrorDetails>,
59    /// Log URL for the dashboard.
60    #[serde(default)]
61    pub log_url: String,
62    /// Response headers.
63    #[serde(default)]
64    pub response_headers: serde_json::Value,
65    /// Request headers.
66    #[serde(default)]
67    pub request_headers: BTreeMap<String, String>,
68    /// Screenshots captured during this scrape.
69    #[serde(default)]
70    pub screenshots: BTreeMap<String, serde_json::Value>,
71    /// Extracted data (if any).
72    #[serde(default)]
73    pub extracted_data: Option<serde_json::Value>,
74    /// Browser data (local/session storage, attachments…).
75    #[serde(default)]
76    pub browser_data: serde_json::Value,
77    /// Iframes discovered during a rendered scrape. Each entry has at least
78    /// a `url` field pointing at the embedded document. Stored as
79    /// `serde_json::Value` because the exact shape varies with the engine
80    /// version (url, url_discovered_as, navigation_id, …).
81    #[serde(default)]
82    pub iframes: serde_json::Value,
83}
84
85/// Inner error details.
86#[derive(Debug, Clone, Deserialize, Default)]
87pub struct ScrapeErrorDetails {
88    /// Error code.
89    #[serde(default)]
90    pub code: String,
91    /// HTTP code.
92    #[serde(default, deserialize_with = "null_as_zero_u16")]
93    pub http_code: u16,
94    /// Error message.
95    #[serde(default)]
96    pub message: String,
97    /// Documentation URL.
98    #[serde(default)]
99    pub doc_url: String,
100    /// Retryable flag.
101    #[serde(default)]
102    pub retryable: bool,
103}
104
105/// Coerce JSON `null | absent | number` into a plain `u16`, where null and
106/// absent both collapse to zero. The server emits `status_code: null` and
107/// `http_code: null` on large-object (blob/clob) offload responses, where
108/// the real status lives in the signed-URL payload rather than the envelope.
109/// Without this, the plain `u16` deserializer rejects `null` and the SDK
110/// can't decode a successful large_object batch part.
111fn null_as_zero_u16<'de, D>(deserializer: D) -> Result<u16, D::Error>
112where
113    D: serde::Deserializer<'de>,
114{
115    Ok(Option::<u16>::deserialize(deserializer)?.unwrap_or(0))
116}
scrapfly_sdk/result/scrape.rs

scrapfly_sdk/result/
scrape.rs
