From 4c8ff090e3314d9a46cc3f5100fe7dcf8d3b056c Mon Sep 17 00:00:00 2001 From: Paolo Antinori Date: Sat, 10 Oct 2026 12:54:01 +0200 Subject: [PATCH 1/3] feat(renderer): fresh impersonated session per hop attempt The hop is a cold retry path, so it now builds a use-once ImpersonatedFetcher per attempt: new TLS session, no pooled identity. A persistent impersonated session is exactly what lets a wall vendor build per-fingerprint reputation and quietly stop serving it, the failure pywaze paid to learn (eifinger/pywaze#110: a pinned impersonated session was re-blocked in six days; #111 fixed it with a fresh session per fallback request). The pin arm keeps the warm pooled client, where pooling pays. REQUEST_PROXY stays honored fail-closed through the same with_proxy constructor. The builder is a factory field (type ImpersonatedHopFactory) so tests can inject a counter: hop_builds_a_fresh_session_per_attempt asserts exactly one construction per hop attempt, clean_site_builds_no_hop_session asserts none on clean sites. Gates: /simplify, code-review --- crates/crw-renderer/src/impersonated.rs | 7 +- crates/crw-renderer/src/lib.rs | 179 ++++++++++++++++++++++-- 2 files changed, 170 insertions(+), 16 deletions(-) diff --git a/crates/crw-renderer/src/impersonated.rs b/crates/crw-renderer/src/impersonated.rs index 568422c7..7fa86194 100644 --- a/crates/crw-renderer/src/impersonated.rs +++ b/crates/crw-renderer/src/impersonated.rs @@ -6,8 +6,11 @@ //! * the auto-chain hop between the plain reqwest HTTP tier and the browser //! ladder, fired only when the plain tier hit a wall-shaped response (the //! Amazon interstitial class) or a transport error, never on SPA/thin/empty -//! shapes that need JS; -//! * the explicit `renderer = "impersonated-http"` pin. +//! shapes that need JS. The hop runs on a FRESH client per attempt (see +//! `FallbackRenderer::impersonated_hop_factory`): no pooled impersonated +//! identity for a wall vendor to build per-fingerprint reputation on; +//! * the explicit `renderer = "impersonated-http"` pin, which uses the warm +//! pooled client: deliberate bulk use is where pooling pays. //! //! Whole module is `#[cfg(feature = "impersonated")]`: a lean build never //! links the wreq/btls TLS stack. diff --git a/crates/crw-renderer/src/lib.rs b/crates/crw-renderer/src/lib.rs index eb73864f..3956563f 100644 --- a/crates/crw-renderer/src/lib.rs +++ b/crates/crw-renderer/src/lib.rs @@ -631,6 +631,12 @@ fn is_hard_block_status(status_code: u16) -> bool { /// where config pools are small but the space of raw proxy URLs is not. const PROXY_CLIENT_CACHE_CAP: usize = 512; +/// Builds the use-once fetcher for one hop attempt. The argument is the +/// resolved REQUEST_PROXY raw URL, if any (None fetches direct). +#[cfg(feature = "impersonated")] +type ImpersonatedHopFactory = + Arc) -> CrwResult> + Send + Sync>; + /// Why the impersonated hop fired; label for logs and the route-decision /// metric. Kept as an enum (not a bool) so the Failover reason mapping cannot /// conflate a wall with a transport failure. @@ -828,6 +834,17 @@ pub struct FallbackRenderer { /// Saved impersonated-tier timeout for the per-proxy cache. #[cfg(feature = "impersonated")] impersonated_timeout_ms: u64, + /// Builds the use-once fetcher for each hop attempt. The hop is a cold + /// retry path, so it gets a FRESH client per attempt: a new TLS session and + /// no pooled identity. A persistent impersonated session lets a wall vendor + /// build per-fingerprint reputation and quietly stop serving it, the + /// failure pywaze paid to learn (eifinger/pywaze#110: a pinned impersonated + /// session was re-blocked in six days; #111 fixed it with a fresh session + /// per fallback request). The pin arm keeps the warm pooled client: bulk + /// deliberate use is exactly where pooling pays. A factory field rather + /// than a method so tests can inject a counter and assert freshness. + #[cfg(feature = "impersonated")] + impersonated_hop_factory: ImpersonatedHopFactory, } impl std::fmt::Debug for FallbackRenderer { @@ -929,6 +946,22 @@ impl FallbackRenderer { None }; + // The hop's use-once client builder (see the field's doc). Built in + // every feature build; inert until a hop actually runs. + #[cfg(feature = "impersonated")] + let impersonated_hop_factory: ImpersonatedHopFactory = { + let timeout = std::time::Duration::from_millis(config.impersonated_timeout()); + Arc::new(move |proxy| { + let fetcher = match proxy { + // Fail-closed like the warm path: a bad proxy URL is a hard + // error, never a silent direct connection. + Some(raw) => impersonated::ImpersonatedFetcher::with_proxy(raw, timeout)?, + None => impersonated::ImpersonatedFetcher::new(timeout)?, + }; + Ok(Arc::new(fetcher) as Arc) + }) + }; + // A pinned backend (Lightpanda/Chrome/Playwright) must have CDP compiled in // AND its matching endpoint configured. `Auto` and `None` remain functional // without CDP — they just won't spawn any JS renderer. @@ -1022,6 +1055,8 @@ impl FallbackRenderer { impersonated_client_cache: std::sync::Mutex::new(std::collections::HashMap::new()), #[cfg(feature = "impersonated")] impersonated_timeout_ms: config.impersonated_timeout(), + #[cfg(feature = "impersonated")] + impersonated_hop_factory, }); } @@ -1337,6 +1372,8 @@ impl FallbackRenderer { impersonated_client_cache: std::sync::Mutex::new(std::collections::HashMap::new()), #[cfg(feature = "impersonated")] impersonated_timeout_ms: config.impersonated_timeout(), + #[cfg(feature = "impersonated")] + impersonated_hop_factory, }) } @@ -1401,8 +1438,8 @@ impl FallbackRenderer { Ok(fetcher) } - /// REQUEST_PROXY-aware impersonated fetcher lookup, mirroring - /// [`Self::http_fetcher_for_request`]: shared tier when no proxy is set, + /// REQUEST_PROXY-aware impersonated fetcher lookup for the PIN arm, + /// mirroring [`Self::http_fetcher_for_request`]: shared tier when no proxy is set, /// warm per-proxy client built with `ImpersonatedFetcher::with_proxy` /// (fail-closed) when one is. When no REQUEST_PROXY is set the shared /// client falls back to wreq's system-proxy matcher, which reads @@ -1549,7 +1586,15 @@ impl FallbackRenderer { // Every metric site in this function shares this label source, so a // typo'd literal cannot fork the series. let kind = RendererKind::ImpersonatedHttp; - let fetcher = match self.impersonated_fetcher_for_request() { + // Fresh, use-once client for this attempt (see the factory field's + // doc): no pooled identity for the wall to learn across hops. + // REQUEST_PROXY is honored fail-closed through the same with_proxy + // constructor the warm path uses. + let proxy_raw = REQUEST_PROXY + .try_with(|p| p.as_ref().map(|e| e.raw().to_string())) + .ok() + .flatten(); + let fetcher = match (self.impersonated_hop_factory)(proxy_raw.as_deref()) { Ok(f) => f, Err(e) => { tracing::warn!( @@ -5851,6 +5896,15 @@ mod tests { assert!(res.html.contains("rendered")); } + /// Inject a mock impersonated tier the way the routing tests need: the + /// presence/pin path reads `impersonated`, the hop takes its fetcher from + /// the factory, so the mock must sit in both. + #[cfg(feature = "impersonated")] + fn inject_impersonated(r: &mut FallbackRenderer, tier: Arc) { + r.impersonated = Some(tier.clone()); + r.impersonated_hop_factory = Arc::new(move |_| Ok(tier.clone())); + } + fn make_renderer_with_mocks(mocks: Vec>) -> FallbackRenderer { // Builds a REAL HTTP fetcher. The forced-JS arm fetches HTTP before the // ladder (content-type check), so any test that exercises it should @@ -6015,7 +6069,7 @@ mod tests { }) as Arc; let mut r = make_renderer_with_mocks(vec![chrome]); r.http = http; - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let result = r .fetch( @@ -6056,7 +6110,7 @@ mod tests { }) as Arc; let mut r = make_renderer_with_mocks(Vec::new()); r.http = http; - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let result = r .fetch( @@ -6078,6 +6132,103 @@ mod tests { ); } + /// The hop builds a FRESH client per attempt: a pooled impersonated + /// identity is exactly what lets a wall vendor accumulate per-fingerprint + /// reputation (eifinger/pywaze#110, fixed by a fresh session per fallback + /// in #111). The counting factory proves one construction per hop attempt. + #[cfg(feature = "impersonated")] + #[tokio::test] + async fn hop_builds_a_fresh_session_per_attempt() { + let http = Arc::new(MockFetcher { + name: "http", + behavior: MockBehavior::OkStatus(200, amazon_interstitial_html()), + }) as Arc; + // The hop attempt misses (the impersonated client gets the same wall), + // so the ladder must finish the job. + let impersonated = Arc::new(MockFetcher { + name: "impersonated-http", + behavior: MockBehavior::OkStatus(200, amazon_interstitial_html()), + }) as Arc; + let chrome = Arc::new(MockFetcher { + name: "chrome", + behavior: MockBehavior::Ok(rich_html("LADDER-")), + }) as Arc; + let mut r = make_renderer_with_mocks(vec![chrome]); + r.http = http; + let built = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let counter = built.clone(); + let tier = impersonated.clone(); + r.impersonated = Some(tier.clone()); + r.impersonated_hop_factory = Arc::new(move |_| { + counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(tier.clone()) + }); + + let result = r + .fetch( + "https://www.amazon.it/dp/B0FHQGLXBP", + &HashMap::new(), + None, + None, + None, + tdl(), + ) + .await + .unwrap(); + assert_eq!( + result.rendered_with.as_deref(), + Some("chrome"), + "a hop that still sees the wall must fall through to the ladder" + ); + assert_eq!( + built.load(std::sync::atomic::Ordering::SeqCst), + 1, + "exactly one fresh session per hop attempt" + ); + } + + /// Clean sites never construct a hop client at all: no wall, no session. + #[cfg(feature = "impersonated")] + #[tokio::test] + async fn clean_site_builds_no_hop_session() { + let http = Arc::new(MockFetcher { + name: "http", + behavior: MockBehavior::Ok(rich_html("CLEAN-")), + }) as Arc; + let impersonated = Arc::new(MockFetcher { + name: "impersonated-http", + behavior: MockBehavior::Panic("no hop may run on a clean site"), + }) as Arc; + let mut r = make_renderer_with_mocks(Vec::new()); + r.http = http; + let built = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let counter = built.clone(); + let tier = impersonated.clone(); + r.impersonated = Some(tier.clone()); + r.impersonated_hop_factory = Arc::new(move |_| { + counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(tier.clone()) + }); + + let result = r + .fetch( + "https://example.com", + &HashMap::new(), + None, + None, + None, + tdl(), + ) + .await + .unwrap(); + assert_eq!(result.rendered_with.as_deref(), Some("http")); + assert_eq!( + built.load(std::sync::atomic::Ordering::SeqCst), + 0, + "no hop session is built when the plain tier succeeds" + ); + } + #[cfg(feature = "impersonated")] #[tokio::test] async fn fetch_auto_impersonated_still_blocked_continues_to_ladder() { @@ -6095,7 +6246,7 @@ mod tests { }) as Arc; let mut r = make_renderer_with_mocks(vec![chrome]); r.http = http; - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let result = r .fetch( @@ -6134,7 +6285,7 @@ mod tests { }) as Arc; let mut r = make_renderer_with_mocks(vec![lp]); r.http = http; - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let result = r .fetch( @@ -6184,7 +6335,7 @@ mod tests { }) as Arc; let mut r = make_renderer_with_mocks(vec![chrome]); r.http = http; - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let result = r .fetch( @@ -6219,7 +6370,7 @@ mod tests { ), }) as Arc; let mut r = make_renderer_with_mocks(Vec::new()); - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let result = r .fetch( @@ -6281,7 +6432,7 @@ mod tests { behavior: MockBehavior::OkStatus(404, rich_html("GONE-")), }) as Arc; let mut r = make_renderer_with_mocks(Vec::new()); - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let result = r .fetch( @@ -6307,7 +6458,7 @@ mod tests { behavior: MockBehavior::OkStatus(403, rich_html("DENIED-")), }) as Arc; let mut r = make_renderer_with_mocks(Vec::new()); - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let res = r .fetch( @@ -6340,7 +6491,7 @@ mod tests { }) as Arc; let mut r = make_renderer_with_mocks(Vec::new()); r.http = http; - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let result = r .fetch( @@ -6399,7 +6550,7 @@ mod tests { behavior: MockBehavior::Panic("a set proxy must route the pin, never the shared tier"), }) as Arc; let mut r = make_renderer_with_mocks(Vec::new()); - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); // A syntactically valid but unreachable proxy (loopback port 1): // the pin must error on the dead proxy rather than fall back to a // direct egress. The malformed-URL case is covered by the module's @@ -6434,7 +6585,7 @@ mod tests { behavior: MockBehavior::OkStatus(200, amazon_interstitial_html()), }) as Arc; let mut r = make_renderer_with_mocks(Vec::new()); - r.impersonated = Some(impersonated); + inject_impersonated(&mut r, impersonated); let res = r .fetch( From bd15a262e420f9563283acece10918da4abd0434 Mon Sep 17 00:00:00 2001 From: Paolo Antinori Date: Sun, 11 Oct 2026 09:51:10 +0200 Subject: [PATCH 2/3] test(renderer): sentinel for the preset's ClientHello shape Asserts the pinned Chrome149 preset carries the post-quantum key share (X25519MLKEM768) in both supported_groups and the actual key_share, plus an HTTP/2 fingerprint hash, by probing a fingerprint-echo endpoint (tls.peet.ws/api/all). A preset that claims Chrome 149 without the PQ share is detectable by post-quantum-aware walls before any HTTP traffic; when this test goes red the answer is a preset bump, not a code fix. Deterministic: reads the handshake shape, not a wall that may not fire. Gates: /simplify, code-review --- crates/crw-renderer/src/impersonated.rs | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/crates/crw-renderer/src/impersonated.rs b/crates/crw-renderer/src/impersonated.rs index 7fa86194..02c4e5ef 100644 --- a/crates/crw-renderer/src/impersonated.rs +++ b/crates/crw-renderer/src/impersonated.rs @@ -672,4 +672,58 @@ mod tests { other => panic!("unexpected rendered_with: {other:?}"), } } + + /// Sentinel: the pinned preset's ClientHello must carry the post-quantum + /// key share real Chrome sends by default since 131 (X25519MLKEM768), and + /// an HTTP/2 fingerprint. A preset that claims Chrome 149 without the PQ + /// share is detectable by post-quantum-aware walls (Akamai flags it + /// before any HTTP traffic) and the preset needs a bump, not a code fix. + /// Deterministic: reads the shape from a fingerprint-echo endpoint, not + /// from a wall that may or may not fire. + #[tokio::test] + #[ignore] + async fn live_preset_clienthello_carries_postquantum_and_h2() { + let f = fetcher().unwrap(); + let r = f + .fetch( + "https://tls.peet.ws/api/all", + &HashMap::new(), + None, + Deadline::from_request_ms(30_000), + ) + .await + .unwrap_or_else(|e| panic!("fingerprint probe failed: {e}")); + assert_eq!(r.status_code, 200, "fingerprint endpoint must answer"); + let v: serde_json::Value = + serde_json::from_str(&r.html).expect("fingerprint endpoint returns JSON"); + + let tls = &v["tls"]; + // Post-quantum key share: both in the offered groups AND actually sent. + let groups = tls["extensions"] + .as_array() + .into_iter() + .flatten() + .find(|e| e["name"].as_str().unwrap_or("").contains("supported_groups")) + .and_then(|e| e["supported_groups"].as_array().cloned()) + .unwrap_or_default(); + let group_names: Vec = groups + .iter() + .filter_map(|g| g.as_str().map(|s| s.to_string())) + .collect(); + assert!( + group_names.iter().any(|g| g.contains("X25519MLKEM768")), + "supported_groups must offer the post-quantum group; got {group_names:?}" + ); + let key_share_str = tls["extensions"].to_string(); + assert!( + key_share_str.contains("X25519MLKEM768"), + "key_share must actually carry the ML-KEM share, not just offer the group" + ); + + // HTTP/2 fingerprint present (the Akamai SETTINGS/pseudo-header shape). + let h2 = v["http2"]["akamai_fingerprint_hash"] + .as_str() + .unwrap_or(""); + assert!(!h2.is_empty(), "HTTP/2 fingerprint hash must be reported"); + } } From 6764e06dc7a13fc7f4f85bcc5200b453c9a70afe Mon Sep 17 00:00:00 2001 From: Paolo Antinori Date: Fri, 16 Oct 2026 21:14:36 +0200 Subject: [PATCH 3/3] chore: retrigger CI (rustup network flake on the check job)