diff --git a/.cargo/config.toml b/.cargo/config.toml index 1302091e0..f8fcfadaf 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -64,6 +64,13 @@ run_cli_macos = "run --package trusted-server-cli --target aarch64-apple-darwin test_cli_linux = "test --package trusted-server-cli --target x86_64-unknown-linux-gnu" test_cli_macos = "test --package trusted-server-cli --target aarch64-apple-darwin" +# --- Host-target lint gates that no adapter alias covers --- +# CI lints these two crates explicitly (see .github/workflows/format.yml), but +# pins the Linux triple, so there was no command a developer could run locally +# to reproduce them. These omit --target and therefore build for the host. +clippy-cli = "clippy -p trusted-server-cli --all-targets --all-features -- -D warnings" +clippy-codegen = "clippy -p trusted-server-openrtb-codegen --all-targets -- -D warnings" + # When a wasm binary IS built, run it under Viceroy. [target.'cfg(all(target_arch = "wasm32"))'] runner = "viceroy run -C ../../fastly.toml -- " diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97402e6f4..1f1bbe27a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -231,9 +231,14 @@ jobs: run: | cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings + - name: Set up Chrome for browser fixture tests + id: setup-chrome + uses: browser-actions/setup-chrome@v1 + - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh + env: + CHROME: ${{ steps.setup-chrome.outputs.chrome-path }} test-typescript: name: vitest diff --git a/.gitignore b/.gitignore index 24b9e06aa..96ffa2a5c 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ src/*.html # leftover local build artifacts (node_modules, target, dist) that remain on disk. /crates/js/ /crates/integration-tests/ +wrangler.integration.generated.toml diff --git a/AGENTS.md b/AGENTS.md index 3b7189204..b6e61ecf9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,6 +123,12 @@ cargo clippy-cloudflare-wasm cargo clippy-spin-native cargo clippy-spin-wasm +# The CLI and the OpenRTB codegen crate are host-target members that no adapter +# alias covers. CI lints both with the Linux triple pinned; these aliases omit +# `--target` so they reproduce it on any host. +cargo clippy-cli +cargo clippy-codegen + # Check compilation (per-target aliases — bare `cargo check` fails at the workspace root) cargo check-fastly && cargo check-axum && cargo check-cloudflare @@ -339,7 +345,7 @@ IntegrationRegistration::builder(ID) Every PR must pass: 1. `cargo fmt --all -- --check` -2. `cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm` +2. `cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm && cargo clippy-cli && cargo clippy-codegen` 3. `cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin` 4. `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` 5. JS build and test (`cd crates/trusted-server-js/lib && npx vitest run`) diff --git a/CHANGELOG.md b/CHANGELOG.md index 083d6e746..261387a7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- TSJS-generated envelopes now send `trustedServer.params.storedRequest: false`, preventing accidental PBS stored lookups without suppressing eligible non-PBS demand. PBS filters unusable impressions after overrides; explicit `true` and legacy omission retain inline-first stored fallback. Publisher intent survives repeated and refresh auctions. Deploy compatible server admission everywhere before serving the new JS, and retain it during rollback while cached clients remain. See the Prebid deployment guide. - Protocol-relative creative URLs now honor `rewrite.exclude_domains`, so excluded creative assets stay direct and excluded absolute or protocol-relative URLs submitted to `/first-party/sign` are rejected. - Server-side ad template bids now always carry `hb_adid` in `window.tsjs.bids`. Bidders that return neither a Prebid Cache UUID nor an `adid` previously produced no `hb_adid` at all, so no `hb_adid` GPT targeting key was set and the Universal Creative render bridge had nothing to match — the winning creative never rendered. The OpenRTB bid `id`, which is mandatory per spec, is now the last-resort source; `cache_id` and `adid` still take priority where present. Blank `cacheId`/`adid` values no longer win that precedence and emit an unusable empty `hb_adid`, and `hb_cache_host`/`hb_cache_path` are now emitted only alongside a real Prebid Cache UUID — without one they pointed the Universal Creative at a guaranteed cache miss instead of letting it fall through to the inline creative. diff --git a/Cargo.lock b/Cargo.lock index 311597aae..f79bc24b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -787,7 +787,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -5396,6 +5396,7 @@ dependencies = [ "futures", "log", "log-fastly", + "rand 0.8.6", "serde", "serde_json", "toml", @@ -5437,8 +5438,11 @@ dependencies = [ "derive_more", "directories", "edgezero-cli", + "edgezero-core", "error-stack", "futures", + "glob", + "http", "http-body-util", "hyper", "hyper-util", @@ -5451,12 +5455,15 @@ dependencies = [ "scraper", "serde", "serde_json", + "similar", + "temp-env", "tempfile", "time", "tokio", "tokio-rustls", "toml", "toml_edit 0.23.10+spec-1.0.0", + "tracing", "trusted-server-core", "url", "webpki-roots", @@ -6014,7 +6021,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index dfd94d0c4..ac0cac621 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,6 +97,7 @@ scraper = "0.24.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.149" sha2 = "0.10.9" +similar = "2.7" simple_logger = "5" spin-sdk = { version = "~6.0", default-features = false, features = ["http", "key-value", "variables"] } subtle = "2.6" @@ -109,6 +110,7 @@ tokio-rustls = "0.26" toml = "1.1" toml_edit = "0.23.10" tower = "0.4" +tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/README.md b/README.md index c606b340a..0ad5b0351 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ ts config init ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 15b6ee59d..09e8c77d2 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -20,6 +20,7 @@ path = "src/main.rs" [dependencies] async-trait = { workspace = true } +axum = { workspace = true } edgezero-adapter-axum = { workspace = true, features = ["axum"] } edgezero-core = { workspace = true } error-stack = { workspace = true } @@ -27,12 +28,11 @@ futures = { workspace = true } log = { workspace = true } reqwest = { workspace = true } simple_logger = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "sync", "time"] } +tower = { workspace = true, features = ["util"] } trusted-server-core = { workspace = true } [dev-dependencies] -axum = { workspace = true } base64 = { workspace = true } temp-env = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -tower = { workspace = true, features = ["util"] } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 38776eb95..fcf8bf98f 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -1,6 +1,7 @@ use core::future::Future; use std::sync::Arc; +use edgezero_adapter_axum::service::EdgeZeroAxumService; use edgezero_core::app::Hooks; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; @@ -574,15 +575,7 @@ impl Hooks for TrustedServerApp { } fn routes() -> RouterService { - let state = match build_state() { - Ok(s) => s, - Err(ref e) => { - log::error!("failed to build application state: {:?}", e); - return startup_error_router(e); - } - }; - - build_router(&state) + Self::routes_with_server_timing_flag().0 } } @@ -603,6 +596,44 @@ impl TrustedServerApp { let state = build_state_with_settings(settings)?; Ok(build_router(&state)) } + + /// The dev server's fully configured tower service: the application + /// router wrapped in the terminal timing layer + /// ([`crate::timing::TimingService`]), with `server_timing_enabled` + /// read from the same settings snapshot that built the router. + /// + /// This is the standard construction path for serving this adapter. + /// [`Hooks::routes`] satisfies the `Hooks` trait contract and returns + /// the bare router without the timing layer; callers who serve traffic + /// should use this instead so `server_timing_enabled` is never + /// silently discarded. + #[must_use] + pub fn dev_server_service() -> crate::timing::TimingService { + let (router, server_timing_enabled) = Self::routes_with_server_timing_flag(); + crate::timing::TimingService::new(EdgeZeroAxumService::new(router), server_timing_enabled) + } + + /// Build the router alongside whether `Server-Timing` emission is + /// enabled, read from the same settings snapshot used to build the + /// router. + /// + /// The Axum dev server's terminal timing layer ([`crate::timing`]) needs + /// this flag once at startup: unlike the Fastly adapter, which rebuilds + /// `Settings` per request, the Axum dev server builds its application + /// state once and reuses the same [`RouterService`] for every request. + #[must_use] + fn routes_with_server_timing_flag() -> (RouterService, bool) { + let state = match build_state() { + Ok(s) => s, + Err(ref e) => { + log::error!("failed to build application state: {:?}", e); + return (startup_error_router(e), false); + } + }; + + let server_timing_enabled = state.settings.observability.server_timing_enabled; + (build_router(&state), server_timing_enabled) + } } fn build_router(state: &Arc) -> RouterService { diff --git a/crates/trusted-server-adapter-axum/src/lib.rs b/crates/trusted-server-adapter-axum/src/lib.rs index 2f15e566d..b1d4c3dd8 100644 --- a/crates/trusted-server-adapter-axum/src/lib.rs +++ b/crates/trusted-server-adapter-axum/src/lib.rs @@ -10,3 +10,6 @@ pub mod app; pub mod middleware; /// Platform-trait implementations backed by env vars and `reqwest`. pub mod platform; +/// Terminal timing layer wrapping the Axum dev server's tower `Service` +/// boundary with the request-phase `Server-Timing` freeze point. +pub mod timing; diff --git a/crates/trusted-server-adapter-axum/src/main.rs b/crates/trusted-server-adapter-axum/src/main.rs index 960982176..4e360ea41 100644 --- a/crates/trusted-server-adapter-axum/src/main.rs +++ b/crates/trusted-server-adapter-axum/src/main.rs @@ -1,6 +1,15 @@ -use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; -use edgezero_core::app::Hooks as _; +use std::net::SocketAddr; + +use axum::Router; +use edgezero_adapter_axum::dev_server::AxumDevServerConfig; +use edgezero_adapter_axum::service::EdgeZeroAxumService; +use tokio::net::TcpListener; +use tokio::runtime::Builder as RuntimeBuilder; +use tokio::signal; +use tower::Service as _; +use tower::service_fn; use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_adapter_axum::timing::TimingService; #[allow(clippy::print_stderr)] fn main() { @@ -20,13 +29,63 @@ fn main() { }; log::info!("Listening on http://{}", config.addr); - let router = TrustedServerApp::routes(); - if let Err(err) = AxumDevServer::with_config(router, config).run() { + let service = TrustedServerApp::dev_server_service(); + if let Err(err) = run(service, config) { log::error!("trusted-server-adapter-axum failed: {err}"); std::process::exit(1); } } +/// Runs the Axum dev server with the request-phase timing terminal layer +/// ([`trusted_server_adapter_axum::timing::TimingService`]) wrapped around +/// `EdgeZeroAxumService`, ahead of `axum::serve`. +/// +/// This does not use `edgezero_adapter_axum::dev_server::AxumDevServer::run`: +/// that helper only accepts a bare [`RouterService`] and builds its own +/// `EdgeZeroAxumService` and `axum::Router` internally, with no seam for an +/// outer service wrapper. Router-generated 404/405 responses bypass +/// `RouterBuilder::middleware` (see `trusted_server_adapter_axum::timing`), +/// so the freeze point has to wrap the tower `Service` boundary itself. +/// Driving `axum::serve` directly here mirrors that helper's own internal +/// bind/wrap/serve/shutdown sequence closely enough to keep behavior +/// identical for callers (`PORT` env var, ctrl-c graceful shutdown). +/// +/// # Errors +/// +/// Returns an error if the Tokio runtime fails to start, the listener fails +/// to bind, or the underlying serve loop errors. +fn run( + service: TimingService, + config: AxumDevServerConfig, +) -> std::io::Result<()> { + let runtime = RuntimeBuilder::new_multi_thread().enable_all().build()?; + runtime.block_on(serve(service, config)) +} + +async fn serve( + service: TimingService, + config: AxumDevServerConfig, +) -> std::io::Result<()> { + let listener = TcpListener::bind(config.addr).await?; + + let axum_router = Router::new().fallback_service(service_fn(move |req| { + let mut svc = service.clone(); + async move { svc.call(req).await } + })); + let make_service = axum_router.into_make_service_with_connect_info::(); + + let server = axum::serve(listener, make_service); + if config.enable_ctrl_c { + server + .with_graceful_shutdown(async { + let _ctrl_c = signal::ctrl_c().await; + }) + .await + } else { + server.await + } +} + /// Read a port number from the `PORT` environment variable. /// /// Returns `None` when the variable is unset. Exits non-zero if the value diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 7dcdd53d8..678f44ea6 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -522,16 +522,13 @@ impl PlatformHttpClient for AxumPlatformHttpClient { /// /// # Degraded features in dev /// -/// KV store is [`trusted_server_core::platform::UnavailableKvStore`] — any route -/// touching synthetic-ID or consent KV will degrade gracefully. A `warn` log is +/// The generic runtime KV slot uses +/// [`trusted_server_core::platform::UnavailableKvStore`]. A `warn` log is /// emitted once per process. pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> RuntimeServices { static KV_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); KV_WARNED.get_or_init(|| { - log::warn!( - "Axum dev server: KV store is unavailable (UnavailableKvStore). \ - Routes that depend on synthetic-ID or consent KV will degrade gracefully." - ); + log::warn!("Axum dev server: generic runtime KV is unavailable (UnavailableKvStore)."); }); let client_ip = edgezero_adapter_axum::context::AxumRequestContext::get(ctx.request()) diff --git a/crates/trusted-server-adapter-axum/src/timing.rs b/crates/trusted-server-adapter-axum/src/timing.rs new file mode 100644 index 000000000..315ebb5ef --- /dev/null +++ b/crates/trusted-server-adapter-axum/src/timing.rs @@ -0,0 +1,310 @@ +//! Terminal timing layer for the Axum dev server. +//! +//! [`TimingService`](crate::timing::TimingService) wraps the tower `Service` +//! boundary the Axum dev server's router sits behind: it creates a +//! [`RequestTimings`](trusted_server_core::request_timing::RequestTimings) +//! collector per request, threads it through request extensions so +//! downstream core handlers can record into it, and on the way back stamps +//! `mark_headers_ready` and appends the `Server-Timing` header via +//! [`append_server_timing_if_private`](trusted_server_core::request_timing::append_server_timing_if_private). +//! +//! This wraps *outside* `RouterService` rather than registering as +//! `RouterBuilder::middleware` because the tower boundary is the terminal +//! freeze point: by the time a response reaches this layer -- after +//! `RouterService::oneshot` inside `EdgeZeroAxumService::call` has +//! converted any dispatch error into a plain response -- every response is +//! covered uniformly regardless of how routing produced it, and the +//! position survives future routing changes. (In this application's router +//! a catch-all fallback spans every path and publisher method, so +//! router-generated 404/405s that bypass middleware are close to +//! unreachable today; the outer position does not depend on them.) +//! +//! `/health` is excluded by path match before a +//! [`RequestTimings`](trusted_server_core::request_timing::RequestTimings) +//! collector is even created: health checks never carry timing data on any +//! adapter. +//! +//! Unlike the Fastly adapter (state built per request, adding +//! `Phase::AppBuild` to the rendered header), the Axum dev server builds its +//! application state once at startup. There is no per-request app-build +//! interval to measure, so `ts-appbuild` never appears in the header here. + +use std::convert::Infallible; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use axum::body::Body as AxumBody; +use axum::http::{Request, Response}; +use tower::Service; +use trusted_server_core::request_timing::{RequestTimings, append_server_timing_if_private}; + +/// Path excluded from timing collection and `Server-Timing` emission: health +/// checks never carry timing data on any adapter. +const HEALTH_PATH: &str = "/health"; + +/// Wraps an inner Axum tower service with the request-phase timing freeze +/// point described in the module docs. +#[derive(Clone)] +pub struct TimingService { + inner: S, + server_timing_enabled: bool, +} + +impl TimingService { + /// Wraps `inner`, appending `Server-Timing` when `server_timing_enabled` + /// is set and the response is conclusively private. + #[must_use] + pub fn new(inner: S, server_timing_enabled: bool) -> Self { + Self { + inner, + server_timing_enabled, + } + } +} + +impl Service> for TimingService +where + S: Service, Response = Response, Error = Infallible> + + Clone + + Send + + 'static, + S::Future: Send + 'static, +{ + type Error = Infallible; + type Future = Pin> + Send>>; + type Response = Response; + + fn call(&mut self, mut req: Request) -> Self::Future { + let mut inner = self.inner.clone(); + + // Excluded before a collector is even created: `/health` never + // carries timing data, on any adapter. + if req.uri().path() == HEALTH_PATH { + return Box::pin(async move { inner.call(req).await }); + } + + let server_timing_enabled = self.server_timing_enabled; + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + Box::pin(async move { + let mut response = inner.call(req).await?; + append_server_timing_if_private(&mut response, &timings, server_timing_enabled); + Ok(response) + }) + } + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::header::CACHE_CONTROL; + use axum::http::{HeaderValue, StatusCode}; + use edgezero_adapter_axum::service::EdgeZeroAxumService; + use edgezero_core::body::Body as EdgeBody; + use edgezero_core::context::RequestContext; + use edgezero_core::error::EdgeError; + use edgezero_core::http::response_builder; + use edgezero_core::router::RouterService; + use tower::{ServiceExt as _, service_fn}; + + /// Builds a private (`cache-control: private, no-store`) response for a + /// handler under test. + fn private_ok_response() -> Result { + Ok(response_builder() + .status(StatusCode::OK) + .header("cache-control", "private, no-store") + .body(EdgeBody::from("ok")) + .expect("should build a private response fixture")) + } + + /// Reads a response header as a UTF-8 string, or `None` if absent. + fn header(response: &Response, name: &str) -> Option { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_emits_header_on_private_response() { + let router = RouterService::builder() + .get("/private", |_ctx: RequestContext| async { + private_ok_response() + }) + .build(); + let mut service = TimingService::new(EdgeZeroAxumService::new(router), true); + + let request = Request::builder() + .uri("/private") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + let server_timing = header(&response, "server-timing").expect("should emit header"); + assert!( + server_timing.contains("ts-total;dur="), + "should carry the collected total: {server_timing}" + ); + assert!( + !server_timing.contains("ts-appbuild"), + "the Axum dev server builds state once at startup, so there is no \ + per-request app-build interval to render: {server_timing}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_suppresses_header_when_flag_is_off() { + let router = RouterService::builder() + .get("/private", |_ctx: RequestContext| async { + private_ok_response() + }) + .build(); + let mut service = TimingService::new(EdgeZeroAxumService::new(router), false); + + let request = Request::builder() + .uri("/private") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + assert!( + header(&response, "server-timing").is_none(), + "should not emit server-timing when server_timing_enabled is false" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_round_trips_phase_timings_recorded_in_the_handler() { + // The collector crosses the adapter boundary as a request extension; + // this pins the round trip end to end: a phase recorded inside a + // core-style handler must come back out in the rendered header, so + // a future adapter conversion that drops request extensions fails + // here instead of silently losing every phase. + let router = RouterService::builder() + .get("/private", |ctx: RequestContext| async move { + if let Some(timings) = ctx.request().extensions().get::() { + timings.record( + trusted_server_core::request_timing::Phase::Filter, + std::time::Duration::from_millis(7), + ); + } + private_ok_response() + }) + .build(); + let mut service = TimingService::new(EdgeZeroAxumService::new(router), true); + + let request = Request::builder() + .uri("/private") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + let server_timing = header(&response, "server-timing").expect("should emit header"); + assert!( + server_timing.contains("ts-filter;dur=7.0"), + "a phase recorded in the handler should survive the adapter round trip: {server_timing}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_404_carries_header_when_private() { + // An empty router has no routes at all, so any path dispatches + // through `RouterInner::dispatch`'s `NotFound` branch -- exactly the + // path that bypasses `RouterBuilder::middleware`. The router's own + // `EdgeError::into_response` does not attach `Cache-Control`, so a + // small wrapping service forces the response private here, standing + // in for whatever upstream layer would normally mark a genuinely + // private 404. This proves the freeze point still runs for a + // router-generated response without weakening + // `append_server_timing_if_private`'s real gating logic. + let empty_router = RouterService::builder().build(); + let inner = EdgeZeroAxumService::new(empty_router); + let force_private = service_fn(move |req: Request| { + let mut svc = inner.clone(); + async move { + let mut response = svc.call(req).await?; + response + .headers_mut() + .insert(CACHE_CONTROL, HeaderValue::from_static("private, no-store")); + Ok::<_, Infallible>(response) + } + }); + let mut service = TimingService::new(force_private, true); + + let request = Request::builder() + .uri("/does-not-exist") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should still be the router's own not-found response" + ); + let server_timing = header(&response, "server-timing") + .expect("a router-generated 404 must still carry the header when private"); + assert!( + server_timing.contains("ts-total;dur="), + "should carry the collected total: {server_timing}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_health_is_excluded() { + let router = RouterService::builder() + .get("/health", |_ctx: RequestContext| async { + private_ok_response() + }) + .build(); + let mut service = TimingService::new(EdgeZeroAxumService::new(router), true); + + let request = Request::builder() + .uri("/health") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + assert!( + header(&response, "server-timing").is_none(), + "/health must never carry a server-timing header" + ); + } +} diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.integration.generated.toml b/crates/trusted-server-adapter-cloudflare/wrangler.integration.generated.toml new file mode 100644 index 000000000..263403193 --- /dev/null +++ b/crates/trusted-server-adapter-cloudflare/wrangler.integration.generated.toml @@ -0,0 +1,16 @@ +name = "trusted-server" +main = "build/index.js" +compatibility_date = "2024-09-23" +# Keep in sync with wrangler.toml. `cache_option_enabled` is required for the +# outbound `CacheMode::NoStore` cache bypass under this compatibility date. +compatibility_flags = ["nodejs_compat", "cache_option_enabled"] +# No [build] section — bundle is pre-built in CI; wrangler dev must not rebuild. + +[[kv_namespaces]] +binding = "TRUSTED_SERVER_KV" +id = "ci-local-kv" + +[vars] +# Placeholder replaced by the integration test harness with a JSON object that +# contains the runtime Trusted Server app-config blob envelope. +TRUSTED_SERVER_CONFIG = '''{"app_config":"{\"data\":{\"auction\":{\"allowed_context_keys\":[],\"creative_store\":\"creative_store\",\"enabled\":false,\"mediator\":null,\"providers\":[],\"timeout_ms\":2000},\"cache\":{\"asset_rules\":[]},\"consent\":{\"check_expiration\":true,\"conflict_resolution\":{\"freshness_threshold_days\":30,\"mode\":\"restrictive\"},\"gdpr\":{\"applies_in\":[\"AT\",\"BE\",\"BG\",\"HR\",\"CY\",\"CZ\",\"DK\",\"EE\",\"FI\",\"FR\",\"DE\",\"GR\",\"HU\",\"IE\",\"IT\",\"LV\",\"LT\",\"LU\",\"MT\",\"NL\",\"PL\",\"PT\",\"RO\",\"SK\",\"SI\",\"ES\",\"SE\",\"IS\",\"LI\",\"NO\",\"GB\"]},\"max_consent_age_days\":395,\"mode\":\"interpreter\",\"us_privacy_defaults\":{\"gpc_implies_optout\":true,\"lspa_covered\":false,\"notice_given\":true},\"us_states\":{\"privacy_states\":[\"CA\",\"VA\",\"CO\",\"CT\",\"UT\",\"MT\",\"OR\",\"TX\",\"FL\",\"DE\",\"IA\",\"NE\",\"NH\",\"NJ\",\"TN\",\"MN\",\"MD\",\"IN\",\"KY\",\"RI\"]}},\"creative_opportunities\":null,\"debug\":{\"auction_html_comment\":false,\"inject_adm_for_testing\":false,\"ja4_endpoint_enabled\":false},\"ec\":{\"cluster_recheck_secs\":3600,\"cluster_trust_threshold\":10,\"ec_store\":\"ec_identity_store\",\"partners\":[{\"api_token\":\"integration-test-token-alpha-32-bytes-ok\",\"batch_rate_limit\":60,\"bidstream_enabled\":true,\"name\":\"Integration Test Partner\",\"openrtb_atype\":3,\"pull_sync_allowed_domains\":[],\"pull_sync_enabled\":false,\"pull_sync_rate_limit\":10,\"pull_sync_ttl_sec\":86400,\"pull_sync_url\":null,\"source_domain\":\"inttest.example.com\",\"ts_pull_token\":null},{\"api_token\":\"integration-test-token-bravo-32-bytes-ok\",\"batch_rate_limit\":60,\"bidstream_enabled\":true,\"name\":\"Integration Test Partner 2\",\"openrtb_atype\":3,\"pull_sync_allowed_domains\":[],\"pull_sync_enabled\":false,\"pull_sync_rate_limit\":10,\"pull_sync_ttl_sec\":86400,\"pull_sync_url\":null,\"source_domain\":\"inttest2.example.com\",\"ts_pull_token\":null}],\"passphrase\":\"integration-test-ec-secret-padded-32\",\"pull_sync_concurrency\":3},\"handlers\":[{\"password\":\"integration-admin-password-32-bytes-ok\",\"path\":\"^/_ts/admin\",\"username\":\"admin\"}],\"image_optimizer\":{\"profile_sets\":{}},\"integrations\":{\"adserver_mock\":{\"context_query_params\":{\"example_segments\":\"segments\"},\"enabled\":false,\"endpoint\":\"https://adserver.example.com/mediate\",\"timeout_ms\":1000},\"aps\":{\"account_id\":\"example-aps-account-id\",\"allow_script_creatives\":false,\"enabled\":true,\"endpoint\":\"https://aps.example.com/e/pb/bid\",\"timeout_ms\":1000},\"datadome\":{\"api_origin\":\"https://api.example.com\",\"cache_ttl_seconds\":3600,\"enabled\":false,\"rewrite_sdk\":true,\"sdk_origin\":\"https://sdk.example.com\"},\"didomi\":{\"api_origin\":\"https://api.example.com\",\"enabled\":false,\"sdk_origin\":\"https://sdk.example.com\"},\"google_tag_manager\":{\"container_id\":\"GTM-EXAMPLE\",\"enabled\":false,\"upstream_url\":\"https://tags.example.com\"},\"gpt\":{\"cache_ttl_seconds\":3600,\"enabled\":false,\"gam_attribution_enabled\":false,\"rewrite_script\":true,\"script_url\":\"https://ads.example.com/gpt.js\"},\"gpt_diagnostics\":{\"enabled\":true},\"lockr\":{\"api_endpoint\":\"https://identity.example.com\",\"app_id\":\"\",\"cache_ttl_seconds\":3600,\"enabled\":false,\"rewrite_sdk\":true,\"sdk_url\":\"https://identity.example.com/trusted-server.js\"},\"nextjs\":{\"enabled\":false,\"max_combined_payload_bytes\":10485760,\"rewrite_attributes\":[\"href\",\"link\",\"siteBaseUrl\",\"siteProductionDomain\",\"url\"]},\"permutive\":{\"api_endpoint\":\"https://api.example.com\",\"enabled\":false,\"organization_id\":\"\",\"project_id\":\"\",\"secure_signals_endpoint\":\"https://secure-signals.example.com\",\"workspace_id\":\"\"},\"prebid\":{\"bidders\":[],\"client_side_bidders\":[],\"debug\":false,\"enabled\":false,\"server_url\":\"https://prebid.example.com/openrtb2/auction\",\"timeout_ms\":1000},\"sourcepoint\":{\"cache_ttl_seconds\":3600,\"cdn_origin\":\"https://cdn.example.com\",\"enabled\":false,\"rewrite_sdk\":true},\"testlight\":{\"enabled\":false,\"endpoint\":\"https://testlight.example.com/openrtb2/auction\",\"rewrite_scripts\":true,\"timeout_ms\":1200}},\"proxy\":{\"allowed_domains\":[],\"asset_routes\":[],\"certificate_check\":false},\"publisher\":{\"cookie_domain\":\"localhost\",\"domain\":\"localhost\",\"max_buffered_body_bytes\":16777216,\"origin_host_header_override\":null,\"origin_url\":\"http://127.0.0.1:8888\",\"proxy_secret\":\"integration-test-proxy-secret\"},\"request_signing\":{\"config_store_id\":\"app_config\",\"enabled\":false,\"secret_store_id\":\"secrets\"},\"response_headers\":{},\"rewrite\":{\"exclude_domains\":[]},\"tester_cookie\":{\"enabled\":false},\"tinybird\":{\"access_dataset\":\"access_logs_raw\",\"access_enabled\":false,\"access_sample_rate\":0.0,\"access_token_secret\":\"tinybird_access_append_token\",\"api_host\":\"\",\"auction_dataset\":\"auction_events_raw\",\"auction_enabled\":true,\"auction_token_secret\":\"tinybird_auction_append_token\",\"enabled\":false,\"max_body_bytes\":1048576,\"secret_store\":\"ts_secrets\"}},\"generated_at\":\"2026-06-23T00:00:00Z\",\"sha256\":\"895f7fad0ce924476d1c04c68b0bf95f463d1fb631a4f639dcc7cf0c510383b4\",\"version\":1}"}''' diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 65320faa6..584085e76 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -28,6 +28,7 @@ log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } +rand = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 190be505c..43df8bdef 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -100,6 +100,7 @@ use edgezero_core::http::{ }; use edgezero_core::router::RouterService; use error_stack::Report; +use trusted_server_core::access_telemetry::{RouteClass, RouteMetadata, publisher_route_template}; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ @@ -108,7 +109,6 @@ use trusted_server_core::auction::{ use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::config_payload::DEFAULT_SECRET_STORE_ID; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; -use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ deny_admin_diagnostic_fallback, handle_admin_ec_lookup, handle_admin_eids_lookup, }; @@ -118,7 +118,9 @@ use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::ec::{EcContext, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::geo::GeoLookupState; use trusted_server_core::http_util::is_navigation_request; use trusted_server_core::integrations::{ IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, RequestFilterRegistryInput, @@ -140,14 +142,17 @@ use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, handle_verify_signature, }; +use trusted_server_core::request_timing::{Phase, RequestTimings}; use trusted_server_core::settings::{ProxyAssetRoute, Settings}; -use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; +use trusted_server_core::settings_data::{ + DEFAULT_CONFIG_STORE_ID, config_key, config_store_name, get_settings_from_config_store, +}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ FastlyPlatformBackend, FastlyPlatformConfigStore, FastlyPlatformGeo, FastlyPlatformHttpClient, - FastlyPlatformSecretStore, UnavailableKvStore, open_kv_store, + FastlyPlatformSecretStore, UnavailableKvStore, }; // --------------------------------------------------------------------------- @@ -164,8 +169,8 @@ pub(crate) struct RuntimeStoreConfig { impl RuntimeStoreConfig { pub(crate) fn from_env(env: &EnvConfig) -> Self { Self { - config_store_name: StoreName::from(env.store_name("config", DEFAULT_CONFIG_STORE_ID)), - config_key: env.store_key("config", DEFAULT_CONFIG_STORE_ID), + config_store_name: config_store_name(env), + config_key: config_key(env), secret_store_name: StoreName::from(env.store_name("secrets", DEFAULT_SECRET_STORE_ID)), } } @@ -237,36 +242,6 @@ fn warn_if_certificate_check_disabled(settings: &Settings) { } } -/// Resolves per-request consent KV store services for routes that read consent data. -/// -/// When `settings.consent.consent_store` is configured and the named KV store cannot -/// be opened, returns `Err` so the caller can respond with 503 (fail-closed). This is -/// intentional hardening over the legacy `route_request` path, which builds -/// `runtime_services` with `UnavailableKvStore` and never opens the named consent -/// store, so it never fails closed — the `EdgeZero` path instead makes consent-dependent -/// routes unavailable rather than proceeding without consent. -/// -/// # Errors -/// -/// Returns an error when the configured consent store cannot be opened. -pub(crate) fn runtime_services_for_consent_route( - settings: &Settings, - runtime_services: &RuntimeServices, -) -> Result> { - let Some(store_name) = settings.consent.consent_store.as_deref() else { - return Ok(runtime_services.clone()); - }; - - open_kv_store(store_name) - .map(|store| runtime_services.clone().with_kv_store(store)) - .map_err(|e| { - Report::new(TrustedServerError::KvStore { - store_name: store_name.to_string(), - message: e.to_string(), - }) - }) -} - // --------------------------------------------------------------------------- // Per-request RuntimeServices // --------------------------------------------------------------------------- @@ -326,6 +301,12 @@ fn uses_dynamic_tsjs_fallback(method: &Method, path: &str) -> bool { *method == Method::GET && path.starts_with("/static/tsjs=") } +/// Coarse route template for every `tsjs` bundle request, used as the +/// `route_template` in the [`RouteMetadata`] attached by the tsjs branch of +/// [`dispatch_fallback`]. Actual filenames vary by module/hash; the prefix +/// alone is the route identity that matters for access telemetry. +const TSJS_ROUTE_TEMPLATE: &str = "/static/tsjs=*"; + // --------------------------------------------------------------------------- // EC request state // --------------------------------------------------------------------------- @@ -387,6 +368,17 @@ impl EcRequestState { services: self.services, } } + + /// Derives the carried [`GeoLookupState`] from this request's geo lookup + /// outcome, so response-phase finalize can reuse it instead of repeating + /// the lookup. `build_ec_request_state` always attempts the lookup, so + /// `None` here means the lookup ran and failed, not that it was skipped. + fn geo_lookup_state(&self) -> GeoLookupState { + match &self.geo_info { + Some(info) => GeoLookupState::Resolved(info.clone()), + None => GeoLookupState::Attempted, + } + } } /// Derives device signals from the request's `User-Agent` header. @@ -442,13 +434,21 @@ fn build_ec_request_state( let eids_cookie = crate::extract_cookie_value(req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(req, COOKIE_SHAREDID); - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed during EC setup: {e}"); - None - }); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let geo_info = { + let _span = timings.span(Phase::Geo); + services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed during EC setup: {e}"); + None + }) + }; let (ec_context, setup_error) = match EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) { @@ -469,7 +469,7 @@ fn build_ec_request_state( // Bot gate: suppress KV-backed EC writes for unrecognized clients, except // consent withdrawals. Revocations keep the write path so tombstones stay // authoritative even for privacy-extension-heavy clients. - let kv_graph = crate::maybe_identity_graph(settings); + let kv_graph = crate::identity_graph_with_timing(settings, &timings); let finalize_kv_graph = if setup_error.is_none() && (is_real_browser || ec_consent_withdrawn(ec_context.consent())) { @@ -521,6 +521,18 @@ async fn run_pre_route_filters( req: &mut Request, geo_info: Option<&GeoInfo>, ) -> PreRoute { + // Only recorded when a filter is actually registered, so unconfigured + // deployments omit ts-filter from the Server-Timing header entirely. + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let _span = state + .registry + .has_request_filters() + .then(|| timings.span(Phase::Filter)); + match state .registry .filter_request(RequestFilterRegistryInput { @@ -554,6 +566,7 @@ fn attach_dispatch_extensions( ec: EcRequestState, effects: RequestFilterEffects, ) -> Response { + response.extensions_mut().insert(ec.geo_lookup_state()); response.extensions_mut().insert(ec.into_finalize_state()); if !effects.response_headers.is_empty() { response.extensions_mut().insert(effects); @@ -590,7 +603,12 @@ async fn execute_named( // Deliberately do not use an EC request-state graph: that // copy is bot-gated, while operators use curl for this // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let kv = crate::identity_graph_with_timing(&state.settings, &timings); handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) } NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), @@ -661,7 +679,12 @@ async fn run_named_route( if req.method() == Method::OPTIONS { cors_preflight_identify(&state.settings, &req) } else { - let kv = crate::require_identity_graph(&state.settings)?; + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let kv = crate::require_identity_graph_with_timing(&state.settings, &timings)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; handle_identify( &state.settings, @@ -675,10 +698,7 @@ async fn run_named_route( NamedRouteHandler::SetTester => handle_set_tester(&state.settings), NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { - // The auction reads consent data, so the consent KV store must be - // available — fail closed with 503 when it is configured but - // cannot be opened, matching legacy behavior. - let consent_services = runtime_services_for_consent_route(&state.settings, services)?; + ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let registry_ref = if partner_registry.is_empty() { None @@ -691,7 +711,7 @@ async fn run_named_route( ec.kv_graph.as_ref(), registry_ref, &mut ec.ec_context, - &consent_services, + services, req, ) .await @@ -703,10 +723,6 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - // Like the auction, page-bids reads consent data, so the consent KV - // store must be available — fail closed with 503 when configured but - // unopenable, matching legacy. - let consent_services = runtime_services_for_consent_route(&state.settings, services)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let registry_ref = if partner_registry.is_empty() { None @@ -720,7 +736,7 @@ async fn run_named_route( }; handle_page_bids( &state.settings, - &consent_services, + services, ec.kv_graph.as_ref(), auction, &mut ec.ec_context, @@ -751,12 +767,18 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let is_real_browser = device_signals.looks_like_browser(); let eids_cookie = crate::extract_cookie_value(&req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); - let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - handle_batch_sync(&kv, &partner_registry, &limiter, req) - }); + let result = + crate::require_identity_graph_with_timing(&state.settings, &timings).and_then(|kv| { + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + handle_batch_sync(&kv, &partner_registry, &limiter, req) + }); let mut response = result.unwrap_or_else(|e| http_error(&e)); // Legacy parity: batch-sync responses still pass through @@ -816,12 +838,35 @@ async fn dispatch_fallback( PreRoute::Continue { effects } => effects, }; + // Assigned exactly once, per branch below, alongside the routing + // decision itself, so the access-telemetry route identity always + // reflects which branch actually dispatched the request — including + // when that branch's handler errors. The asset-route sub-branch is an + // early return handled separately by `dispatch_asset_fallback`, so it + // never reaches (or needs to assign) this binding. + let route_metadata: Option; + let result = if uses_dynamic_tsjs_fallback(&method, &path) { + route_metadata = Some(RouteMetadata { + route_class: RouteClass::Tsjs, + route_template: TSJS_ROUTE_TEMPLATE.to_owned(), + }); handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SurrogateControl) } else if state.registry.has_route(&method, &path) { // Integration-proxy responses are not bounded by // publisher.max_buffered_body_bytes. Publisher fallback below uses the // publisher-specific streaming finalizer instead. + // The matched route pattern is an integration-defined literal + // (bounded and content-free by construction), so telemetry keeps it + // verbatim instead of running the request path through the lossy + // publisher classifier. + route_metadata = Some(RouteMetadata { + route_class: RouteClass::IntegrationProxy, + route_template: state + .registry + .matched_route_pattern(&method, &path) + .map_or_else(|| "/other/*".to_owned(), str::to_owned), + }); state .registry .handle_proxy(ProxyDispatchInput { @@ -849,14 +894,43 @@ async fn dispatch_fallback( .then(|| state.settings.asset_route_for_path(&path)) .flatten(); if let Some(asset_route) = matched_asset_route { - return dispatch_asset_fallback(state, services, req, asset_route, &effects).await; + // The template is the operator-configured route prefix, so it + // is bounded and content-free by construction (unlike request + // paths, which need `publisher_route_template`). + let asset_metadata = RouteMetadata { + route_class: RouteClass::Asset, + route_template: format!("{}/*", asset_route.prefix.trim_end_matches('/')), + }; + let mut response = dispatch_asset_fallback( + state, + services, + req, + asset_route, + &effects, + ec.geo_lookup_state(), + ) + .await; + response.extensions_mut().insert(asset_metadata); + return response; } + route_metadata = Some(RouteMetadata { + route_class: RouteClass::PublisherHtml, + route_template: publisher_route_template( + &path, + &state.settings.observability.route_sections, + ), + }); + // Generate an EC ID if needed — mirrors the legacy catch-all arm. // Only for document navigations by recognised browsers; subresource // requests may lack consent signals such as Sec-GPC. - let is_publisher_navigation = ec.is_real_browser && is_navigation_request(&req); - if is_publisher_navigation + let is_publisher_navigation = is_navigation_request(&req); + if is_publisher_navigation { + ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); + } + if ec.is_real_browser + && is_publisher_navigation && let Err(err) = ec .ec_context .generate_if_needed(&state.settings, ec.kv_graph.as_ref()) @@ -864,57 +938,49 @@ async fn dispatch_fallback( log::warn!("EC generation failed for publisher proxy: {err:?}"); } - // Publisher pages read consent data, so the consent KV store must be - // available — fail closed with 503 when it is configured but cannot - // be opened, matching legacy behavior. - match runtime_services_for_consent_route(&state.settings, services) { - Ok(publisher_services) => { - // Run the server-side auction with the configured creative- - // opportunity slots and collect dispatched bids from the lazy - // publisher body stream. `handle_publisher_request` matches the - // slots against the request path. The partner registry plus the - // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with - // server-side EIDs, same as the legacy auction. - let slots = state.settings.creative_opportunity_slots(); - match PartnerRegistry::from_config(&state.settings.ec.partners) { - Ok(partner_registry) => { - let auction = AuctionDispatch { - orchestrator: &state.orchestrator, - slots, - registry: Some(&partner_registry), - }; - match handle_publisher_request( - &state.settings, - &publisher_services, - ec.kv_graph.as_ref(), - &mut ec.ec_context, - auction, - req, - EdgeCacheHeader::SurrogateControl, + // Run the server-side auction with the configured creative- + // opportunity slots and collect dispatched bids from the lazy + // publisher body stream. `handle_publisher_request` matches the + // slots against the request path. The partner registry plus the + // EC identity-graph KV (`ec.kv_graph`) enriches the bid request with + // server-side EIDs, same as the legacy auction. + let slots = state.settings.creative_opportunity_slots(); + match PartnerRegistry::from_config(&state.settings.ec.partners) { + Ok(partner_registry) => { + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots, + registry: Some(&partner_registry), + }; + match handle_publisher_request( + &state.settings, + services, + ec.kv_graph.as_ref(), + &mut ec.ec_context, + auction, + req, + EdgeCacheHeader::SurrogateControl, + ) + .await + { + Ok(pub_response) => { + // Origin start succeeded on the sole publisher- + // page path: authorize orphan recovery now, and + // only for real-browser document navigations. + // Restricting it here keeps identity rotation + // within the publisher-navigation boundary — + // named routes, integration proxies, and filter + // short circuits never reach this point. + ec.ec_context.set_recovery_eligible(is_publisher_navigation); + publisher_response_into_streaming_response( + pub_response, + &method, + Arc::clone(&state.settings), + state.registry.as_ref(), + Arc::clone(&state.orchestrator), + services.clone(), ) .await - { - Ok(pub_response) => { - // Origin start succeeded on the sole publisher- - // page path: authorize orphan recovery now, and - // only for real-browser document navigations. - // Restricting it here keeps identity rotation - // within the publisher-navigation boundary — - // named routes, integration proxies, and filter - // short circuits never reach this point. - ec.ec_context.set_recovery_eligible(is_publisher_navigation); - publisher_response_into_streaming_response( - pub_response, - &method, - Arc::clone(&state.settings), - state.registry.as_ref(), - Arc::clone(&state.orchestrator), - publisher_services.clone(), - ) - .await - } - Err(e) => Err(e), - } } Err(e) => Err(e), } @@ -923,7 +989,10 @@ async fn dispatch_fallback( } }; - let response = result.unwrap_or_else(|e| http_error(&e)); + let mut response = result.unwrap_or_else(|e| http_error(&e)); + if let Some(metadata) = route_metadata { + response.extensions_mut().insert(metadata); + } attach_dispatch_extensions(response, ec, effects) } @@ -947,7 +1016,10 @@ fn asset_response_carries_body(method: &Method, status: StatusCode) -> bool { /// [`AssetProxyCachePolicy`] out via response extensions so `edgezero_main` /// can reapply protected cache directives after finalization. EC finalization /// is intentionally skipped: no [`EcFinalizeState`] is attached, matching the -/// legacy `should_finalize_ec = false` behavior for asset responses. +/// legacy `should_finalize_ec = false` behavior for asset responses. The +/// caller's [`GeoLookupState`] is still attached, since `build_ec_request_state` +/// already attempted the lookup before the asset route was matched — this is +/// the one exit path that carries geo state without an `EcFinalizeState`. /// /// Like legacy `route_request`, asset bodies are streamed straight to the client /// with no cap: the origin stream is attached to the response and `edgezero_main` @@ -962,6 +1034,7 @@ async fn dispatch_asset_fallback( req: Request, asset_route: &ProxyAssetRoute, effects: &RequestFilterEffects, + geo_state: GeoLookupState, ) -> Response { log::info!("No explicit route matched; proxying via configured asset route"); @@ -983,6 +1056,7 @@ async fn dispatch_asset_fallback( } response.extensions_mut().insert(cache_policy); + response.extensions_mut().insert(geo_state); attach_request_filter_effects(&mut response, effects); response } @@ -991,6 +1065,7 @@ async fn dispatch_asset_fallback( response .extensions_mut() .insert(AssetProxyCachePolicy::NoStorePrivate); + response.extensions_mut().insert(geo_state); attach_request_filter_effects(&mut response, effects); response } @@ -1113,6 +1188,10 @@ struct NamedRoute { path: &'static str, primary_methods: &'static [Method], handler: NamedRouteHandler, + /// Access-telemetry traffic category for this row. Attached verbatim + /// alongside `path` (the route-table pattern) to every response this + /// route produces — see [`named_route_handler`]. + route_class: RouteClass, } const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ @@ -1130,21 +1209,25 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/.well-known/trusted-server.json", primary_methods: &[Method::GET], handler: NamedRouteHandler::TrustedServerDiscovery, + route_class: RouteClass::Other, }, NamedRoute { path: "/verify-signature", primary_methods: &[Method::POST], handler: NamedRouteHandler::VerifySignature, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/keys/rotate", primary_methods: &[Method::POST], handler: NamedRouteHandler::RotateKey, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/keys/deactivate", primary_methods: &[Method::POST], handler: NamedRouteHandler::DeactivateKey, + route_class: RouteClass::Ec, }, // Admin EC lookup: the bare route reads the EC ID from the caller's // `ts-ec` cookie; the parameterized route takes an explicit EC ID. @@ -1152,11 +1235,13 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/_ts/admin/ec", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcLookup, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/ec/{id}", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcLookup, + route_class: RouteClass::Ec, }, // Admin EIDs echo: decodes the request's ts-eids/sharedId cookies with // an ingestion preview. Pure request inspection — no KV access. @@ -1164,6 +1249,7 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/_ts/admin/eids", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEidsLookup, + route_class: RouteClass::Ec, }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a // 404 instead of executing key operations: the production basic-auth handler @@ -1175,36 +1261,43 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/admin/keys/rotate", primary_methods: LEGACY_ADMIN_DENY_METHODS, handler: NamedRouteHandler::LegacyAdminDenied, + route_class: RouteClass::Other, }, NamedRoute { path: "/admin/keys/deactivate", primary_methods: LEGACY_ADMIN_DENY_METHODS, handler: NamedRouteHandler::LegacyAdminDenied, + route_class: RouteClass::Other, }, NamedRoute { path: "/_ts/api/v1/batch-sync", primary_methods: &[Method::POST], handler: NamedRouteHandler::BatchSync, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/api/v1/identify", primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::Identify, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/set-tester", primary_methods: &[Method::GET], handler: NamedRouteHandler::SetTester, + route_class: RouteClass::Other, }, NamedRoute { path: "/_ts/clear-tester", primary_methods: &[Method::GET], handler: NamedRouteHandler::ClearTester, + route_class: RouteClass::Other, }, NamedRoute { path: "/auction", primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, + route_class: RouteClass::AuctionApi, }, // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS // preflight guard for this side-effecting endpoint. @@ -1212,6 +1305,7 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: PAGE_BIDS_PATH, primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::PageBids, + route_class: RouteClass::AuctionApi, }, // Deprecated double-underscore alias. tsjs bundles served before the // `/_ts/page-bids` rename keep requesting this path from already-loaded @@ -1222,21 +1316,29 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: PAGE_BIDS_LEGACY_PATH, primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::PageBids, + route_class: RouteClass::AuctionApi, }, + // Classified `Other` rather than `IntegrationProxy`: that class is + // reserved for `state.registry.handle_proxy` (the js-integration proxy + // dispatch in `dispatch_fallback`), which these first-party proxy routes + // do not go through. NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], handler: NamedRouteHandler::FirstPartyProxy, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/click", primary_methods: &[Method::GET], handler: NamedRouteHandler::FirstPartyClick, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/sign", primary_methods: &[Method::GET, Method::POST], handler: NamedRouteHandler::FirstPartySign, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/proxy-rebuild", @@ -1245,16 +1347,35 @@ const NAMED_ROUTES: &[NamedRoute] = &[ // POST is blocked by CORS and the guard navigates here for a 302 instead. primary_methods: &[Method::GET, Method::POST], handler: NamedRouteHandler::FirstPartyProxyRebuild, + route_class: RouteClass::Other, }, ]; +/// Wraps [`execute_named`], attaching a [`RouteMetadata`] extension carrying +/// `route_class` and the route-table pattern (`route_template`, verbatim, +/// with parameters left as placeholders) to every response the handler +/// produces — including its early-return diagnostic and setup-error arms, +/// since the attachment happens once around the whole future rather than in +/// each branch. fn named_route_handler( state: Arc, handler: NamedRouteHandler, + route_class: RouteClass, + route_template: &'static str, ) -> impl Fn(RequestContext) -> HandlerFuture + Clone + Send + Sync + 'static { move |ctx: RequestContext| { let state = Arc::clone(&state); - Box::pin(execute_named(state, ctx, handler)) + Box::pin(async move { + execute_named(state, ctx, handler) + .await + .map(|mut response| { + response.extensions_mut().insert(RouteMetadata { + route_class, + route_template: route_template.to_owned(), + }); + response + }) + }) } } @@ -1315,7 +1436,12 @@ impl TrustedServerApp { router = router.route( route.path, method.clone(), - named_route_handler(Arc::clone(state), route.handler), + named_route_handler( + Arc::clone(state), + route.handler, + route.route_class, + route.path, + ), ); } @@ -1374,19 +1500,23 @@ mod tests { use std::time::Duration; use super::{ - AppState, AuctionDispatch, EcContext, EdgeCacheHeader, HandlerFuture, NAMED_ROUTES, - NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RuntimeStoreConfig, - TrustedServerApp, build_orchestrator_with_plan, build_per_request_services, - build_state_from_settings, compile_auction_plan, handle_publisher_request, - publisher_response_into_streaming_response, startup_error_router, + AppState, AuctionDispatch, EcContext, EdgeCacheHeader, EidSyncSource, HandlerFuture, + NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RouteClass, + RouteMetadata, RuntimeStoreConfig, TSJS_ROUTE_TEMPLATE, TrustedServerApp, + build_orchestrator_with_plan, build_per_request_services, build_state_from_settings, + compile_auction_plan, handle_publisher_request, publisher_response_into_streaming_response, + startup_error_router, }; use base64::Engine as _; use bytes::Bytes; - use edgezero_core::app::Hooks as _; + use edgezero_core::app::{Hooks as _, StoreMetadata}; use edgezero_core::body::Body; use edgezero_core::context::RequestContext; use edgezero_core::env_config::EnvConfig; - use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; + use edgezero_core::http::{ + HeaderValue, Method, Request, Response, StatusCode, header, request_builder, + response_builder, + }; use edgezero_core::key_value_store::NoopKvStore; use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; @@ -1394,21 +1524,23 @@ mod tests { use error_stack::Report; use futures::executor::block_on; - use serde_json::json; - use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; + use trusted_server_core::constants::{HEADER_X_GEO_COUNTRY, HEADER_X_GEO_INFO_AVAILABLE}; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::error::TrustedServerError; + use trusted_server_core::geo::GeoLookupState; use trusted_server_core::integrations::{ HeaderMutation, IntegrationRegistry, IntegrationRequestFilter, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; use trusted_server_core::platform::{ - ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpClient, - PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, PlatformResponse, - PlatformSelectResult, PlatformTemplateCache, PlatformTemplateCacheReservation, - RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, - TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, + ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformGeo, + PlatformHttpClient, PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, PlatformTemplateCache, + PlatformTemplateCacheReservation, RuntimeServices, TemplateCacheError, TemplateCacheKey, + TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, + TemplateMetadata, }; + use trusted_server_core::request_timing::RequestTimings; use trusted_server_core::settings::Settings; #[test] @@ -1487,53 +1619,6 @@ mod tests { assert_eq!(stores.secret_store_name.as_ref(), "trusted_server_secrets"); } - fn settings_with_missing_consent_store() -> Settings { - Settings::from_toml( - r#" - [[handlers]] - path = "^/(_ts/)?admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [proxy] - allowed_domains = ["*.example", "*.example.com"] - - [ec] - passphrase = "test-passphrase-at-least-32-bytes!!" - - [request_signing] - enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - - [consent] - consent_store = "missing-consent-store" - - [integrations.prebid] - enabled = true - external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" - - [integrations.datadome] - enabled = true - - [auction] - enabled = true - [auction.providers.prebid] - protocol = "openrtb-2.6" - profile = "prebid-server" - endpoint = "https://test-prebid.com/openrtb2/auction" - timeout_ms = 2000 - "#, - ) - .expect("should parse EdgeZero app test settings") - } - fn app_state_for_settings(settings: Settings) -> Arc { build_state_from_settings(settings).expect("should build app state from settings") } @@ -1604,6 +1689,33 @@ mod tests { TrustedServerApp::routes_for_state(&state) } + #[test] + fn trusted_server_app_declares_runtime_store_metadata() { + let stores = TrustedServerApp::stores(); + + assert_eq!( + stores.config, + Some(StoreMetadata { + default: "trusted_server_config", + ids: &["trusted_server_config"], + }) + ); + assert_eq!( + stores.kv, + Some(StoreMetadata { + default: "trusted_server_kv", + ids: &["trusted_server_kv"], + }) + ); + assert_eq!( + stores.secrets, + Some(StoreMetadata { + default: "trusted_server_secrets", + ids: &["trusted_server_secrets"], + }) + ); + } + #[test] fn per_request_services_register_the_fastly_template_assembler() { let state = build_state_from_settings(test_settings()).expect("should build test state"); @@ -1634,12 +1746,12 @@ mod tests { ); } - /// Builds a router whose `AppState` uses a registry containing the given - /// request filters (and no routes), so dispatch-level request-filter - /// behavior can be exercised without a real integration. - fn router_with_request_filters( + /// Builds an `AppState` whose registry contains the given request + /// filters (and no routes), so dispatch-level request-filter behavior can + /// be exercised without a real integration. + fn state_with_request_filters( filters: Vec>, - ) -> RouterService { + ) -> Arc { let settings = test_settings(); let plan = Arc::new( trusted_server_core::auction::compile_auction_plan(&settings) @@ -1651,7 +1763,7 @@ mod tests { let registry = IntegrationRegistry::from_request_filters(filters); let default_kv_store = Arc::new(crate::platform::UnavailableKvStore) as Arc; - let state = Arc::new(super::AppState { + Arc::new(super::AppState { auction_telemetry_sink: Arc::new( trusted_server_core::auction::NoopAuctionTelemetrySink, ), @@ -1659,8 +1771,15 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, - }); - TrustedServerApp::routes_for_state(&state) + }) + } + + /// Builds a router on top of [`state_with_request_filters`] so + /// dispatch-level request-filter behavior can be exercised end-to-end. + fn router_with_request_filters( + filters: Vec>, + ) -> RouterService { + TrustedServerApp::routes_for_state(&state_with_request_filters(filters)) } /// Continues routing while mutating the request and emitting a response @@ -2312,6 +2431,181 @@ mod tests { ); } + /// `Authorization: Basic` header value for `test_settings()`'s + /// `^/_ts/admin` handler (`admin` / `admin-pass`). + fn admin_basic_auth_header() -> edgezero_core::http::HeaderValue { + let credentials = base64::engine::general_purpose::STANDARD.encode("admin:admin-pass"); + format!("Basic {credentials}") + .parse() + .expect("should parse basic-auth header value") + } + + #[test] + fn named_route_attaches_the_table_pattern_verbatim_even_with_a_real_id_in_the_path() { + // A named-route response must carry the route-TABLE pattern + // (`{id}` left as a placeholder), never the caller's actual matched + // path segment — this is what keeps a real EC identifier out of + // access telemetry, independent of anything the row-serialization + // layer does. + let router = test_router(); + let ec_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"; + let mut req = empty_request(Method::GET, &format!("/_ts/admin/ec/{ec_id}")); + req.headers_mut() + .insert(header::AUTHORIZATION, admin_basic_auth_header()); + let response = route(&router, req); + + let metadata = response + .extensions() + .get::() + .expect("named-route responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Ec); + assert_eq!(metadata.route_template, "/_ts/admin/ec/{id}"); + assert!( + !metadata.route_template.contains(ec_id), + "the attached template must never contain the matched id" + ); + } + + #[test] + fn named_route_attaches_metadata_even_on_a_read_only_diagnostic_early_return() { + // AdminEidsLookup is handled by an early-return arm inside + // execute_named, before the normal EC lifecycle runs (see the + // "read-only diagnostics" comment there). named_route_handler wraps + // the whole future, so the attachment must still happen here too. + let router = test_router(); + let mut req = empty_request(Method::GET, "/_ts/admin/eids"); + req.headers_mut() + .insert(header::AUTHORIZATION, admin_basic_auth_header()); + let response = route(&router, req); + + let metadata = response + .extensions() + .get::() + .expect("even a read-only diagnostic early-return response should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Ec); + assert_eq!(metadata.route_template, "/_ts/admin/eids"); + } + + #[test] + fn tsjs_fallback_attaches_tsjs_route_metadata() { + let router = test_router(); + let response = route( + &router, + empty_request(Method::GET, "/static/tsjs=tsjs-unified.min.js"), + ); + + let metadata = response + .extensions() + .get::() + .expect("tsjs fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Tsjs); + assert_eq!(metadata.route_template, TSJS_ROUTE_TEMPLATE); + } + + #[test] + fn integration_proxy_fallback_attaches_integration_proxy_route_metadata() { + // test_settings() enables the prebid integration, which registers a + // proxy route at /integrations/prebid/bundle.js. + let router = test_router(); + let response = route( + &router, + empty_request(Method::GET, "/integrations/prebid/bundle.js"), + ); + + let metadata = response + .extensions() + .get::() + .expect("integration-proxy fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::IntegrationProxy); + assert_eq!( + metadata.route_template, "/integrations/prebid/bundle.js", + "should carry the registered route pattern verbatim, not a classifier output" + ); + } + + #[test] + fn publisher_fallback_attaches_publisher_html_route_metadata() { + let router = test_router(); + let response = route(&router, empty_request(Method::GET, "/news/some-article")); + + let metadata = response + .extensions() + .get::() + .expect("publisher fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::PublisherHtml); + assert_eq!( + metadata.route_template, "/other/*", + "the default empty section allowlist should collapse publisher paths" + ); + } + + fn browser_request(method: Method, path: &str, fetch_destination: &str) -> Request { + let mut request = empty_request(method, path); + request.headers_mut().insert( + "sec-fetch-dest", + HeaderValue::from_bytes(fetch_destination.as_bytes()) + .expect("should parse fetch destination"), + ); + request.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + request + } + + fn eid_sync_source_of(response: &Response) -> Option { + response + .extensions() + .get::() + .expect("response should carry EC finalization state") + .ec_context + .eid_sync_source() + } + + #[test] + fn dispatch_limits_returning_user_eid_sync_to_navigation_and_auction() { + let router = test_router(); + + let navigation = route( + &router, + browser_request(Method::GET, "/article", "document"), + ); + assert_eq!( + eid_sync_source_of(&navigation), + Some(EidSyncSource::Navigation) + ); + + let mut navigation_without_browser_signals = + browser_request(Method::GET, "/another-article", "document"); + navigation_without_browser_signals + .extensions_mut() + .remove::(); + let navigation_without_browser_signals = route(&router, navigation_without_browser_signals); + assert_eq!( + eid_sync_source_of(&navigation_without_browser_signals), + Some(EidSyncSource::Navigation), + "route classification should not depend on EC generation's browser gate" + ); + + let auction = route(&router, browser_request(Method::POST, "/auction", "empty")); + assert_eq!(eid_sync_source_of(&auction), Some(EidSyncSource::Auction)); + + for request in [ + browser_request(Method::GET, "/static/tsjs=prebid", "script"), + browser_request(Method::GET, "/analytics.gif", "image"), + browser_request(Method::GET, "/integrations/prebid/bundle.js", "script"), + ] { + let response = route(&router, request); + assert_eq!( + eid_sync_source_of(&response), + None, + "static, analytics, and integration requests must not persist EID cookies" + ); + } + } + #[test] fn browser_device_signals_from_extension_reach_ec_finalize_state() { // Regression guard for the EdgeZero JA4/H2 signal loss: `edgezero_main` @@ -2574,27 +2868,6 @@ mod tests { ); } - #[test] - fn dispatch_auction_with_missing_consent_store_returns_503() { - let state = app_state_for_settings(settings_with_missing_consent_store()); - let router = TrustedServerApp::routes_for_state(&state); - let body = json!({ "adUnits": [] }).to_string(); - let req = request_builder() - .method(Method::POST) - .uri("/auction") - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(body)) - .expect("should build auction request"); - - let response = route(&router, req); - - assert_eq!( - response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "auction route should fail closed when configured consent store cannot be opened" - ); - } - #[test] fn dispatch_unregistered_method_returns_405_at_router_level() { // Documents the known router-level behavior for verbs outside the @@ -2626,54 +2899,6 @@ mod tests { ); } - #[test] - fn edgezero_missing_consent_store_breaks_only_consent_routes() { - let state = app_state_for_settings(settings_with_missing_consent_store()); - let router = TrustedServerApp::routes_for_state(&state); - - let admin_response = route( - &router, - empty_request(Method::POST, "/_ts/admin/keys/rotate"), - ); - assert_eq!( - admin_response.status(), - StatusCode::UNAUTHORIZED, - "admin auth behavior should not depend on consent KV availability" - ); - - let auction_request = request_builder() - .method(Method::POST) - .uri("/auction") - .body(Body::from(r#"{"adUnits":[]}"#)) - .expect("should build auction request"); - let auction_response = route(&router, auction_request); - assert_eq!( - auction_response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "auction should fail closed when configured consent KV cannot be opened" - ); - - let publisher_response = route(&router, empty_request(Method::GET, "/articles/example")); - assert_eq!( - publisher_response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "publisher fallback should fail closed when configured consent KV cannot be opened" - ); - - // Integration routes must NOT require the consent KV — runtime_services_for_consent_route - // is wired only into the publisher and auction branches of dispatch_fallback, not into - // the integration proxy branch. A missing consent store must not 503 integration routes. - let integration_response = route( - &router, - empty_request(Method::GET, "/integrations/datadome/tags.js"), - ); - assert_ne!( - integration_response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "integration routes should be unaffected by a missing consent KV store" - ); - } - #[test] fn dispatch_fallback_asset_route_skips_ec_finalization() { // Parity guard for the configured asset-route fallback: a GET matching a @@ -2732,6 +2957,61 @@ mod tests { ); } + #[test] + fn asset_fallback_carries_geo_state_without_ec_finalize_state() { + // The asset-route fallback is the one exit path that skips + // EcFinalizeState but must still carry GeoLookupState, since + // build_ec_request_state (and its geo lookup) already ran before the + // asset route was matched. Without this, the finalize step would + // silently repeat the lookup for every asset request. + let settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [proxy] + + [[proxy.asset_routes]] + prefix = "/.image/" + origin_url = "https://assets.example.com" + "#, + ) + .expect("should parse asset-route settings"); + let state = build_state_from_settings(settings).expect("should build state"); + let router = TrustedServerApp::routes_for_state(&state); + + let response = route(&router, empty_request(Method::GET, "/.image/banner.png")); + + assert!( + response.extensions().get::().is_some(), + "asset-route responses should still carry GeoLookupState even though \ + EC finalization is skipped" + ); + assert!( + response + .extensions() + .get::() + .is_none(), + "asset-route responses must skip EC finalization (no EcFinalizeState)" + ); + } + struct FixedBackend; impl PlatformBackend for FixedBackend { @@ -3148,6 +3428,7 @@ mod tests { req, asset_route, &effects, + trusted_server_core::geo::GeoLookupState::NotAttempted, )); assert_eq!( @@ -3191,6 +3472,193 @@ mod tests { ); } + /// A [`PlatformGeo`] stub that counts every `lookup` call and always + /// returns the same canned result, used to prove the request-phase geo + /// lookup is never repeated during finalize. + struct CountingGeo { + calls: Arc, + result: Option, + } + + impl PlatformGeo for CountingGeo { + fn lookup(&self, _: Option) -> Result, Report> { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(self.result.clone()) + } + } + + fn sample_geo_info() -> GeoInfo { + GeoInfo { + city: "Testville".to_string(), + country: "US".to_string(), + continent: "NorthAmerica".to_string(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + } + } + + fn runtime_services_with_geo(geo: Arc) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(geo) + .client_info(ClientInfo::default()) + .build() + } + + #[test] + fn finalize_reuses_request_phase_geo_without_second_lookup() { + // Dispatching a publisher route runs build_ec_request_state, which + // attempts the geo lookup once and carries the result via + // GeoLookupState. The finalize step (resolve_geo_for_response) must + // reuse that carried value instead of calling the geo backend again. + let calls = Arc::new(AtomicUsize::new(0)); + let geo = Arc::new(CountingGeo { + calls: Arc::clone(&calls), + result: Some(sample_geo_info()), + }); + let state = app_state_for_settings(test_settings()); + let services = runtime_services_with_geo(geo); + let req = empty_request(Method::GET, "/some-page"); + + let response = block_on(super::dispatch_fallback(&state, &services, req)); + + let carried = response + .extensions() + .get::() + .cloned() + .expect("dispatch should attach GeoLookupState"); + assert!( + matches!(carried, GeoLookupState::Resolved(_)), + "a successful lookup should carry Resolved" + ); + + let geo_info = + crate::middleware::resolve_geo_for_response(&response, &carried, None, |_| { + panic!("finalize must not repeat a resolved geo lookup"); + }); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "only the request-phase lookup should have run" + ); + + let mut response = response; + geo_info + .expect("geo info should have resolved") + .set_response_headers(&mut response); + assert!( + response.headers().get(HEADER_X_GEO_COUNTRY).is_some(), + "x-geo-country should still be set on the response after reusing the carried geo" + ); + } + + #[test] + fn failed_lookup_is_not_retried() { + // When the request-phase lookup fails (returns None), dispatch must + // carry GeoLookupState::Attempted rather than NotAttempted, and + // finalize must not retry it. + let calls = Arc::new(AtomicUsize::new(0)); + let geo = Arc::new(CountingGeo { + calls: Arc::clone(&calls), + result: None, + }); + let state = app_state_for_settings(test_settings()); + let services = runtime_services_with_geo(geo); + let req = empty_request(Method::GET, "/some-page"); + + let response = block_on(super::dispatch_fallback(&state, &services, req)); + + let carried = response + .extensions() + .get::() + .cloned() + .expect("dispatch should attach GeoLookupState even for a failed lookup"); + assert!( + matches!(carried, GeoLookupState::Attempted), + "a failed lookup should carry Attempted, not Resolved or NotAttempted" + ); + + let geo_info = + crate::middleware::resolve_geo_for_response(&response, &carried, None, |_| { + panic!("finalize must not retry a failed geo lookup"); + }); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "only the request-phase lookup should have run" + ); + assert!( + geo_info.is_none(), + "no geo info should be available after a failed lookup" + ); + } + + #[test] + fn filter_span_recorded_when_request_filter_runs() { + // The Filter phase span should only be recorded when the registry + // actually has a request filter registered, so unconfigured + // deployments omit ts-filter from the Server-Timing header entirely. + let state = state_with_request_filters(vec![Arc::new(RecordingRequestFilter)]); + let services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let mut req = empty_request(Method::GET, "/some-page"); + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + let _ = block_on(super::run_pre_route_filters( + &state, &services, &mut req, None, + )); + + assert!( + timings.snapshot().filter_ms.is_some(), + "should record the Filter phase span when a request filter is registered and runs" + ); + } + + #[test] + fn filter_span_not_recorded_when_no_request_filters_registered() { + // Mirror test: an empty registry must never record the Filter span, + // even though run_pre_route_filters still runs (as a no-op loop). + let state = state_with_request_filters(Vec::new()); + let services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let mut req = empty_request(Method::GET, "/some-page"); + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + let _ = block_on(super::run_pre_route_filters( + &state, &services, &mut req, None, + )); + + assert!( + timings.snapshot().filter_ms.is_none(), + "should omit the Filter phase span when no request filters are registered" + ); + } + #[test] fn dispatch_runs_request_filter_and_threads_response_effects() { // Regression guard for the EdgeZero request-filter bypass: the publisher @@ -3294,22 +3762,79 @@ mod tests { } #[test] - fn filter_short_circuit_response_is_not_recovery_eligible() { + fn filter_short_circuit_response_is_not_eligible_for_eid_persistence() { // A request-filter short circuit (e.g. a DataDome challenge/block) must - // not authorize orphan recovery even for a would-be publisher - // navigation: no publisher page was served. + // not authorize orphan recovery or EID persistence. No publisher page + // or auction was served, so the challenged request must not write EIDs. + // Explicit consent withdrawal remains independently eligible. let router = router_with_request_filters(vec![Arc::new(ChallengeRequestFilter)]); - let response = route(&router, browser_navigation_request("/some-page")); + let navigation = route(&router, browser_navigation_request("/some-page")); assert_eq!( - response.status(), + navigation.status(), StatusCode::FORBIDDEN, - "the challenge filter should short-circuit routing" + "the challenge filter should short-circuit navigation routing" ); assert!( - !recovery_eligible_of(&response), + !recovery_eligible_of(&navigation), "a short-circuit filter response must not authorize orphan recovery" ); + assert_eq!( + eid_sync_source_of(&navigation), + None, + "a challenged navigation must not authorize EID persistence" + ); + + let auction = route(&router, browser_request(Method::POST, "/auction", "empty")); + assert_eq!( + auction.status(), + StatusCode::FORBIDDEN, + "the challenge filter should short-circuit auction routing" + ); + assert_eq!( + eid_sync_source_of(&auction), + None, + "a challenged auction must not authorize EID persistence" + ); + } + + /// Joins every instance of a response header into one comma-separated + /// string (mirroring how a client sees repeated header fields), or + /// `None` if the header is absent. + fn response_header(response: &Response, name: &str) -> Option { + let values: Vec<&str> = response + .headers() + .get_all(name) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + if values.is_empty() { + None + } else { + Some(values.join(", ")) + } + } + + #[test] + fn server_timing_emitted_on_private_response_when_enabled() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(Body::empty()) + .expect("should build a private response fixture"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, true); + + let header = response_header(&response, "server-timing").expect("should emit header"); + assert!( + header.contains("ts-total;dur="), + "should carry the stored total: {header}" + ); + assert_eq!( + header.matches("ts-total").count(), + 1, + "should emit exactly one TS-owned metric set" + ); } #[test] @@ -3325,4 +3850,85 @@ mod tests { "an origin-start failure must not authorize orphan recovery" ); } + + #[test] + fn server_timing_absent_when_flag_off() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(Body::empty()) + .expect("should build a private response fixture"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, false); + + assert!( + response_header(&response, "server-timing").is_none(), + "should not emit server-timing when the flag is off" + ); + } + + #[test] + fn server_timing_absent_on_cacheable_responses() { + // tsjs route policy: public, long max-age, immutable. + let mut tsjs_response = response_builder() + .header("cache-control", "public, max-age=31536000, immutable") + .body(Body::empty()) + .expect("should build a tsjs-style response fixture"); + // A bare shared-cacheable response with no private/no-store directive. + let mut public_response = response_builder() + .header("cache-control", "max-age=60") + .body(Body::empty()) + .expect("should build a bare max-age response fixture"); + + crate::apply_server_timing_header(&mut tsjs_response, &RequestTimings::new(), true); + crate::apply_server_timing_header(&mut public_response, &RequestTimings::new(), true); + + assert!( + response_header(&tsjs_response, "server-timing").is_none(), + "should not emit on the public immutable tsjs cache policy" + ); + assert!( + response_header(&public_response, "server-timing").is_none(), + "should not emit on a bare shared-cacheable max-age response" + ); + } + + #[test] + fn server_timing_absent_when_no_cache_control_header_exists() { + // The fail-closed case: absence of Cache-Control is not evidence of + // privacy, so emission must be suppressed rather than defaulted on. + let mut response = response_builder() + .body(Body::empty()) + .expect("should build a response with no cache-control header"); + + crate::apply_server_timing_header(&mut response, &RequestTimings::new(), true); + + assert!( + response_header(&response, "server-timing").is_none(), + "should not emit when the response carries no Cache-Control at all" + ); + } + + #[test] + fn preexisting_server_timing_values_survive() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .header("server-timing", "upstream;dur=1") + .body(Body::empty()) + .expect("should build a private response fixture carrying an upstream Server-Timing"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, true); + + let header = + response_header(&response, "server-timing").expect("should still carry a header"); + assert!( + header.contains("upstream;dur=1"), + "should preserve the pre-existing entry: {header}" + ); + assert!( + header.contains("ts-total"), + "should append the TS-owned set: {header}" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 367a3e33f..a1c04a79e 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1,5 +1,8 @@ use std::sync::Arc; +use rand::Rng as _; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + use edgezero_adapter_fastly::config_store::FastlyConfigStore as EdgeZeroFastlyConfigStore; use edgezero_adapter_fastly::request::into_core_request; use edgezero_adapter_fastly::runtime_env_config; @@ -13,7 +16,13 @@ use error_stack::Report; use fastly::http::Method as FastlyMethod; use fastly::{Request as FastlyRequest, Response as FastlyResponse}; +use trusted_server_core::access_telemetry::{ + AccessTelemetrySnapshot, RouteClass, RouteMetadata, access_event_row, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::constants::{ + ENV_FASTLY_IS_STAGING, ENV_FASTLY_POP, ENV_FASTLY_SERVICE_ID, ENV_FASTLY_SERVICE_VERSION, +}; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; @@ -22,10 +31,13 @@ use trusted_server_core::ec::pull_sync::{ }; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; +use trusted_server_core::geo::GeoLookupState; use trusted_server_core::integrations::RequestFilterEffects; use trusted_server_core::platform::PlatformGeo as _; -use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::platform::TimedKvStore; use trusted_server_core::proxy::{AssetProxyCachePolicy, stream_asset_body}; +use trusted_server_core::publisher::TemplateCacheResponseState; +use trusted_server_core::request_timing::{Phase, RequestTimings, append_server_timing_if_private}; use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; @@ -85,19 +97,18 @@ fn main() { } logging::init_logger(); - edgezero_main(req); + let env = runtime_env_config(TrustedServerApp::stores()); + let runtime_stores = RuntimeStoreConfig::from_env(&env); + edgezero_main(req, &runtime_stores); } /// Handles a request through the `EdgeZero` router path. -fn edgezero_main(mut req: FastlyRequest) { - let runtime_env = runtime_env_config(TrustedServerApp::stores()); - let runtime_stores = RuntimeStoreConfig::from_env(&runtime_env); - +fn edgezero_main(mut req: FastlyRequest, runtime_stores: &RuntimeStoreConfig) { // Short-circuit the JA4 debug probe before app construction. Must run here // because TLS/JA4 accessors are only available on FastlyRequest before // conversion to edgezero types. if req.get_method() == FastlyMethod::GET && req.get_path() == "/_ts/debug/ja4" { - match load_settings_from_config_store(&runtime_stores) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) if settings.debug.ja4_endpoint_enabled => { build_ja4_debug_response(&req).send_to_client(); } @@ -114,20 +125,43 @@ fn edgezero_main(mut req: FastlyRequest) { return; } - let config_store = - match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { - Ok(cs) => cs, - Err(e) => { - log::error!("failed to open config store: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; + let timings = RequestTimings::new(); - let (app, app_state) = TrustedServerApp::build_app_with_state(&runtime_stores); + let (config_store, app, app_state) = { + let _appbuild = timings.span(Phase::AppBuild); + let config_store = + match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { + Ok(cs) => cs, + Err(e) => { + log::error!("failed to open config store: {e}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + return; + } + }; + let (app, app_state) = TrustedServerApp::build_app_with_state(runtime_stores); + (config_store, app, app_state) + }; let settings_snapshot = app_state.as_ref().map(|state| Arc::clone(&state.settings)); + let server_timing_enabled = settings_snapshot + .as_deref() + .is_some_and(|settings| settings.observability.server_timing_enabled); + // Both read once here rather than at each `send_edgezero_response` call + // site: if `app_state` failed to build, there is no settings snapshot to + // read them from at all, so every call site would need the same + // degraded-mode fallback. `access_sample_rate` defaults to `0.0` (never + // sampled in) and `publisher_domain` to `"unknown"` in that case. + let access_sample_rate = settings_snapshot + .as_deref() + .map_or(0.0, |settings| settings.tinybird.access_sample_rate); + let access_telemetry_enabled = settings_snapshot + .as_deref() + .is_some_and(|settings| settings.tinybird.enabled && settings.tinybird.access_enabled); + let publisher_domain = settings_snapshot.as_deref().map_or_else( + || "unknown".to_owned(), + |settings| settings.publisher.domain.clone(), + ); let trusted_client_ip = settings_snapshot .as_deref() .and_then(|settings| settings.trusted_client_ip.as_ref()); @@ -147,6 +181,10 @@ fn edgezero_main(mut req: FastlyRequest) { req.set_header("fastly-ssl", "1"); } + // Capture the method before dispatch consumes the request. The resolved + // client IP is retained below in `ClientInfo`. + let request_method = req.get_method_str().to_owned(); + // Strip any client-supplied x-ts-tls-* headers before injecting the trusted // values from the Fastly SDK. Must run after sanitize_fastly_forwarded_headers. req.remove_header("x-ts-tls-protocol"); @@ -178,6 +216,7 @@ fn edgezero_main(mut req: FastlyRequest) { core_req.extensions_mut().insert(config_store); core_req.extensions_mut().insert(device_signals); core_req.extensions_mut().insert(client_info); + core_req.extensions_mut().insert(timings.clone()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -196,14 +235,34 @@ fn edgezero_main(mut req: FastlyRequest) { let ec_state = response.extensions_mut().remove::(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); + // Read rather than pop: the access-telemetry snapshot built later in + // `send_edgezero_response` reads this same extension, so it must still + // be attached to `response` at that point. + let geo_lookup_state = response + .extensions() + .get::() + .cloned() + .unwrap_or(GeoLookupState::NotAttempted); if !take_finalize_sentinel(&mut response) { if let Some(settings) = settings_snapshot.as_deref() { - apply_entry_point_finalize_headers(settings, &mut response, client_ip); + apply_entry_point_finalize_headers( + settings, + &mut response, + client_ip, + &geo_lookup_state, + &timings, + ); } else { - match load_settings_from_config_store(&runtime_stores) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) => { - apply_entry_point_finalize_headers(&settings, &mut response, client_ip); + apply_entry_point_finalize_headers( + &settings, + &mut response, + client_ip, + &geo_lookup_state, + &timings, + ); } Err(e) => { log::warn!("entry-point finalize skipped: failed to reload settings: {e:?}"); @@ -218,10 +277,30 @@ fn edgezero_main(mut req: FastlyRequest) { if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response, &timings) { Ok(partner_registry) => { - send_edgezero_response(response, request_filter_effects.as_ref()); - run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method.clone(), + publisher_domain: publisher_domain.clone(), + access_sample_rate, + access_telemetry_enabled, + }, + ); + run_post_send_steps( + || { + run_edgezero_pull_sync_after_send( + settings, + &partner_registry, + &ec_state, + ) + }, + || emit_access_telemetry_after_send(settings, &outcome, &timings), + ); return; } Err(e) => { @@ -231,15 +310,36 @@ fn edgezero_main(mut req: FastlyRequest) { } } } else { - match load_settings_from_config_store(&runtime_stores) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize( + &settings, + &mut ec_state, + &mut response, + &timings, + ) { Ok(partner_registry) => { - send_edgezero_response(response, request_filter_effects.as_ref()); - run_edgezero_pull_sync_after_send( - &settings, - &partner_registry, - &ec_state, + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method.clone(), + publisher_domain: publisher_domain.clone(), + access_sample_rate, + access_telemetry_enabled, + }, + ); + run_post_send_steps( + || { + run_edgezero_pull_sync_after_send( + &settings, + &partner_registry, + &ec_state, + ); + }, + || emit_access_telemetry_after_send(&settings, &outcome, &timings), ); return; } @@ -257,7 +357,27 @@ fn edgezero_main(mut req: FastlyRequest) { } } - send_edgezero_response(response, request_filter_effects.as_ref()); + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method, + publisher_domain, + access_sample_rate, + access_telemetry_enabled, + }, + ); + // The asset/admin/error fallback path: no `EcFinalizeState` (or the ec + // finalize branch above failed), so there is no pull-sync dispatch here + // at all — telemetry is the only post-send step. When `app_state` never + // built there is nothing to emit either: `access_telemetry_enabled` was + // necessarily false without a settings snapshot, so the outcome carries + // no access snapshot, and reloading settings here could not change that. + if let Some(settings) = settings_snapshot.as_deref() { + emit_access_telemetry_after_send(settings, &outcome, &timings); + } } fn edge_error_response(error: EdgeError) -> HttpResponse { @@ -285,24 +405,36 @@ fn apply_entry_point_finalize_headers( settings: &Settings, response: &mut HttpResponse, client_ip: Option, + geo_state: &GeoLookupState, + timings: &RequestTimings, ) { - let geo_info = resolve_geo_for_response(response, client_ip, |client_ip| { + let geo_info = resolve_geo_for_response(response, geo_state, client_ip, |client_ip| { + let _span = timings.span(Phase::Geo); FastlyPlatformGeo.lookup(client_ip).unwrap_or_else(|e| { log::warn!("entry-point geo lookup failed: {e}"); None }) }); apply_finalize_headers(settings, geo_info.as_ref(), response); + + // This path runs only when the middleware chain was bypassed (e.g. a + // router-level 404/405 for an unregistered method), so `geo_state` may + // still be `NotAttempted` even after a fresh lookup just ran above. + // Write the resolved outcome back so the access-telemetry snapshot built + // later in `send_edgezero_response` sees what was actually looked up; + // 401 handling lives in the shared helper. + middleware::write_back_geo_lookup_state(response, geo_info.as_ref()); } fn apply_edgezero_ec_finalize( settings: &Settings, ec_state: &mut EcFinalizeState, response: &mut HttpResponse, + timings: &RequestTimings, ) -> Result> { let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; let finalize_kv_graph = if ec_state.use_finalize_kv { - maybe_identity_graph(settings) + identity_graph_with_timing(settings, timings) } else { None }; @@ -323,11 +455,270 @@ fn run_edgezero_pull_sync_after_send( partner_registry: &PartnerRegistry, ec_state: &EcFinalizeState, ) { - if ec_state.is_real_browser - && let Some(context) = build_pull_sync_context(&ec_state.ec_context) - { - run_pull_sync_after_send(settings, partner_registry, &context, &ec_state.services); + if !ec_state.is_real_browser { + return; } + + let prepared_context = build_pull_sync_context(&ec_state.ec_context, partner_registry); + let Some((context, kv)) = + prepare_pull_sync_after_send(prepared_context, || require_identity_graph(settings)) + else { + return; + }; + + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + dispatch_pull_sync( + settings, + &kv, + partner_registry, + &limiter, + &context, + &ec_state.services, + ); +} + +fn prepare_pull_sync_after_send( + context: Option, + graph_factory: F, +) -> Option<(PullSyncContext, KvIdentityGraph)> +where + F: FnOnce() -> Result>, +{ + let context = context?; + let kv = match graph_factory() { + Ok(kv) => kv, + Err(err) => { + log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); + return None; + } + }; + Some((context, kv)) +} + +/// Runs the post-send steps in their contract order: EC identity pull-sync +/// first, then access-telemetry emission. +/// +/// Every `edgezero_main` site that has both steps routes through this +/// function, so the ordering is owned in exactly one place and the +/// sequence test can instrument it; `request_elapsed` is already stamped +/// before either step because `send_edgezero_response` stamps it before +/// returning. +fn run_post_send_steps(pull_sync: impl FnOnce(), emit_access_telemetry: impl FnOnce()) { + pull_sync(); + emit_access_telemetry(); +} + +/// Builds and emits the access-telemetry row for one delivered response, +/// when access telemetry is enabled and this request is sampled in. +/// +/// Called last at every `send_edgezero_response` call site in +/// [`edgezero_main`] — after `run_edgezero_pull_sync_after_send` on the two +/// EC-finalized paths, and directly after send on the asset/admin/error +/// fallback path, which never builds an [`EcFinalizeState`] or route-scoped +/// `RuntimeServices` at all. The Tinybird transport context is therefore +/// constructed fresh from `settings` here rather than threaded through +/// either of those per-route types, so every response class can emit. +/// +/// Sampled-out requests return silently — that is the expected, high-volume +/// case and not worth a log line. The sampling roll uses the rate stored on +/// the snapshot itself, so the emission probability always matches the +/// row's `sample_rate` column by construction. Every other drop (row +/// build, token load, send, or non-2xx status — all folded into +/// `emit_access_event`'s `Result`) logs exactly one warning naming the +/// reason. +fn emit_access_telemetry_after_send( + settings: &Settings, + outcome: &DeliveryOutcome, + timings: &RequestTimings, +) { + if !settings.tinybird.enabled || !settings.tinybird.access_enabled { + return; + } + + // No snapshot means access telemetry was disabled when the response + // was sent (the flag is read once, before dispatch); nothing to emit. + let Some(snapshot) = &outcome.snapshot else { + return; + }; + + let since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let epoch_ms = u64::try_from(since_epoch.as_millis()).unwrap_or(u64::MAX); + // Sample with the rate stored on the snapshot itself — the same value + // serialized into the row's `sample_rate` column — so the emission + // probability and the row's claimed rate cannot diverge, which the + // documented `sum(1.0 / sample_rate)` volume estimator depends on. + let roll = rand::thread_rng().r#gen::(); + if !tinybird::sampled_in(snapshot.sample_rate, roll) { + return; + } + + let row = access_event_row(snapshot, &timings.snapshot(), epoch_ms); + let target = tinybird::TinybirdEventsTarget::from_access_config(settings.tinybird.clone()); + let result = futures::executor::block_on(tinybird::emit_access_event( + &platform::FastlyPlatformHttpClient, + &target, + row, + )); + if let Err(error) = result { + log::warn!("access telemetry emission dropped: {error:?}"); + } +} + +/// Per-response context threaded into [`send_edgezero_response`] so the +/// function stays at or under seven parameters. +struct SendContext { + /// The request's phase-timing collector. + timings: RequestTimings, + /// Whether `observability.server_timing_enabled` is set. + server_timing_enabled: bool, + /// The request's HTTP method, captured before the request was consumed + /// by dispatch. + method: String, + /// The configured publisher domain. + publisher_domain: String, + /// The configured access-telemetry sample rate. + access_sample_rate: f64, + /// Whether `tinybird.enabled` and `tinybird.access_enabled` were both + /// set when settings were first read. Gates building the + /// [`AccessTelemetrySnapshot`] at all: the snapshot costs env reads and + /// `String` allocations on the pre-send path, which a disabled + /// deployment (the default) should not pay. + access_telemetry_enabled: bool, +} + +/// Outcome of handing a finalized response to the client. +pub(crate) struct DeliveryOutcome { + /// Response body size in bytes. + #[allow(dead_code)] + pub bytes: u64, + /// Whether delivery completed or failed partway. Collected as + /// groundwork; not yet emitted on any surface. + #[allow(dead_code)] + pub result: DeliveryResult, + /// Access-telemetry dimensions captured for this response at the + /// freeze point. `None` when access telemetry was disabled at snapshot + /// time; the emitter treats that as nothing to send. + pub snapshot: Option, +} + +/// Whether [`send_edgezero_response`] completed delivery or failed partway. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum DeliveryResult { + /// The response was handed to the client in full. + Complete, + /// Delivery started but did not finish cleanly: some bytes reached the + /// client's transport before a stream error, or the transport could not + /// be closed cleanly after every byte was written. + Partial, + /// Delivery failed before any bytes reached the client. + Error, +} + +/// Thin Fastly-adapter wrapper around +/// [`append_server_timing_if_private`], the freeze point shared with the +/// Axum adapter's terminal timing layer. See that function's doc for the +/// emission rules (always stamps `mark_headers_ready`; appends rather than +/// overwrites; never promotes a response to shared-cacheable). +pub(crate) fn apply_server_timing_header( + response: &mut HttpResponse, + timings: &RequestTimings, + server_timing_enabled: bool, +) { + append_server_timing_if_private(response, timings, server_timing_enabled); +} + +/// A [`Write`](std::io::Write) wrapper that tallies bytes successfully written +/// to the inner writer. +/// +/// Wraps the client transport during a streaming drive so a truncated or +/// failed drive still reports how many bytes actually reached it, instead of +/// the placeholder `0` a failed/aborted drive would otherwise report. +struct CountingWriter { + inner: W, + bytes: u64, +} + +impl CountingWriter { + fn new(inner: W) -> Self { + Self { inner, bytes: 0 } + } + + /// Bytes successfully written to the inner writer so far. + fn bytes(&self) -> u64 { + self.bytes + } + + fn into_inner(self) -> W { + self.inner + } +} + +impl std::io::Write for CountingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.bytes = self.bytes.saturating_add(written as u64); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +/// Drives a streaming `EdgeZero` body through `output`, tallying bytes written +/// and timing the drive into `timings`. +/// +/// Stamps `resp_bytes` and `request_elapsed` immediately once the drive +/// returns — before the caller does anything transport-specific (finishing +/// the streaming body, logging) — so `request_elapsed` never includes that +/// work. Returns the counting writer (so the caller can recover both the +/// tallied byte count and the wrapped transport) alongside the drive's +/// result. +fn drive_streaming_body( + body: EdgeBody, + output: W, + timings: &RequestTimings, +) -> (CountingWriter, Result<(), Report>) { + let mut counting = CountingWriter::new(output); + let drive_started = Instant::now(); + let result = futures::executor::block_on(stream_asset_body(body, &mut counting)); + timings.record(Phase::Stream, drive_started.elapsed()); + timings.set_resp_bytes(counting.bytes()); + timings.mark_request_elapsed(); + (counting, result) +} + +/// Classifies a completed streaming drive into a [`DeliveryResult`]. +/// +/// A drive that failed after writing at least one byte delivered a truncated +/// response rather than nothing at all, so it is [`DeliveryResult::Partial`], +/// not [`DeliveryResult::Error`]. +/// +/// The `Ok(())` arm exists for the classifier's totality, not for the +/// production caller: `send_edgezero_response` consumes this value only in +/// its `Err` branch and re-derives the success outcome from +/// `streaming_body.finish()`. +fn classify_stream_delivery( + drive_result: &Result<(), Report>, + bytes: u64, +) -> DeliveryResult { + match drive_result { + Ok(()) => DeliveryResult::Complete, + Err(_) if bytes > 0 => DeliveryResult::Partial, + Err(_) => DeliveryResult::Error, + } +} + +/// Stamps `resp_bytes`/`request_elapsed` for an already-materialized body, +/// immediately before it is handed to the Fastly client transport, and +/// returns its byte length. +fn record_buffered_delivery(body: &EdgeBody, timings: &RequestTimings) -> u64 { + let bytes = u64::try_from(body.as_bytes().map(<[u8]>::len).unwrap_or(0)).unwrap_or(u64::MAX); + timings.set_resp_bytes(bytes); + timings.mark_request_elapsed(); + bytes } /// Sends a finalized `EdgeZero` response to the client. @@ -338,8 +729,24 @@ fn run_edgezero_pull_sync_after_send( fn send_edgezero_response( mut response: HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, -) { + context: &SendContext, +) -> DeliveryOutcome { apply_terminal_response_effects(&mut response, request_filter_effects); + apply_server_timing_header( + &mut response, + &context.timings, + context.server_timing_enabled, + ); + + // Built right after the freeze point and before `into_parts()` + // consumes `response`: nothing else survives to post-send on every + // path (the request was consumed by dispatch, and `EcFinalizeState` + // is absent on asset, admin, and error paths). Skipped entirely when + // access telemetry is disabled, so the default configuration pays no + // env reads or allocations here. + let snapshot = context + .access_telemetry_enabled + .then(|| build_access_telemetry_snapshot(&response, context)); let (parts, body) = response.into_parts(); @@ -349,25 +756,133 @@ fn send_edgezero_response( parts, EdgeBody::empty(), )); - let mut streaming_body = skeleton.stream_to_client(); - match futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) { - Ok(()) => { - if let Err(e) = streaming_body.finish() { + let (counting, drive_result) = + drive_streaming_body(body, skeleton.stream_to_client(), &context.timings); + let bytes = counting.bytes(); + let streaming_body = counting.into_inner(); + // Computed before `drive_result` is matched by value below, since + // the `Err` arm there moves its `Report` out. + let result = classify_stream_delivery(&drive_result, bytes); + match drive_result { + Ok(()) => match streaming_body.finish() { + Ok(()) => DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot, + }, + Err(e) => { + // Every byte was handed to the transport (the drive + // above returned Ok), but the transport itself could + // not close cleanly — the client may still see a + // truncated response. log::error!("failed to finish EdgeZero streaming body: {e}"); + DeliveryOutcome { + bytes, + result: DeliveryResult::Partial, + snapshot, + } } - } + }, Err(e) => { log::error!("EdgeZero streaming failed: {e:?}"); drop(streaming_body); + DeliveryOutcome { + bytes, + result, + snapshot, + } } } } once => { + let bytes = record_buffered_delivery(&once, &context.timings); compat::to_fastly_response(HttpResponse::from_parts(parts, once)).send_to_client(); + DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot, + } + } + } +} + +/// Builds the [`AccessTelemetrySnapshot`] for `response` at the +/// `Server-Timing` freeze point. +/// +/// Reads route identity, geo country, and template-cache state from typed +/// response extensions rather than the headers those extensions back — +/// operator-configured response headers can override a managed header, so +/// reading a header here could silently drift from what actually happened. +/// Falls back to `"unknown"`/[`RouteClass::Other`] sentinels when an +/// extension was never attached (router-generated, asset, and other +/// responses that never passed through a `RouteMetadata`-attaching +/// wrapper). +fn build_access_telemetry_snapshot( + response: &HttpResponse, + context: &SendContext, +) -> AccessTelemetrySnapshot { + let (route_class, route_template) = match response.extensions().get::() { + Some(metadata) => (metadata.route_class, metadata.route_template.clone()), + None => (RouteClass::Other, "unknown".to_owned()), + }; + + let country = match response.extensions().get::() { + Some(GeoLookupState::Resolved(info)) => info.country.clone(), + Some(GeoLookupState::Attempted | GeoLookupState::NotAttempted) | None => { + "unknown".to_owned() } + }; + + let template_cache_state = response + .extensions() + .get::() + .map_or_else(|| "unknown".to_owned(), |state| state.as_str().to_owned()); + + let body_mode = if matches!(response.body(), EdgeBody::Stream(_)) { + "streamed" + } else { + "buffered" + }; + + AccessTelemetrySnapshot { + method: context.method.clone(), + status: response.status().as_u16(), + route_class, + route_template, + publisher_domain: context.publisher_domain.clone(), + env: resolve_env_dimension(), + service_id: env_var_or_unknown(ENV_FASTLY_SERVICE_ID), + pop: env_var_or_unknown(ENV_FASTLY_POP), + ts_version: env_var_or_unknown(ENV_FASTLY_SERVICE_VERSION), + country, + template_cache_state, + body_mode, + sample_rate: context.access_sample_rate, + } +} + +/// Derives the `env` access-telemetry dimension from the same +/// `FASTLY_IS_STAGING` input that drives the `x-ts-env` response header +/// (see [`apply_finalize_headers`]), never from [`Settings`] — `Settings` +/// has no environment field and does not gain one for this. +/// +/// `"unknown"` covers contexts where the variable is entirely absent (for +/// example native unit tests run outside Fastly Compute); on the Fastly +/// platform the variable is always present, as either `"1"` or not. +fn resolve_env_dimension() -> String { + match std::env::var(ENV_FASTLY_IS_STAGING) { + Ok(value) if value == "1" => "staging".to_owned(), + Ok(_) => "production".to_owned(), + Err(_) => "unknown".to_owned(), } } +/// Reads a Fastly-provided environment variable, defaulting to `"unknown"` +/// when unset. +fn env_var_or_unknown(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()) +} + /// Apply every late response mutation, then restore privacy invariants before headers commit. fn apply_terminal_response_effects( response: &mut HttpResponse, @@ -439,34 +954,31 @@ fn build_ja4_debug_response(req: &FastlyRequest) -> FastlyResponse { .with_body(body) } -pub(crate) fn maybe_identity_graph(settings: &Settings) -> Option { - settings - .ec - .ec_store - .as_ref() - .map(|store_name| KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) -} - -fn run_pull_sync_after_send( +/// Constructs a `KvIdentityGraph` wrapped in the [`Phase::EcKv`] timing +/// decorator, for request-path callers with a `RequestTimings` handle. +/// +/// Returns `None` when `ec.ec_store` is not configured, matching +/// [`require_identity_graph_with_timing`]'s contract on every other axis. +pub(crate) fn identity_graph_with_timing( settings: &Settings, - partner_registry: &PartnerRegistry, - context: &PullSyncContext, - services: &RuntimeServices, -) { - let kv = match require_identity_graph(settings) { - Ok(kv) => kv, - Err(err) => { - log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); - return; - } - }; - - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - dispatch_pull_sync(settings, &kv, partner_registry, &limiter, context, services); + timings: &RequestTimings, +) -> Option { + settings.ec.ec_store.as_ref().map(|store_name| { + KvIdentityGraph::new(TimedKvStore::new( + FastlyEcKvStore::new(store_name), + timings.clone(), + )) + }) } /// Constructs a `KvIdentityGraph` from settings, or returns an error if the /// `ec_store` config is not set. +/// +/// Deliberately untimed: pull-sync (this function's only caller) runs after +/// `send_edgezero_response`'s Server-Timing freeze point, so a decorated +/// store here would record into a handle nothing ever renders. +/// Request-path callers with a `RequestTimings` handle use +/// [`require_identity_graph_with_timing`] instead. pub(crate) fn require_identity_graph( settings: &Settings, ) -> Result> { @@ -479,6 +991,27 @@ pub(crate) fn require_identity_graph( Ok(KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) } +/// Constructs a `KvIdentityGraph` wrapped in the [`Phase::EcKv`] timing +/// decorator, or returns an error if the `ec_store` config is not set. +/// +/// Request-path sibling of [`require_identity_graph`], which pull-sync uses +/// unwrapped because pull-sync runs after the Server-Timing freeze point. +pub(crate) fn require_identity_graph_with_timing( + settings: &Settings, + timings: &RequestTimings, +) -> Result> { + let store_name = settings.ec.ec_store.as_deref().ok_or_else(|| { + Report::new(TrustedServerError::KvStore { + store_name: "ec.ec_store".to_owned(), + message: "ec.ec_store is not configured".to_owned(), + }) + })?; + Ok(KvIdentityGraph::new(TimedKvStore::new( + FastlyEcKvStore::new(store_name), + timings.clone(), + ))) +} + /// Extracts a named cookie value from the request's `Cookie` header. pub(crate) fn extract_cookie_value(req: &HttpRequest, name: &str) -> Option { let cookie_header = req.headers().get("cookie").and_then(|v| v.to_str().ok())?; @@ -507,12 +1040,18 @@ pub(crate) fn derive_device_signals(req: &FastlyRequest) -> DeviceSignals { #[cfg(test)] mod tests { + use std::sync::Mutex; + use super::*; + use base64::Engine as _; use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; use fastly::mime; + use std::time::Duration; use trusted_server_core::integrations::HeaderMutation; + use trusted_server_core::platform::RuntimeServices; + use trusted_server_core::request_timing::AuctionWaitPlacement; fn test_settings() -> Settings { Settings::from_toml( @@ -540,6 +1079,43 @@ mod tests { .expect("should parse test settings") } + /// A minimal [`AccessTelemetrySnapshot`] fixture for tests that only + /// need a `DeliveryOutcome` to exist, not its telemetry content. + fn sample_access_snapshot() -> AccessTelemetrySnapshot { + AccessTelemetrySnapshot { + method: "GET".to_owned(), + status: 200, + route_class: RouteClass::Other, + route_template: "/other/*".to_owned(), + publisher_domain: "unknown".to_owned(), + env: "unknown".to_owned(), + service_id: "unknown".to_owned(), + pop: "unknown".to_owned(), + ts_version: "unknown".to_owned(), + country: "unknown".to_owned(), + template_cache_state: "unknown".to_owned(), + body_mode: "buffered", + sample_rate: 0.0, + } + } + + #[test] + fn pull_sync_noop_states_skip_post_send_graph_factory() { + let calls = std::cell::Cell::new(0); + let result = prepare_pull_sync_after_send(None, || { + calls.set(calls.get() + 1); + Err(Report::new(TrustedServerError::KvStore { + store_name: "unexpected".to_owned(), + message: "graph factory should not run".to_owned(), + })) + }); + assert!( + result.is_none(), + "a skipped pull-sync plan should return none" + ); + assert_eq!(calls.get(), 0, "should not invoke the graph factory"); + } + #[test] fn health_response_short_circuits_get_health() { let req = FastlyRequest::get("https://example.com/health"); @@ -775,9 +1351,10 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); - let geo_info = resolve_geo_for_response(&response, None, |_| { - panic!("should skip entry-point geo lookup for 401 responses"); - }); + let geo_info = + resolve_geo_for_response(&response, &GeoLookupState::NotAttempted, None, |_| { + panic!("should skip entry-point geo lookup for 401 responses"); + }); apply_finalize_headers(&settings, geo_info.as_ref(), &mut response); assert_eq!( @@ -843,4 +1420,431 @@ mod tests { "should include sec-ch-ua-platform fallback" ); } + + fn ec_finalize_settings() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + ec_store = "ec_identity_store" + + [[ec.partners]] + name = "Example Partner" + source_domain = "example.com" + api_token = "test-vendor-token-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + "#, + ) + .expect("should parse EC finalize test settings") + } + + /// Minimal `RuntimeServices` for `EcFinalizeState.services`. Real + /// `FastlyPlatform*` handles are used as inert placeholders: EC + /// finalization never calls through them, it only satisfies the field. + fn inert_runtime_services() -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore) + as Arc) + .backend(Arc::new(crate::platform::FastlyPlatformBackend)) + .http_client(Arc::new(crate::platform::FastlyPlatformHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(trusted_server_core::platform::ClientInfo::default()) + .build() + } + + #[test] + fn ec_finalize_kv_lands_before_freeze() { + // A pre-seeded EC entry (see fastly.toml's ec_identity_store fixture) + // for a returning user carrying an eids cookie that matches the + // configured partner. This drives ec_finalize_response into + // ingest_eid_cookies, which reads and writes the KV identity graph. + let settings = ec_finalize_settings(); + let ec_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"; + let eids = serde_json::json!([{ + "source": "example.com", + "uids": [{ "id": "example-uid", "atype": 1 }] + }]); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode(eids.to_string()); + let request = edgezero_core::http::request_builder() + .method(fastly::http::Method::GET) + .uri("https://test-publisher.com/article") + .header("cookie", format!("ts-ec={ec_id}; ts-eids={eids_cookie}")) + .body(EdgeBody::empty()) + .expect("should build EC finalize test request"); + + let services = inert_runtime_services(); + let geo_info = trusted_server_core::platform::GeoInfo { + city: String::new(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + }; + let mut ec_context = trusted_server_core::ec::EcContext::read_from_request_with_geo( + &settings, + &request, + &services, + Some(&geo_info), + ) + .expect("should read EC context from a non-regulated request"); + ec_context.set_eid_sync_source(trusted_server_core::ec::EidSyncSource::Navigation); + assert!( + ec_context.ec_was_present(), + "the pre-seeded ts-ec cookie should be recognized" + ); + + let mut ec_state = EcFinalizeState { + ec_context, + use_finalize_kv: true, + eids_cookie: Some(eids_cookie), + sharedid_cookie: None, + is_real_browser: true, + services, + }; + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(EdgeBody::empty()) + .expect("should build EC finalize response fixture"); + let timings = RequestTimings::new(); + + // Mirrors edgezero_main's ordering: EC finalize runs, then the freeze + // point (apply_server_timing_header, called just before + // response.into_parts() inside send_edgezero_response) renders the + // header. Calling both directly exercises exactly this order without + // requiring a live Fastly client connection. + apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response, &timings) + .expect("should finalize EC response"); + apply_server_timing_header(&mut response, &timings, true); + + let header = response + .headers() + .get("server-timing") + .and_then(|v| v.to_str().ok()) + .expect("should emit a Server-Timing header"); + assert!( + header.contains("ts-kv"), + "the freeze point must run after EC finalization recorded KV time: {header}" + ); + } + + #[test] + fn delivery_outcome_reports_bytes_and_request_elapsed_set() { + let timings = RequestTimings::new(); + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"hello "), + bytes::Bytes::from_static(b"world"), + ])); + + let (counting, drive_result) = drive_streaming_body(body, Vec::new(), &timings); + drive_result.expect("streaming a well-formed body should not fail"); + let bytes = counting.bytes(); + let outcome = DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot: Some(sample_access_snapshot()), + }; + + assert_eq!( + counting.into_inner(), + b"hello world", + "should write every byte to the underlying transport" + ); + assert_eq!( + outcome.bytes, + "hello world".len() as u64, + "DeliveryOutcome.bytes should equal the streamed body length" + ); + + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.resp_bytes, + Some("hello world".len() as u64), + "should stamp resp_bytes to the tallied byte count" + ); + assert!( + snapshot.request_elapsed_ms.is_some(), + "should stamp request_elapsed once the drive returns" + ); + } + + #[test] + fn buffered_delivery_stamps_bytes_and_request_elapsed() { + let timings = RequestTimings::new(); + let body = EdgeBody::from(b"a buffered body".to_vec()); + + let bytes = record_buffered_delivery(&body, &timings); + + assert_eq!( + bytes, + "a buffered body".len() as u64, + "should report the buffered body length" + ); + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.resp_bytes, + Some("a buffered body".len() as u64), + "should stamp resp_bytes for the buffered path too" + ); + assert!( + snapshot.request_elapsed_ms.is_some(), + "should stamp request_elapsed for the buffered path too" + ); + } + + fn send_context_fixture() -> SendContext { + SendContext { + timings: RequestTimings::new(), + server_timing_enabled: false, + method: "GET".to_owned(), + publisher_domain: "test-publisher.com".to_owned(), + access_sample_rate: 0.25, + access_telemetry_enabled: true, + } + } + + #[test] + fn access_snapshot_defaults_when_no_extensions_are_attached() { + // Router-generated 404/405 responses and other paths that never pass + // through a RouteMetadata-attaching wrapper must still produce a + // usable snapshot: RouteClass::Other and "unknown" sentinels, never + // a missing/panicking build. + let response = response_builder() + .status(404) + .body(EdgeBody::empty()) + .expect("should build response"); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert_eq!(snapshot.status, 404); + assert_eq!(snapshot.method, "GET"); + assert!(matches!(snapshot.route_class, RouteClass::Other)); + assert_eq!(snapshot.route_template, "unknown"); + assert_eq!(snapshot.country, "unknown"); + assert_eq!(snapshot.template_cache_state, "unknown"); + assert_eq!(snapshot.body_mode, "buffered"); + assert_eq!(snapshot.publisher_domain, "test-publisher.com"); + assert_eq!(snapshot.sample_rate, 0.25); + } + + #[test] + fn access_snapshot_reads_route_geo_and_template_cache_extensions() { + let mut response = response_builder() + .status(200) + .body(EdgeBody::empty()) + .expect("should build response"); + response.extensions_mut().insert(RouteMetadata { + route_class: RouteClass::AuctionApi, + route_template: "/auction".to_owned(), + }); + response.extensions_mut().insert(GeoLookupState::Resolved( + trusted_server_core::platform::GeoInfo { + city: String::new(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + }, + )); + response + .extensions_mut() + .insert(TemplateCacheResponseState::Hit); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert!(matches!(snapshot.route_class, RouteClass::AuctionApi)); + assert_eq!(snapshot.route_template, "/auction"); + assert_eq!(snapshot.country, "US"); + assert_eq!(snapshot.template_cache_state, "hit"); + } + + #[test] + fn access_snapshot_treats_attempted_geo_lookup_as_unknown_country() { + let mut response = response_builder() + .status(200) + .body(EdgeBody::empty()) + .expect("should build response"); + response.extensions_mut().insert(GeoLookupState::Attempted); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert_eq!( + snapshot.country, "unknown", + "an attempted-but-unresolved lookup must not surface a stale country" + ); + } + + #[test] + fn access_snapshot_body_mode_reflects_the_response_body_variant() { + let streamed = response_builder() + .status(200) + .body(EdgeBody::stream(futures::stream::empty())) + .expect("should build streaming response"); + let buffered = response_builder() + .status(200) + .body(EdgeBody::from(b"hi".to_vec())) + .expect("should build buffered response"); + let context = send_context_fixture(); + + assert_eq!( + build_access_telemetry_snapshot(&streamed, &context).body_mode, + "streamed" + ); + assert_eq!( + build_access_telemetry_snapshot(&buffered, &context).body_mode, + "buffered" + ); + } + + #[test] + fn stream_drive_records_stream_ms_covering_the_in_stream_auction_wait() { + // A streaming seam wait (Task 6, publisher.rs) records into the same + // `RequestTimings` handle the adapter drives with. `Phase::Stream` + // wraps the entire drive, so it must cover — and therefore be at + // least as large as — any `AuctionWait` recorded while the body was + // being polled. + let timings = RequestTimings::new(); + let wait_timings = timings.clone(); + let stream = futures::stream::once(async move { + let waited = Duration::from_millis(5); + std::thread::sleep(waited); + wait_timings.record_auction_wait(AuctionWaitPlacement::InStream, waited); + bytes::Bytes::from_static(b"") + }); + let body = EdgeBody::stream(stream); + + let (_counting, drive_result) = drive_streaming_body(body, Vec::new(), &timings); + drive_result.expect("streaming a well-formed body should not fail"); + + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.auction_wait_placement, + Some(AuctionWaitPlacement::InStream), + "should preserve the placement recorded from inside the polled body" + ); + let auction_wait_ms = snapshot + .auction_wait_ms + .expect("should record the auction wait"); + let stream_ms = snapshot.stream_ms.expect("should record the stream drive"); + assert!( + stream_ms >= auction_wait_ms, + "the drive's Phase::Stream span must cover the in-stream auction wait: \ + stream_ms={stream_ms} auction_wait_ms={auction_wait_ms}" + ); + } + + #[test] + fn classify_stream_delivery_treats_bytes_written_before_an_error_as_partial() { + let err = Report::new(TrustedServerError::Proxy { + message: "boom".to_string(), + }); + assert_eq!( + classify_stream_delivery(&Err(err), 42), + DeliveryResult::Partial, + "bytes already on the wire before a stream error is a truncated delivery" + ); + } + + #[test] + fn classify_stream_delivery_treats_an_error_with_no_bytes_as_error() { + let err = Report::new(TrustedServerError::Proxy { + message: "boom".to_string(), + }); + assert_eq!( + classify_stream_delivery(&Err(err), 0), + DeliveryResult::Error, + "a failure before any byte reached the client is a clean failure, not a truncation" + ); + } + + #[test] + fn classify_stream_delivery_treats_ok_as_complete() { + assert_eq!( + classify_stream_delivery(&Ok(()), 123), + DeliveryResult::Complete + ); + } + + #[test] + fn post_send_order_is_elapsed_then_pull_sync_then_telemetry() { + // The full contract sequence, instrumented through the real seams: + // `send_edgezero_response` stamps `request_elapsed` before + // returning, and `run_post_send_steps` (which every production + // site with both steps routes through) owns pull-sync-then- + // telemetry ordering. + let log: Arc>> = Arc::new(Mutex::new(Vec::new())); + let timings = RequestTimings::new(); + let response = response_builder() + .body(EdgeBody::from("ok")) + .expect("should build response"); + + let outcome = send_edgezero_response( + response, + None, + &SendContext { + timings: timings.clone(), + server_timing_enabled: false, + method: "GET".to_owned(), + publisher_domain: "test-publisher.com".to_owned(), + access_sample_rate: 1.0, + access_telemetry_enabled: true, + }, + ); + assert!( + timings.snapshot().request_elapsed_ms.is_some(), + "request_elapsed should be stamped before any post-send step runs" + ); + assert!( + outcome.snapshot.is_some(), + "the access snapshot should exist for the enabled context" + ); + + let pull_log = Arc::clone(&log); + let emit_log = Arc::clone(&log); + run_post_send_steps( + move || { + pull_log + .lock() + .expect("should lock order log") + .push("pull_sync") + }, + move || { + emit_log + .lock() + .expect("should lock order log") + .push("telemetry") + }, + ); + + assert_eq!( + *log.lock().expect("should lock order log"), + vec!["pull_sync", "telemetry"], + "pull-sync must dispatch before telemetry emits" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 283f16255..7bd577363 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -24,8 +24,9 @@ use trusted_server_core::constants::{ ENV_FASTLY_IS_STAGING, ENV_FASTLY_SERVICE_VERSION, HEADER_X_GEO_INFO_AVAILABLE, HEADER_X_TS_ENV, HEADER_X_TS_VERSION, }; -use trusted_server_core::geo::GeoInfo; +use trusted_server_core::geo::{GeoInfo, GeoLookupState}; use trusted_server_core::platform::{ClientInfo, PlatformGeo}; +use trusted_server_core::request_timing::{Phase, RequestTimings}; use trusted_server_core::settings::Settings; pub(crate) const HEADER_X_TS_FINALIZED: &str = "x-ts-finalized"; @@ -71,6 +72,12 @@ impl Middleware for FinalizeResponseMiddleware { || FastlyRequestContext::get(ctx.request()).and_then(|c| c.client_ip), |info| info.client_ip, ); + let timings = ctx + .request() + .extensions() + .get::() + .cloned() + .unwrap_or_default(); let mut response = match next.run(ctx).await { Ok(r) => r, @@ -80,13 +87,24 @@ impl Middleware for FinalizeResponseMiddleware { } }; - let geo_info = resolve_geo_for_response(&response, client_ip, |ip| { + let carried = response + .extensions() + .get::() + .cloned() + .unwrap_or(GeoLookupState::NotAttempted); + let geo_info = resolve_geo_for_response(&response, &carried, client_ip, |ip| { + let _span = timings.span(Phase::Geo); self.geo.lookup(ip).unwrap_or_else(|e| { log::warn!("geo lookup failed: {e}"); None }) }); + // Mirrors the entry-point finalize site in `main.rs` + // (`apply_entry_point_finalize_headers`); 401 handling lives in the + // shared helper. + write_back_geo_lookup_state(&mut response, geo_info.as_ref()); + apply_finalize_headers(&self.settings, geo_info.as_ref(), &mut response); response .headers_mut() @@ -145,14 +163,20 @@ impl Middleware for AuthMiddleware { // Shared geo resolution helper // --------------------------------------------------------------------------- -/// Resolves geo for a response, skipping the lookup for 401 responses. +/// Resolves geo for a response, skipping the lookup for 401 responses and +/// reusing a request-phase lookup when one was already carried. /// -/// Returns `None` for authentication rejections (401) without calling `lookup_geo` -/// to avoid unnecessary work and exposing geo data to unauthenticated callers. -/// All other responses call `lookup_geo` and return its result. +/// Returns `None` for authentication rejections (401) without consulting +/// `carried` or calling `lookup_geo`, to avoid unnecessary work and exposing +/// geo data to unauthenticated callers. Otherwise dispatches on `carried`: +/// a [`GeoLookupState::Resolved`] value is reused as-is, a +/// [`GeoLookupState::Attempted`] value is treated as no geo info without +/// retrying the lookup, and [`GeoLookupState::NotAttempted`] falls back to +/// calling `lookup_geo`. /// /// Used by both [`FinalizeResponseMiddleware`] and the entry-point finalization -/// in `main.rs` so the 401-skip rule is defined in one place. +/// in `main.rs` so the 401-skip rule and the dedupe rule are each defined in +/// one place. /// /// # Parity note /// @@ -162,8 +186,29 @@ impl Middleware for AuthMiddleware { /// is intentionally more conservative: geo data is not sent to any /// unauthenticated caller regardless of whether the 401 originated from this /// server or the upstream origin. +/// Writes the resolved geo outcome back onto the response as a +/// [`GeoLookupState`] extension, so a downstream access-telemetry snapshot +/// sees what was actually looked up rather than the stale carried-in state. +/// +/// Skips the write on a 401: [`resolve_geo_for_response`] returns `None` +/// for unauthorized responses before consulting the carried state, so +/// writing `Attempted` there would overwrite a carried `Resolved` with a +/// value that was never looked up, and the row would lose a country it +/// legitimately had. +pub(crate) fn write_back_geo_lookup_state(response: &mut Response, geo_info: Option<&GeoInfo>) { + if response.status() == StatusCode::UNAUTHORIZED { + return; + } + let resolved_state = match geo_info { + Some(geo) => GeoLookupState::Resolved(geo.clone()), + None => GeoLookupState::Attempted, + }; + response.extensions_mut().insert(resolved_state); +} + pub(crate) fn resolve_geo_for_response( response: &Response, + carried: &GeoLookupState, client_ip: Option, lookup_geo: F, ) -> Option @@ -171,9 +216,12 @@ where F: FnOnce(Option) -> Option, { if response.status() == StatusCode::UNAUTHORIZED { - None - } else { - lookup_geo(client_ip) + return None; + } + match carried { + GeoLookupState::Resolved(geo) => Some(geo.clone()), + GeoLookupState::Attempted => None, + GeoLookupState::NotAttempted => lookup_geo(client_ip), } } @@ -250,6 +298,51 @@ pub(crate) use trusted_server_core::response_privacy::{ mod tests { use super::*; + #[test] + fn geo_write_back_preserves_resolved_state_on_401() { + // A 401 short-circuits geo resolution before the carried state is + // consulted, so the write-back must not downgrade a carried + // Resolved to Attempted (which would cost the row its country). + let mut response = response_builder() + .status(StatusCode::UNAUTHORIZED) + .body(Body::empty()) + .expect("should build a 401 response"); + response + .extensions_mut() + .insert(GeoLookupState::Resolved(sample_geo_info())); + + write_back_geo_lookup_state(&mut response, None); + + match response.extensions().get::() { + Some(GeoLookupState::Resolved(geo)) => { + assert_eq!( + geo.country, + sample_geo_info().country, + "should keep the carried country" + ); + } + other => panic!("should keep the Resolved state on a 401, got {other:?}"), + } + } + + #[test] + fn geo_write_back_records_attempted_on_non_401_miss() { + let mut response = response_builder() + .status(StatusCode::OK) + .body(Body::empty()) + .expect("should build a 200 response"); + + write_back_geo_lookup_state(&mut response, None); + + assert!( + matches!( + response.extensions().get::(), + Some(GeoLookupState::Attempted) + ), + "should record an attempted-but-missed lookup on ordinary responses" + ); + } + use std::collections::HashMap; use std::net::IpAddr; use std::sync::{Arc, Mutex}; @@ -280,6 +373,19 @@ mod tests { RequestContext::new(req, PathParams::new(HashMap::new())) } + fn sample_geo_info() -> GeoInfo { + GeoInfo { + city: "Testville".to_string(), + country: "US".to_string(), + continent: "NorthAmerica".to_string(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + } + } + struct FixedGeo(Option); impl PlatformGeo for FixedGeo { @@ -682,6 +788,67 @@ mod tests { ); } + #[test] + fn finalize_handle_writes_back_resolved_geo_state_after_fallback_lookup() { + // The request phase never attempted a geo lookup (no GeoLookupState + // extension on the handler's response), so the middleware resolves + // one via the fallback closure. That resolved outcome must be + // written back into response extensions -- mirroring + // apply_entry_point_finalize_headers in main.rs -- so a downstream + // access-telemetry snapshot sees the freshly resolved country + // instead of a stale/missing GeoLookupState. + let settings = settings_with_response_headers(vec![]); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(FixedGeo(Some(sample_geo_info()))), + ); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + let response = block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should succeed"); + + match response.extensions().get::() { + Some(GeoLookupState::Resolved(info)) => { + assert_eq!( + info.country, "US", + "should carry the fallback-resolved geo info" + ); + } + other => { + panic!("expected GeoLookupState::Resolved after a fallback lookup, got {other:?}") + } + } + } + + #[test] + fn finalize_handle_writes_back_attempted_geo_state_when_fallback_finds_nothing() { + // The fallback lookup ran but resolved no geo info. The middleware + // must still record that the lookup was attempted, so a later + // consumer of the extension does not mistake this for + // GeoLookupState::NotAttempted and retry the lookup. + let settings = settings_with_response_headers(vec![]); + let middleware = + FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + let response = block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should succeed"); + + assert!( + matches!( + response.extensions().get::(), + Some(GeoLookupState::Attempted) + ), + "should write back Attempted when the fallback lookup finds no geo info" + ); + } + #[test] fn finalize_handle_marks_response_as_finalized() { let settings = settings_with_response_headers(vec![]); @@ -763,6 +930,30 @@ mod tests { ); } + #[test] + #[allow(clippy::panic)] + fn geo_lookup_skipped_for_unauthorized_responses() { + // The 401 short-circuit in resolve_geo_for_response must win + // regardless of what state the request phase carried in, and must + // never invoke the fallback lookup closure. + let mut response = empty_response(); + *response.status_mut() = StatusCode::UNAUTHORIZED; + + for carried in [ + GeoLookupState::NotAttempted, + GeoLookupState::Attempted, + GeoLookupState::Resolved(sample_geo_info()), + ] { + let geo_info = resolve_geo_for_response(&response, &carried, None, |_| { + panic!("401 responses must never trigger a geo lookup"); + }); + assert!( + geo_info.is_none(), + "401 responses should never resolve geo info, regardless of carried state" + ); + } + } + // --------------------------------------------------------------------------- // AuthMiddleware::handle tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index a29b48fda..c241dd33c 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -1,16 +1,12 @@ //! Fastly-backed implementations of the platform traits defined in //! `trusted-server-core::platform`. -use std::io::Read as _; -use std::net::IpAddr; -use std::sync::Arc; - use bytes::Bytes; -use edgezero_adapter_fastly::key_value_store::FastlyKvStore; -use edgezero_core::key_value_store::KvError; use error_stack::{Report, ResultExt}; use fastly::geo::{Geo, geo_lookup}; use fastly::{ConfigStore, Request, SecretStore}; +use std::io::Read as _; +use std::net::IpAddr; use crate::backend::BackendConfig; pub(crate) use trusted_server_core::platform::UnavailableKvStore; @@ -18,9 +14,8 @@ use trusted_server_core::platform::{ BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, - PlatformImageOptimizerParams, PlatformImageOptimizerRegion, PlatformKvStore, - PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, - StoreName, + PlatformImageOptimizerParams, PlatformImageOptimizerRegion, PlatformPendingRequest, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, }; use trusted_server_core::settings::TrustedClientIpConfig; @@ -729,16 +724,6 @@ pub fn client_info_from_request(req: &Request, client_ip: Option) -> Cli } } -/// Open a named KV store as a [`PlatformKvStore`] implementation. -/// -/// # Errors -/// -/// Returns [`KvError::Unavailable`] when the store does not exist, or -/// [`KvError::Internal`] when the Fastly SDK fails to open it. -pub fn open_kv_store(store_name: &str) -> Result, KvError> { - FastlyKvStore::open(store_name).map(|store| Arc::new(store) as Arc) -} - // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index f315a7b56..5c0348fac 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -10,10 +10,15 @@ use trusted_server_core::auction::telemetry::{ AuctionEventBatch, AuctionTelemetrySink, NoopAuctionTelemetrySink, }; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use trusted_server_core::platform::{ + PlatformBackend as _, PlatformBackendSpec, PlatformHttpClient, PlatformHttpRequest, + RuntimeServices, +}; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::{Settings, TinybirdSettings}; +use crate::platform::FastlyPlatformBackend; + const TINYBIRD_EVENTS_PATH: &str = "/v0/events"; const TINYBIRD_NDJSON_CONTENT_TYPE: &str = "application/x-ndjson"; const TINYBIRD_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(2); @@ -21,9 +26,14 @@ const TINYBIRD_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(2); const TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH: usize = 512; /// Build the configured auction telemetry sink. +/// +/// Auction emission requires both the Tinybird master toggle +/// (`tinybird.enabled`) and the auction-specific toggle +/// (`tinybird.auction_enabled`), so access-log telemetry can be enabled +/// independently without also emitting auction events. #[must_use] pub(crate) fn auction_sink_from_settings(settings: &Settings) -> Arc { - if settings.tinybird.enabled { + if settings.tinybird.enabled && settings.tinybird.auction_enabled { Arc::new(FastlyTinybirdAuctionTelemetrySink::new( settings.tinybird.clone(), )) @@ -39,7 +49,7 @@ struct FastlyTinybirdAuctionTelemetrySink { } #[derive(Debug, Clone)] -struct TinybirdEventsTarget { +pub(crate) struct TinybirdEventsTarget { api_host: String, dataset: String, append_token: Redacted, @@ -63,6 +73,28 @@ impl TinybirdEventsTarget { max_body_bytes: config.max_body_bytes, } } + + /// Builds the Events API target for the access-log datasource. + /// + /// Shares [`from_config`](Self::from_config)'s host, resolved-token, and + /// body-size-limit derivation, but points at `access_dataset` and + /// `access_token_secret` instead of the auction pair, so access-log + /// emission never shares a datasource or token with auction telemetry + /// even though both configs come from the same [`TinybirdSettings`]. + pub(crate) fn from_access_config(config: TinybirdSettings) -> Self { + let uri = tinybird_events_uri(&config.api_host, &config.access_dataset); + let backend_spec = tinybird_backend_spec(&config.api_host); + Self { + api_host: config.api_host, + dataset: config.access_dataset, + append_token: config + .access_token_secret + .expect("should contain a resolved Tinybird access token when enabled"), + uri, + backend_spec, + max_body_bytes: config.max_body_bytes, + } + } } impl FastlyTinybirdAuctionTelemetrySink { @@ -182,6 +214,132 @@ impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { } } +// --------------------------------------------------------------------------- +// Access telemetry: confirmed-delivery emitter +// --------------------------------------------------------------------------- + +/// Decides whether one request's access-telemetry row should be emitted. +/// +/// `roll` is a uniform draw from `[0, 1)`; callers pass +/// `rand::thread_rng().r#gen::()`, which the wasm32-wasip1 guest backs with +/// real WASI randomness (the EC generation path already relies on this and +/// the CI wasm release build verifies it). Comparing the draw directly +/// against `rate` keeps the sampling probability exactly `rate` for every +/// positive value: there is no bucket quantization, so rates below one in a +/// million sample proportionally instead of never, and emitted rows' +/// `sample_rate` matches the probability they were sampled at, which the +/// `sum(1.0 / sample_rate)` volume estimator depends on. +/// +/// `rate <= 0.0` never samples and `rate >= 1.0` always samples, for any +/// `roll` in `[0, 1)`. `0.0` cannot actually occur while `access_enabled` +/// is `true` (`Settings` validation requires `access_sample_rate > 0.0` in +/// that case), but this function stays total rather than leaning on that +/// invariant. +#[must_use] +pub(crate) fn sampled_in(rate: f64, roll: f64) -> bool { + roll < rate +} + +/// Builds the Events API POST request for one access-log row. +fn build_access_events_request( + target: &TinybirdEventsTarget, + body: String, + auth_header: HeaderValue, +) -> Result> { + request_builder() + .method(Method::POST) + .uri(target.uri.as_str()) + .header(header::AUTHORIZATION, auth_header) + .header(header::CONTENT_TYPE, TINYBIRD_NDJSON_CONTENT_TYPE) + .body(Body::from(body)) + .change_context(TrustedServerError::Proxy { + message: "failed to build Tinybird Events API request".to_owned(), + }) +} + +/// Sends one confirmed access-log row to the Tinybird Events API and waits +/// for the response. +/// +/// Unlike [`FastlyTinybirdAuctionTelemetrySink::emit_auction_events`] (fire- +/// and-forget, dispatched mid-request so it never adds latency to the +/// response), this runs post-delivery: the response has already reached the +/// client, so there is no latency budget left to protect, and the send can +/// afford to wait for — and validate — the reply. `client` is the adapter's +/// stateless platform HTTP client in production +/// ([`crate::platform::FastlyPlatformHttpClient`]); accepting it as `&dyn +/// PlatformHttpClient` here (rather than that concrete type) is what lets +/// tests substitute a recording double instead of performing a real network +/// send, matching how [`RuntimeServices::http_client`] is consumed +/// elsewhere. `target` is derived from settings once at the post-send call +/// site rather than threaded through any per-route state. +/// +/// A non-2xx status is reported as `Err` naming the status; there is no +/// retry — the caller logs exactly one warning and moves on. +/// +/// # Errors +/// +/// Returns `Err` when the row exceeds the configured request-body limit, the +/// resolved access-log APPEND token is invalid, the backend cannot be registered, +/// the request cannot be built or sent, or the Tinybird Events API responds +/// with a non-2xx status. +pub(crate) async fn emit_access_event( + client: &dyn PlatformHttpClient, + target: &TinybirdEventsTarget, + mut row: String, +) -> Result<(), Report> { + // Match the auction sink's NDJSON framing: every row is + // newline-terminated, and the terminator counts toward the body limit. + if !row.ends_with('\n') { + row.push('\n'); + } + let body_len = row.len(); + if body_len > target.max_body_bytes { + return Err(Report::new(TrustedServerError::Proxy { + message: format!( + "Tinybird access telemetry request body has {body_len} bytes, exceeding {} byte limit", + target.max_body_bytes + ), + })); + } + + let auth_header = + FastlyTinybirdAuctionTelemetrySink::authorization_header(target.append_token.expose())?; + let backend_name = FastlyPlatformBackend + .ensure(&target.backend_spec) + .change_context(TrustedServerError::Proxy { + message: "Tinybird backend registration failed".to_owned(), + })?; + let request = build_access_events_request(target, row, auth_header)?; + + log::info!( + "sending access telemetry to Tinybird dataset={} host={} backend={}", + target.dataset, + target.api_host, + backend_name + ); + + // The response body is never consumed, so stream it: buffered + // conversion on Fastly materializes the body before the size limit is + // enforced, which a chunked response could abuse. + let response = client + .send(PlatformHttpRequest::new(request, backend_name).with_stream_response()) + .await + .change_context(TrustedServerError::Proxy { + message: "failed to send Tinybird access telemetry request".to_owned(), + })?; + + if response.response.status().is_success() { + Ok(()) + } else { + Err(Report::new(TrustedServerError::Proxy { + message: format!( + "Tinybird access telemetry request failed with status {}", + response.response.status() + ), + })) + } +} + fn tinybird_backend_spec(api_host: &str) -> PlatformBackendSpec { PlatformBackendSpec { scheme: "https".to_owned(), @@ -306,28 +464,33 @@ mod tests { uri: String, headers: Vec<(String, String)>, body: Vec, + stream_response: bool, } + /// Records outbound requests and, for [`PlatformHttpClient::send`] (the + /// blocking variant `emit_access_event` uses), returns a synthetic + /// response carrying `respond_status` instead of performing a real + /// network send. #[derive(Default)] struct RecordingHttpClient { requests: Mutex>, select_calls: Mutex, + respond_status: Mutex, } - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RecordingHttpClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) + impl RecordingHttpClient { + /// Status [`PlatformHttpClient::send`] should reply with. Irrelevant + /// to auction-sink tests, which only exercise `send_async`. + fn respond_with(status: u16) -> Self { + Self { + respond_status: Mutex::new(status), + ..Self::default() + } } - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { + fn record(&self, request: PlatformHttpRequest) { let backend_name = request.backend_name; + let stream_response = request.stream_response; let (parts, body) = request.request.into_parts(); let headers = parts .headers @@ -345,11 +508,41 @@ mod tests { uri: parts.uri.to_string(), headers, body: body.into_bytes().unwrap_or_default().to_vec(), + stream_response, }; self.requests .lock() .expect("should lock recorded requests") .push(recorded); + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RecordingHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.record(request); + let status = *self + .respond_status + .lock() + .expect("should lock configured response status"); + let response = edgezero_core::http::response_builder() + .status( + edgezero_core::http::StatusCode::from_u16(status) + .expect("should build a valid test status code"), + ) + .body(edgezero_core::body::Body::empty()) + .expect("should build test response"); + Ok(PlatformResponse::new(response)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.record(request); Ok(PlatformPendingRequest::new(()).with_backend_name("tinybird-backend")) } @@ -430,18 +623,52 @@ mod tests { fn enabled_config() -> TinybirdSettings { TinybirdSettings { enabled: true, + auction_enabled: true, api_host: "api.us-east.aws.tinybird.co".to_owned(), secret_store: None, auction_dataset: "auction_events_raw".to_owned(), auction_token_secret: Some(Redacted::new("append-token".to_owned())), access_enabled: false, access_dataset: "access_logs_raw".to_owned(), - access_token_secret: None, + access_token_secret: Some(Redacted::new("access-append-token".to_owned())), access_sample_rate: 0.0, max_body_bytes: 1024 * 1024, } } + #[test] + fn sink_from_settings_disables_when_auction_enabled_is_false() { + let settings = Settings { + tinybird: TinybirdSettings { + auction_enabled: false, + ..enabled_config() + }, + ..Settings::default() + }; + + let sink = auction_sink_from_settings(&settings); + + assert!( + !sink.is_enabled(), + "auction telemetry should stay off when auction_enabled is false, even if tinybird.enabled is true" + ); + } + + #[test] + fn sink_from_settings_enables_when_both_toggles_are_true() { + let settings = Settings { + tinybird: enabled_config(), + ..Settings::default() + }; + + let sink = auction_sink_from_settings(&settings); + + assert!( + sink.is_enabled(), + "auction telemetry should be on when both tinybird.enabled and tinybird.auction_enabled are true" + ); + } + #[test] fn events_uri_targets_dataset_on_region_host() { assert_eq!( @@ -618,6 +845,139 @@ mod tests { ); } + #[test] + fn access_emitter_rejects_oversized_row_before_sending() { + let mut config = enabled_config(); + config.max_body_bytes = 1024; + let target = TinybirdEventsTarget::from_access_config(config); + let http_client = RecordingHttpClient::respond_with(202); + let row = "x".repeat(1025); + + let result = futures::executor::block_on(emit_access_event(&http_client, &target, row)); + + let error = result.expect_err("should reject a row above the configured body limit"); + assert!( + error.to_string().contains("1024"), + "error should name the configured body limit: {error}" + ); + assert!( + http_client + .requests + .lock() + .expect("should lock recorded requests") + .is_empty(), + "should not send an oversized access row" + ); + } + + #[test] + fn access_emitter_posts_ndjson_and_validates_2xx() { + // Runtime settings carry the access APPEND token after startup secret + // resolution, so post-delivery emission does not reopen a secret store. + let target = TinybirdEventsTarget::from_access_config(enabled_config()); + let http_client = RecordingHttpClient::respond_with(202); + let row = r#"{"status":200}"#.to_owned(); + + futures::executor::block_on(emit_access_event(&http_client, &target, row.clone())) + .expect("should accept a 202 response"); + + let requests = http_client + .requests + .lock() + .expect("should lock recorded requests"); + assert_eq!(requests.len(), 1, "should send exactly one request"); + assert_eq!( + requests[0].uri, + "https://api.us-east.aws.tinybird.co/v0/events?name=access_logs_raw" + ); + assert_eq!(requests[0].method, Method::POST.to_string()); + assert_eq!( + header_value(&requests[0].headers, header::AUTHORIZATION.as_str()), + Some("Bearer access-append-token") + ); + let body = std::str::from_utf8(&requests[0].body).expect("should record utf8 body"); + assert_eq!( + body, + format!("{row}\n"), + "should send newline-delimited JSON, matching the auction sink's framing" + ); + assert!( + requests[0].stream_response, + "should stream the Tinybird response: the body is never consumed" + ); + } + + #[test] + fn access_emitter_warns_and_drops_on_non_2xx() { + let target = TinybirdEventsTarget::from_access_config(enabled_config()); + let http_client = RecordingHttpClient::respond_with(422); + + let result = futures::executor::block_on(emit_access_event( + &http_client, + &target, + r#"{"status":422}"#.to_owned(), + )); + + let error = result.expect_err("a 422 response should be reported as an error"); + assert!( + error.to_string().contains("422"), + "error should name the failing status: {error}" + ); + assert_eq!( + http_client + .requests + .lock() + .expect("should lock recorded requests") + .len(), + 1, + "should not retry after a non-2xx response" + ); + } + + #[test] + fn sampled_in_boundary_rates_are_unconditional() { + assert!( + sampled_in(1.0, 0.0), + "a 1.0 sample rate should always sample in" + ); + assert!( + sampled_in(1.0, 0.999_999), + "a 1.0 sample rate should sample in for the largest roll" + ); + assert!( + !sampled_in(0.0, 0.0), + "a 0.0 sample rate should never sample in, even on a zero roll" + ); + assert!( + !sampled_in(-1.0, 0.0), + "a negative rate should never sample in" + ); + } + + #[test] + fn sampled_in_keeps_exact_probability_for_tiny_rates() { + // The previous bucket-quantized sampler truncated rates below one + // in a million to a zero threshold, silently emitting nothing. + // Direct comparison keeps every positive rate proportional. + let rate = 0.000_000_1; + assert!( + sampled_in(rate, rate / 2.0), + "a roll below a tiny positive rate should sample in" + ); + assert!( + !sampled_in(rate, rate * 2.0), + "a roll above a tiny positive rate should sample out" + ); + assert!( + !sampled_in(0.000_001_9, 0.000_001_95), + "no downward quantization: the boundary sits exactly at the rate" + ); + assert!( + sampled_in(0.000_001_9, 0.000_001_85), + "rolls just under the rate should sample in" + ); + } + fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { headers .iter() diff --git a/crates/trusted-server-adapter-spin/spin.toml b/crates/trusted-server-adapter-spin/spin.toml index 7055f9563..76737cea8 100644 --- a/crates/trusted-server-adapter-spin/spin.toml +++ b/crates/trusted-server-adapter-spin/spin.toml @@ -18,6 +18,8 @@ version = "0.1.0" # Operators enabling request_signing must declare one encoded secret variable for # each private signing key. Public signing metadata remains in the KV store. [variables] +v_current_x2dkid = { default = "" } +v_active_x2dkids = { default = "" } # These declared variables match the example config's secret key names. Regenerate # or extend them for deployment-specific keys, including handler key names such as # `admin_password` or `api_handler_password`. Replace the empty defaults with values @@ -44,6 +46,8 @@ allowed_outbound_hosts = ["https://*:*", "http://*:*"] key_value_stores = ["default"] [component.trusted-server.variables] +v_current_x2dkid = "{{ v_current_x2dkid }}" +v_active_x2dkids = "{{ v_active_x2dkids }}" v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret = "{{ v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret }}" v_trusted_x5fserver_x5fsecrets_v_trusted_x5fclient_x5fip_x5fshared_x5fsecret = "{{ v_trusted_x5fserver_x5fsecrets_v_trusted_x5fclient_x5fip_x5fshared_x5fsecret }}" v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase = "{{ v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase }}" diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index a81911186..c60aa2271 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -195,14 +195,9 @@ fn spin_secret_variable_name( /// /// Delegates all operations through `KvHandle`'s raw-bytes API. Spin KV has no /// native TTL support, so [`put_bytes_with_ttl`](KvStore::put_bytes_with_ttl) -/// *errors* (`KvError::Validation`) rather than silently writing a non-expiring -/// record — the privacy-safe failure mode. Consequently TTL-backed consent -/// persistence (`save_consent_to_kv`) is unavailable on Spin: each write returns -/// the error, which the core caller logs and treats as non-fatal (consistent -/// with all adapters — failing to persist consent never breaks the request, and -/// not persisting is the safe direction). Operators configuring -/// `settings.consent.consent_store` on Spin should expect stored-consent -/// fallback not to function. +/// returns `KvError::Validation` rather than silently writing a non-expiring +/// record. Callers of the generic platform KV interface must handle that +/// capability difference explicitly. struct KvHandleAdapter(KvHandle); #[async_trait::async_trait(?Send)] @@ -788,6 +783,12 @@ mod tests { use edgezero_core::body::Body; use edgezero_core::config_store::{ConfigStore, ConfigStoreError}; + use edgezero_core::context::RequestContext; + use edgezero_core::http::request_builder; + use edgezero_core::params::PathParams; + use flate2::Compression; + use flate2::write::GzEncoder; + use std::io::Write as _; use trusted_server_core::platform::AuctionTargetId; #[test] @@ -799,12 +800,6 @@ mod tests { "Spin outbound HTTP does not expose an enforceable hard total request deadline" ); } - use edgezero_core::context::RequestContext; - use edgezero_core::http::request_builder; - use edgezero_core::params::PathParams; - use flate2::Compression; - use flate2::write::GzEncoder; - use std::io::Write as _; struct InMemoryConfigStore(std::collections::BTreeMap); @@ -954,6 +949,30 @@ mod tests { ); } + #[test] + fn spin_variable_name_encodes_trusted_server_keys() { + assert_eq!( + spin_variable_name("current-kid", PlatformError::ConfigStore) + .expect("should encode current kid key"), + "v_current_x2dkid" + ); + assert_eq!( + spin_variable_name("active-kids", PlatformError::ConfigStore) + .expect("should encode active kids key"), + "v_active_x2dkids" + ); + assert_eq!( + spin_variable_name("ts-2026-05-25", PlatformError::ConfigStore) + .expect("should encode generated kid"), + "v_ts_x2d2026_x2d05_x2d25" + ); + // Digit-leading keys are rejected at the encoder boundary. + assert!( + spin_variable_name("2026-key", PlatformError::ConfigStore).is_err(), + "should reject digit-leading key" + ); + } + #[test] fn spin_variable_name_encodes_secret_key_components() { assert_eq!( diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 44ad1d443..0f93651c5 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -17,18 +17,24 @@ workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] chromiumoxide = { workspace = true } clap = { workspace = true } +derive_more = { workspace = true } edgezero-cli = { workspace = true } +edgezero-core = { workspace = true } futures = { workspace = true } +glob = { workspace = true } +http = { workspace = true } log = { workspace = true } rand = { workspace = true } regex = { workspace = true } scraper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +similar = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } +tracing = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } which = { workspace = true } @@ -42,7 +48,6 @@ which = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] base64 = { workspace = true } bytes = { workspace = true } -derive_more = { workspace = true } directories = { workspace = true } error-stack = { workspace = true } http-body-util = { workspace = true } @@ -59,8 +64,9 @@ tokio-rustls = { workspace = true } webpki-roots = { workspace = true } [target.'cfg(target_os = "macos")'.dev-dependencies] -tokio = { workspace = true, features = ["test-util"] } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs new file mode 100644 index 000000000..48ab71f3e --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -0,0 +1,830 @@ +//! Pure comparison of configured expected slots against browser ad evidence. +//! +//! This module is collector-independent and Chrome-free: it takes decoded +//! [`BrowserAdEvidence`] plus the [`ExpectedSlot`] set and produces a +//! [`PageVerificationResult`] with per-slot statuses, warnings, and unmatched +//! extra evidence, mirroring spec §5.3–§5.6. +//! +use serde::Deserialize; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +use crate::ad_templates::expected::ExpectedSlot; +use crate::ad_templates::output::Warning; + +/// The phase in which a piece of evidence was observed. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhase { + /// Observed during the initial load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A DOM element ID observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct DomEvidence { + /// The element ID. + pub dom_id: String, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// A GPT slot observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct GptSlotEvidence { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `(width, height)` pairs (non-numeric dropped upstream). + pub sizes: Vec<(u32, u32)>, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// An `apstag.fetchBids` call the page made, if any were recorded. +/// +/// The collector no longer hooks `apstag`: server-side APS configuration is +/// metadata rather than a client assertion, so a missing client call is not a +/// finding. The field and this shape stay for the evidence payload's schema, and +/// the list arrives empty. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "decoded for schema stability; the collector records no APS calls" +)] +pub struct ApsFetchBidsEvidence { + /// The APS slot ID requested. + pub slot_id: String, + /// Sizes requested for the slot. + pub sizes: Vec<(u32, u32)>, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// A `/__ts/page-bids` observation for SPA routes (spec §5.2). +/// +/// DEFERRED in Phase 1: kept as forward scaffolding so the decoded evidence shape +/// stays forward-compatible. Not populated by the collector or surfaced in JSON. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "reserved decoded shape for the optional bids phase" +)] +pub struct PageBidsEvidence { + /// The slot ID present in the page-bids response. + pub slot_id: String, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// All read-only ad evidence decoded from a single browser page. +#[derive(Debug, Clone, Deserialize)] +pub struct BrowserAdEvidence { + /// DOM element IDs matching configured prefixes. + pub dom_ids: Vec, + /// GPT slots observed via `defineSlot` and `getSlots()`. + pub gpt_slots: Vec, + /// `apstag.fetchBids` calls observed. + pub aps_calls: Vec, + /// `/__ts/page-bids` observations (deferred; default empty). + #[serde(default)] + #[allow(dead_code, reason = "reserved for the optional bids phase")] + pub page_bids: Vec, + /// Collector-level warnings (no page HTML/cookies/storage). + #[serde(default)] + pub warnings: Vec, +} + +/// Summary of the runtime ad-stack gate for a page. +#[derive(Debug, Clone, Copy)] +pub struct RuntimeGateSummary { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, +} + +impl RuntimeGateSummary { + /// Builds a summary from a computed runtime expectation. + #[must_use] + pub fn from_expected(expected: RuntimeAdStackExpected) -> Self { + Self { expected } + } + + #[cfg(test)] + fn unknown_allowed() -> Self { + Self::from_expected(RuntimeAdStackExpected::Unknown) + } + + #[cfg(test)] + fn auction_disabled() -> Self { + Self::from_expected(RuntimeAdStackExpected::No) + } +} + +/// Confirmation status for a single configured slot (compare-side mirror of the +/// output `SlotStatus`). +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker cannot confirm this slot type; this is not page drift. + Unconfirmable, +} + +/// The verification result for one audited page. +#[derive(Debug, Clone)] +pub struct PageVerificationResult { + /// Whether the runtime ad stack was expected to run for this page. + pub runtime_ad_stack_expected: RuntimeAdStackExpected, + /// Per-slot results, in expected-slot order. + pub slots: Vec, + /// Live evidence that matched no configured slot. + pub extra_evidence: Vec, +} + +impl PageVerificationResult { + /// Whether `--strict` should fail for this page. + /// + /// False when the runtime ad stack is not expected to run (a known gate + /// suppressed it); otherwise true if any slot is missing or partial. Provider + /// warnings and extra evidence alone never fail strict. + #[must_use] + pub fn strict_failed(&self) -> bool { + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } +} + +/// Per-slot verification result. +#[derive(Debug, Clone)] +pub struct SlotResult { + /// The configured slot id. + pub id: String, + /// The confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + pub phase: Option, + /// The live evidence observed for this slot. + pub evidence: SlotEvidence, + /// Slot-level warnings (size, provider, etc.). + pub warnings: Vec, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone)] +pub struct SlotEvidence { + /// The resolved DOM element ID, if any. + pub dom_id: Option, + /// The matched GPT slot, if any. + pub gpt: Option, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone)] +pub struct ExtraEvidence { + /// Evidence kind. Only `gpt` is produced today; the field is a string so a + /// later evidence source can be added without changing the JSON schema. + pub kind: String, + /// The phase it was observed in. + pub phase: EvidencePhase, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes. + pub sizes: Vec<(u32, u32)>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +fn warning(code: &str, message: String) -> Warning { + Warning { + code: code.to_string(), + message, + } +} + +/// Resolves the slot root DOM element per spec §5.3. +/// +/// Exact `div_id` match first, then the first element whose ID starts with +/// `div_id`, ignoring `-container` wrappers. +fn resolve_dom<'a>(dom_ids: &'a [DomEvidence], div_id: &str) -> Option<&'a DomEvidence> { + if let Some(exact) = dom_ids.iter().find(|dom| dom.dom_id == div_id) { + return Some(exact); + } + dom_ids + .iter() + .find(|dom| dom.dom_id.starts_with(div_id) && !dom.dom_id.ends_with("-container")) +} + +/// Returns true when a GPT slot's element ID matches the resolved DOM id (or its +/// `-container`), per spec §5.4. +fn gpt_div_matches(gpt_div: &str, expected: &ExpectedSlot, resolved_dom_id: Option<&str>) -> bool { + match resolved_dom_id { + Some(dom_id) => gpt_div == dom_id || gpt_div == format!("{dom_id}-container"), + None => { + gpt_div == expected.div_id + || (gpt_div.starts_with(&expected.div_id) && !gpt_div.ends_with("-container")) + } + } +} + +fn banner_sizes(expected: &ExpectedSlot) -> Vec<(u32, u32)> { + expected + .formats + .iter() + .filter(|format| format.media_type == MediaType::Banner) + .map(|format| (format.width, format.height)) + .collect() +} + +/// Compares configured expected slots against decoded browser evidence. +#[must_use] +pub fn compare_page_evidence( + expected: &[ExpectedSlot], + evidence: &BrowserAdEvidence, + gate: RuntimeGateSummary, +) -> PageVerificationResult { + let mut consumed_gpt = vec![false; evidence.gpt_slots.len()]; + let mut slots = Vec::with_capacity(expected.len()); + + for slot in expected { + let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); + let resolved_id = resolved.map(|dom| dom.dom_id.clone()); + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) + }); + + let banner = banner_sizes(slot); + let mut warnings = Vec::new(); + // `expected_slots_for_path` drops a slot whose template does not render, + // so on the verify path this arm is unreachable; it exists for callers + // that build expected slots directly, and as a guard if that filter ever + // changes. + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime omits this slot on this path", + slot.id + ), + )); + } + + let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { + consumed_gpt[idx] = true; + let gpt = &evidence.gpt_slots[idx]; + let dom_id = resolved_id.clone().or_else(|| Some(gpt.div_id.clone())); + if banner.is_empty() { + warnings.push(warning( + "unsupported_format", + format!( + "slot `{}` has only non-banner formats; not confirmable in Phase 1", + slot.id + ), + )); + ( + SlotStatus::Unconfirmable, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if gpt.sizes.is_empty() { + warnings.push(warning( + "out_of_page_slot", + format!( + "slot `{}` matched an out-of-page GPT slot with no sizes", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if banner.iter().any(|size| gpt.sizes.contains(size)) { + let extra: Vec<(u32, u32)> = gpt + .sizes + .iter() + .copied() + .filter(|size| !banner.contains(size)) + .collect(); + if !extra.is_empty() { + warnings.push(warning( + "extra_observed_size", + format!("slot `{}` observed extra GPT sizes {extra:?}", slot.id), + )); + } + let missing: Vec<(u32, u32)> = banner + .iter() + .copied() + .filter(|size| !gpt.sizes.contains(size)) + .collect(); + if !missing.is_empty() { + warnings.push(warning( + "configured_size_not_observed", + format!( + "slot `{}` configured sizes {missing:?} were not observed", + slot.id + ), + )); + } + ( + SlotStatus::Confirmed, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else { + warnings.push(warning( + "incompatible_sizes", + format!( + "slot `{}` GPT path and div matched but no configured size overlapped", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } + } else if let Some(dom) = resolved { + warnings.push(warning( + "dom_without_gpt", + "DOM element matched, but no GPT slot evidence was observed".to_string(), + )); + ( + SlotStatus::Partial, + Some(dom.dom_id.clone()), + None, + Some(dom.phase), + ) + } else { + (SlotStatus::Missing, None, None, None) + }; + + slots.push(SlotResult { + id: slot.id.clone(), + status, + phase, + evidence: SlotEvidence { + dom_id: dom_for_evidence, + gpt: gpt_for_evidence, + }, + warnings, + }); + } + + let extra_evidence = evidence + .gpt_slots + .iter() + .enumerate() + .filter(|(idx, _)| !consumed_gpt[*idx]) + .map(|(_, gpt)| ExtraEvidence { + kind: "gpt".to_string(), + phase: gpt.phase, + dom_id: Some(gpt.div_id.clone()), + gam_unit_path: Some(gpt.gam_unit_path.clone()), + sizes: gpt.sizes.clone(), + reason: "no_configured_slot_matched".to_string(), + }) + .collect(); + + PageVerificationResult { + runtime_ad_stack_expected: gate.expected, + slots, + extra_evidence, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::expected::ExpectedFormat; + + fn dom(id: &str) -> DomEvidence { + DomEvidence { + dom_id: id.to_string(), + phase: EvidencePhase::InitialLoad, + } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { + slot_id: slot_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn evidence( + doms: Vec, + gpts: Vec, + aps: Vec, + ) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn expected_slot( + id: &str, + div_id: &str, + gam_unit_path: &str, + sizes: &[(u32, u32)], + providers: &[&str], + ) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: sizes + .iter() + .map(|&(width, height)| ExpectedFormat { + width, + height, + media_type: MediaType::Banner, + }) + .collect(), + providers: providers.iter().copied().map(String::from).collect(), + page_patterns: Vec::new(), + } + } + + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: vec![ExpectedFormat { + width: 0, + height: 0, + media_type: MediaType::Video, + }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + result.slots[0].warnings.is_empty(), + "confirmed slot should carry no warnings" + ); + } + + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "dom_without_gpt") + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Missing); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot( + "header", + "ad-header-0-", + "/123/homepage/header", + &[(728, 90)], + &[], + ); + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].evidence.dom_id.as_deref(), + Some("ad-header-0-_R_abc123"), + "prefix match should skip -container" + ); + assert_eq!(result.slots[0].status, SlotStatus::Partial); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot( + "/123/publisher/right-rail", + "ad-right-rail-0", + &[(300, 250)], + ), + ], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert_eq!(result.extra_evidence.len(), 1); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!( + !result.strict_failed(), + "extra evidence alone must not fail strict" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::auction_disabled(), + ); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No); + assert_eq!(result.slots[0].status, SlotStatus::Missing); + assert!( + !result.strict_failed(), + "missing slot must not fail strict when ad stack is No" + ); + } + + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "incompatible_sizes") + ); + } + + #[test] + fn non_banner_only_slot_is_unconfirmable_and_does_not_fail_strict() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "unsupported_format") + ); + assert!( + !result.strict_failed(), + "checker limitations should not fail strict" + ); + } + + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot( + "/123/news/atf", + "ad-atf-0-container", + &[(300, 250)], + )], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "container element id is a valid GPT div match" + ); + } + + #[test] + fn sizeless_live_slot_is_partial_when_config_declares_banner_sizes() { + let expected = expected_slot( + "interstitial", + "ad-oop-", + "/123/news/oop", + &[(300, 250)], + &[], + ); + let evidence = evidence( + vec![dom("ad-oop-0")], + vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "out_of_page_slot") + ); + assert!( + result.strict_failed(), + "a live sizeless slot drifting from configured banner sizes must fail strict" + ); + } + + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + !result.slots[0] + .warnings + .iter() + .any(|w| w.code.starts_with("aps_")), + "matching APS should not warn" + ); + } + + #[test] + fn server_side_aps_config_does_not_require_client_fetch_bids_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "missing APS does not flip status" + ); + assert!(result.slots[0].warnings.is_empty()); + assert!( + !result.strict_failed(), + "provider warning alone must not fail strict" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs new file mode 100644 index 000000000..9392963ff --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -0,0 +1,336 @@ +//! Pure expected-slot projection from the runtime creative-opportunity matcher. +//! +//! This module owns path/URL normalization and converts the slots matched by +//! [`match_slots`] into stable, owned [`ExpectedSlot`] records for output and +//! browser-evidence comparison. It must not duplicate glob-matching semantics. + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{CreativeOpportunitiesConfig, match_slots}; +use url::Url; + +/// The expected slots for a single page path, in configured slot order. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlots { + /// The page path the slots were matched against. + pub path: String, + /// Matched slots projected into stable records, in configured order. + pub slots: Vec, +} + +/// A single configured slot expected to appear for a page path. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlot { + /// The slot identifier. + pub id: String, + /// Resolved HTML `div` element ID (override or the slot id). + pub div_id: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` only for manually constructed comparison fixtures. Projection + /// omits a slot when the runtime cannot render it for this path. + pub gam_unit_path: Option, + /// Configured ad formats. + pub formats: Vec, + /// Configured provider names, in `aps`, `prebid` order. + pub providers: Vec, + /// Glob patterns configured for this slot. + pub page_patterns: Vec, +} + +/// A configured ad format as a stable width/height/media-type record. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedFormat { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Configured media type. + pub media_type: MediaType, +} + +/// Projects the slots matching `path` into stable expected-slot records. +/// +/// Uses [`match_slots`] so glob semantics stay identical to the runtime, and +/// preserves configured slot order. `path` is assumed already normalized via +/// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. +// Shared projection used by the audit verifier; the static commands match slots +// directly against the runtime matcher. +#[must_use] +pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); + let slots = match_slots(&config.slot, path) + .into_iter() + .filter_map(|slot| { + let gam_unit_path = slot.render_gam_unit_path(&config.gam_network_id, §ion)?; + Some(ExpectedSlot { + id: slot.id.clone(), + div_id: slot.resolved_div_id().to_string(), + gam_unit_path: Some(gam_unit_path), + formats: slot + .formats + .iter() + .map(|format| ExpectedFormat { + width: format.width, + height: format.height, + media_type: format.media_type.clone(), + }) + .collect(), + providers: provider_names(slot), + page_patterns: slot.page_patterns.clone(), + }) + }) + .collect(); + + ExpectedSlots { + path: path.to_string(), + slots, + } +} + +fn provider_names( + slot: &trusted_server_core::creative_opportunities::CreativeOpportunitySlot, +) -> Vec { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps".to_string()); + } + if slot.providers.prebid.is_some() { + providers.push("prebid".to_string()); + } + providers +} + +/// Normalizes a page path or full URL into a request path. +/// +/// Full `scheme://` inputs are parsed and reduced to their path; bare inputs have +/// query and fragment stripped and a leading `/` ensured. Empty paths become `/`. +/// +/// # Errors +/// +/// Returns a user-facing string when a `scheme://` input cannot be parsed as a URL. +pub fn normalize_path_or_url(input: &str) -> Result { + let path_input = input.split(['?', '#']).next().unwrap_or(input); + let scheme_prefix = path_input.split_once("://").map(|(scheme, _)| scheme); + let has_url_scheme = scheme_prefix.is_some_and(|scheme| { + let mut chars = scheme.chars(); + chars.next().is_some_and(|ch| ch.is_ascii_alphabetic()) + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '-' | '.')) + }); + if has_url_scheme { + let url = Url::parse(input).map_err(|err| format!("invalid URL `{input}`: {err}"))?; + let path = url.path(); + return Ok(if path.is_empty() { + "/".to_string() + } else { + path.to_string() + }); + } + + let base = Url::parse("https://path-normalizer.example/") + .expect("should parse static path normalization base"); + let relative = input.trim_start_matches('/'); + let normalized = base + .join(&format!("./{relative}")) + .map_err(|error| format!("invalid path `{input}`: {error}"))?; + Ok(normalized.path().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [{page_patterns}]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + \n\ + [slot.providers.prebid]\n\ + bidders = {{}}\n" + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(&["/news/*", "/"]); + let expected = expected_slots_for_path("/news/story", &config); + + assert_eq!(expected.path, "/news/story"); + assert_eq!( + expected + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(), + ["atf"] + ); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); + assert_eq!(expected.slots[0].providers, ["prebid"]); + assert_eq!( + expected.slots[0].formats, + vec![ExpectedFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }] + ); + } + + #[test] + fn expected_slots_default_resolution_without_overrides() { + let toml = "gam_network_id = \"42\"\n\ + \n\ + [[slot]]\n\ + id = \"footer\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let expected = expected_slots_for_path("/", &config); + assert_eq!(expected.slots[0].div_id, "footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); + assert!(expected.slots[0].providers.is_empty()); + } + + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_omit_dynamic_template_the_runtime_cannot_render() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. The runtime omits this slot for the request path, so diagnostics + // must not match it against a truncated or otherwise different path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert!( + expected.slots.is_empty(), + "the runtime omits an over-limit dynamic slot on this path" + ); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!( + normalize_path_or_url("https://www.example.com/news/story?x=1#top") + .expect("should normalize"), + "/news/story" + ); + assert_eq!( + normalize_path_or_url("news/story?x=1").expect("should normalize"), + "/news/story" + ); + } + + #[test] + fn normalize_path_or_url_roots_empty_input() { + assert_eq!( + normalize_path_or_url("https://www.example.com").expect("should normalize"), + "/" + ); + assert_eq!(normalize_path_or_url("").expect("should normalize"), "/"); + } + + #[test] + fn normalize_path_or_url_uses_identical_url_rules_for_bare_paths() { + assert_eq!( + normalize_path_or_url("/a/../b").expect("should normalize bare dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("https://example.com/a/../b") + .expect("should normalize URL dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("/a b").expect("should encode bare path"), + "/a%20b" + ); + assert_eq!( + normalize_path_or_url("/r?to=https://example.com") + .expect("query URL should not change input classification"), + "/r" + ); + assert_eq!( + normalize_path_or_url("/news:latest").expect("colon should stay in bare path"), + "/news:latest", + "a colon in the first segment must not be parsed as a URL scheme" + ); + assert_eq!( + normalize_path_or_url("https://example.com/news:latest") + .expect("colon should stay in URL path"), + "/news:latest", + "bare and absolute forms should normalize identically" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/mod.rs b/crates/trusted-server-cli/src/ad_templates/mod.rs new file mode 100644 index 000000000..3c26bf121 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/mod.rs @@ -0,0 +1,7 @@ +//! Pure, host-only ad-template CLI logic shared by the static `ts config +//! ad-templates ...` commands and the browser-backed `ts audit ad-templates +//! verify` command. + +pub mod compare; +pub mod expected; +pub mod output; diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs new file mode 100644 index 000000000..afcf78ed0 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -0,0 +1,483 @@ +//! Stable, serializable output model for ad-template diagnostics. +//! +//! These types mirror the `--json` contract in +//! `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` §8. +//! Field names and declaration order are load-bearing: `serde` serializes struct +//! fields in declaration order, so the order here must match the spec examples. +//! +//! The model is consumed by the `ts audit ad-templates verify` orchestrator, +//! which assembles these wire types from the URL/gate context and comparison result. + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 + || (0x7f..=0x9f).contains(&code) + || (0x202a..=0x202e).contains(&code) + || (0x2066..=0x2069).contains(&code) +} + +/// Confirmation status for a single configured slot. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker does not support confirming this slot type. + Unconfirmable, +} + +/// JSON rendering of the runtime ad-stack expectation. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeAdStackExpectedJson { + /// The server-side ad stack is expected to run. + Yes, + /// A known gate blocks the server-side ad stack. + No, + /// Consent or another gate is unprovable. + Unknown, +} + +impl From for RuntimeAdStackExpectedJson { + fn from(value: RuntimeAdStackExpected) -> Self { + match value { + RuntimeAdStackExpected::Yes => Self::Yes, + RuntimeAdStackExpected::No => Self::No, + RuntimeAdStackExpected::Unknown => Self::Unknown, + } + } +} + +/// State of a single runtime gate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GateState { + /// The gate passed. + Pass, + /// The gate blocked the ad stack. + Fail, + /// The gate state could not be proven. + Unknown, +} + +/// Evidence-collection phase, rendered for JSON output. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhaseJson { + /// Observed during the initial page load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A structured warning with a stable machine code and human message. +/// +/// `Serialize` for output; `Deserialize` because the browser collector payload +/// carries warning objects decoded into the comparison input. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Warning { + /// Stable machine-readable code (e.g. `dom_without_gpt`). + pub code: String, + /// Human-readable message; JSON consumers must not parse this. + pub message: String, +} + +/// Top-level `--json` document for `ts audit ad-templates verify`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct VerificationReport { + /// True when no strict failure and no page-level error occurred. + pub ok: bool, + /// Whether `--strict` was set. + pub strict: bool, + /// One entry per requested URL, in input order. + pub pages: Vec, + /// Run-level warnings not attributable to a single page. + /// + /// Always empty today — every warning the verifier raises belongs to a page + /// or a slot. Kept because the JSON schema declares it, so a consumer can + /// read it unconditionally. + pub warnings: Vec, +} + +/// A single audited page result. +/// +/// `error` is declared immediately after `path` so the serialized key order +/// matches the spec §8 `navigation_failed` shape; on normal pages it is `None` +/// and skipped, leaving the runtime/gates fields in §8 order. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PageJson { + /// The requested URL. + pub url: String, + /// The final URL after redirects, or `null` on navigation failure. + pub final_url: Option, + /// The requested URL's path. + pub requested_path: String, + /// The final path used for matching, or `null` on navigation failure. + pub path: Option, + /// Present only on a page-level collection failure. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Three-state runtime ad-stack expectation; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + /// Per-gate evidence; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + /// Number of configured slots matched for the final path; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + /// Per-slot verification results. + pub slots: Vec, + /// Live ad-slot evidence with no matching configured slot. + pub extra_evidence: Vec, + /// Page-level warnings. + pub warnings: Vec, +} + +/// Runtime gate states for a page, one field per spec §5.2 gate. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Gates { + /// Request method is `GET`. + pub method_get: GateState, + /// Request is a top-level navigation. + pub navigation: GateState, + /// Request is not a prefetch. + pub not_prefetch: GateState, + /// Request is not from a known bot. + pub not_bot: GateState, + /// At least one configured slot matched the final path. + pub matched_slots: GateState, + /// The `[auction].enabled` kill switch is on. + pub auction_enabled: GateState, + /// The `[creative_opportunities].enabled` template switch is on. + pub ad_templates_enabled: GateState, + /// Consent allows the auction (often `unknown` for live requests). + pub consent_allows_auction: GateState, +} + +/// A single configured slot's verification result. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotJson { + /// The configured slot id. + pub id: String, + /// The slot's confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + #[serde(skip_serializing_if = "Option::is_none")] + pub phase: Option, + /// The configured shape of the slot (no `id`/`page_patterns` per §8). + pub configured: ConfiguredJson, + /// The live evidence observed for this slot. + pub evidence: SlotEvidenceJson, + /// Slot-level warnings (e.g. provider or size warnings). + pub warnings: Vec, +} + +/// The configured shape of a slot, as rendered in §8 `configured`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ConfiguredJson { + /// Resolved div element ID. + pub div_id: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, + /// Configured formats. + pub formats: Vec, + /// Configured provider names. + pub providers: Vec, +} + +/// A configured format, as rendered in §8. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct FormatJson { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type string (`banner`, `video`, `native`). + pub media_type: String, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotEvidenceJson { + /// The resolved DOM element ID observed, if any. + pub dom_id: Option, + /// GPT slot evidence, if any (no `phase` key per §8). + pub gpt: Option, +} + +/// GPT slot evidence, as rendered in §8 `evidence.gpt`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct GptEvidenceJson { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ExtraEvidenceJson { + /// Evidence kind: `dom`, `gpt`, or `aps`. + pub kind: String, + /// The phase the evidence was observed in. + pub phase: EvidencePhaseJson, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +#[cfg(test)] +impl VerificationReport { + fn example_confirmed_with_extra_evidence() -> Self { + VerificationReport { + ok: true, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/news/story".to_string(), + final_url: Some("https://www.example.com/news/story".to_string()), + requested_path: "/news/story".to_string(), + path: Some("/news/story".to_string()), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::Unknown), + gates: Some(Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: GateState::Pass, + auction_enabled: GateState::Pass, + ad_templates_enabled: GateState::Pass, + consent_allows_auction: GateState::Unknown, + }), + matched_slot_count: Some(1), + slots: vec![SlotJson { + id: "atf".to_string(), + status: SlotStatus::Confirmed, + phase: Some(EvidencePhaseJson::InitialLoad), + configured: ConfiguredJson { + div_id: "ad-atf-".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), + formats: vec![FormatJson { + width: 300, + height: 250, + media_type: "banner".to_string(), + }], + providers: vec!["aps".to_string()], + }, + evidence: SlotEvidenceJson { + dom_id: Some("ad-atf-0".to_string()), + gpt: Some(GptEvidenceJson { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![[300, 250]], + }), + }, + warnings: Vec::new(), + }], + extra_evidence: vec![ExtraEvidenceJson { + kind: "gpt".to_string(), + phase: EvidencePhaseJson::InitialLoad, + dom_id: Some("ad-right-rail-0".to_string()), + gam_unit_path: Some("/123/publisher/right-rail".to_string()), + sizes: vec![[300, 250]], + reason: "no_configured_slot_matched".to_string(), + }], + warnings: vec![Warning { + code: "redirected".to_string(), + message: "navigation redirected to the final path".to_string(), + }], + }], + warnings: Vec::new(), + } + } + + fn example_navigation_failed() -> Self { + VerificationReport { + ok: false, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/broken".to_string(), + final_url: None, + requested_path: "/broken".to_string(), + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: "failed to read main document navigation response".to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + }], + warnings: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + assert_eq!( + escape_terminal_text("safe\u{202E}forged\u{2066}tail"), + "safe\\u{202E}forged\\u{2066}tail", + "Unicode bidi controls should be rendered inert" + ); + } + + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!( + value["pages"][0]["gates"]["consent_allows_auction"], + "unknown" + ); + assert_eq!(value["pages"][0]["slots"][0]["status"], "confirmed"); + assert_eq!( + value["pages"][0]["slots"][0]["evidence"]["gpt"]["sizes"][0][0], + 300 + ); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + // `configured` excludes id/page_patterns per §8. + assert!(value["pages"][0]["slots"][0]["configured"]["id"].is_null()); + assert!(value["pages"][0]["slots"][0]["configured"]["page_patterns"].is_null()); + // `evidence.gpt` has no `phase` key per §8. + assert!(value["pages"][0]["slots"][0]["evidence"]["gpt"]["phase"].is_null()); + } + + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!( + page.get("runtime_ad_stack_expected").is_none(), + "runtime field absent on error page" + ); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!( + page.get("matched_slot_count").is_none(), + "matched_slot_count absent on error page" + ); + assert_eq!(value["ok"], false); + } + + #[test] + fn missing_slot_json_omits_evidence_phase() { + let slot = SlotJson { + id: "missing".to_string(), + status: SlotStatus::Missing, + phase: None, + configured: ConfiguredJson { + div_id: "ad-missing-".to_string(), + gam_unit_path: Some("/123/publisher/missing".to_string()), + formats: Vec::new(), + providers: Vec::new(), + }, + evidence: SlotEvidenceJson { + dom_id: None, + gpt: None, + }, + warnings: Vec::new(), + }; + + let value = serde_json::to_value(slot).expect("should serialize missing slot"); + + assert!( + value.get("phase").is_none(), + "missing evidence should not claim an initial-load phase" + ); + } +} diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs new file mode 100644 index 000000000..bee536146 --- /dev/null +++ b/crates/trusted-server-cli/src/app_config.rs @@ -0,0 +1,171 @@ +//! Shared effective Trusted Server app-config loading for the `ts` CLI. +//! +//! Both the static `ts config ad-templates ...` commands and the browser-backed +//! `ts audit ad-templates verify` command load the same effective app config +//! through [`load_settings`], so config-path resolution and the `EdgeZero` +//! environment overlay stay consistent across command families. + +use std::path::{Path, PathBuf}; + +use clap::Args; +use edgezero_core::app_config::{self, AppConfigLoadOptions}; +use edgezero_core::manifest::ManifestLoader; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::settings::Settings; + +/// Shared local app-config flags accepted by every config/audit ad-template command. +#[derive(Clone, Debug, Args)] +pub struct AppConfigArgs { + /// Path to `trusted-server.toml`. Defaults to `.toml` beside `edgezero.toml`. + #[arg(long)] + pub app_config: Option, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Skip app-config environment overlay. + #[arg(long)] + pub no_env: bool, +} + +/// Effective settings plus the resolved app-config path they were loaded from. +#[derive(Debug)] +pub struct LoadedSettings { + /// The `trusted-server.toml` path the settings were loaded from. + pub app_config_path: PathBuf, + /// The deserialized effective settings. + pub settings: Settings, +} + +/// Loads the effective Trusted Server settings described by `args`. +/// +/// Resolves the app-config path from `args` (or the manifest's `.toml` +/// default), applies the `EdgeZero` environment overlay unless `no_env` is set, and +/// returns the deserialized [`Settings`]. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded, has no +/// `[app].name`, or the resolved app-config file cannot be read or parsed. When an +/// explicit `--app-config` path is given and is missing, the error names that +/// exact path rather than silently falling back. +pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +#[cfg(test)] +pub(crate) fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +/// Resolves the operator-owned app-config path without deserializing settings. +/// +/// Mutating recovery commands use this when the existing config may already be +/// invalid but still needs a narrowly scoped structural repair. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded or has no +/// `[app].name` and no explicit config path was supplied. +pub fn resolve_app_config_file(args: &AppConfigArgs) -> Result { + if let Some(path) = &args.app_config { + return Ok(path.clone()); + } + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + Ok(resolve_app_config_path(None, &args.manifest, &app_name)) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = env_overlay; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) +} + +fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js new file mode 100644 index 000000000..6938808f5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -0,0 +1,241 @@ +// Bounded ad-template evidence collector, injected before publisher scripts run. +// +// This body runs inside an IIFE that defines `__TS_CONFIG` (the configured div +// prefixes). It records evidence into `window.__tsAdTemplateEvidence` +// and never captures page HTML, cookies, storage, request bodies, or arbitrary DOM. +// It always calls original page functions with unchanged arguments and never +// spoofs the browser automation flag. + +const __ts_config = typeof __TS_CONFIG === "object" && __TS_CONFIG ? __TS_CONFIG : {} +const __ts_prefixes = Array.isArray(__ts_config.div_prefixes) ? __ts_config.div_prefixes : [] + +const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence || { + dom_ids: [], + gpt_slots: [], + aps_calls: [], + warnings: [] +}) + +const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") + +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 128 +const __ts_max_string_length = 512 +const __ts_wrapped_googletags = new WeakSet() + +function __ts_text(value) { + return String(value).slice(0, __ts_max_string_length) +} + +// Truncation has to be visible: surplus configured slots classify Missing, and +// `--strict` counts that, so a silent drop is indistinguishable from real drift. +let __ts_truncated = false +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) { + list.push(entry) + return + } + if (__ts_truncated) return + __ts_truncated = true + if (__ts_ev.warnings.length < __ts_max_entries) { + __ts_ev.warnings.push({ + code: "evidence_truncated", + message: "an evidence list hit the " + __ts_max_entries + "-entry cap; results are incomplete" + }) + } +} + +function __ts_warn(code, error) { + __ts_push(__ts_ev.warnings, { code, message: __ts_text(error) }) +} + +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) return null + return [width, height] +} + +function __ts_warn_ignored_size(width, height) { + const numeric = Number.isInteger(width) && Number.isInteger(height) + const outOfRange = + numeric && (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) + __ts_push(__ts_ev.warnings, { + code: outOfRange ? "size_out_of_range" : "fluid_size_ignored", + message: outOfRange ? "GPT size outside u32 range ignored" : "non-integer GPT size ignored" + }) +} + +function __ts_normalize_sizes(sizes) { + const out = [] + if (!Array.isArray(sizes)) return out + // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. + const pairs = typeof sizes[0] === "number" ? [sizes] : sizes + for (const size of pairs) { + if (out.length >= __ts_max_entries) break + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) + } else { + __ts_warn_ignored_size( + Array.isArray(size) ? size[0] : undefined, + Array.isArray(size) ? size[1] : undefined + ) + } + } + return out +} + +function __ts_record_define_slot(adUnitPath, sizes, divId) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(adUnitPath), + div_id: __ts_text(divId), + sizes: __ts_normalize_sizes(sizes), + phase: __ts_phase() + }) +} + +function __ts_wrap_googletag(googletag) { + if (!googletag || (typeof googletag !== "object" && typeof googletag !== "function")) { + return googletag + } + if (__ts_wrapped_googletags.has(googletag)) return googletag + __ts_wrapped_googletags.add(googletag) + // Wrap defineSlot so both direct calls and calls dispatched from the cmd queue + // are recorded (queued callbacks call this same wrapped function). + const originalDefineSlot = googletag.defineSlot + if (typeof originalDefineSlot === "function") { + try { + const descriptor = Object.getOwnPropertyDescriptor(googletag, "defineSlot") + Object.defineProperty(googletag, "defineSlot", { + configurable: true, + enumerable: descriptor ? descriptor.enumerable : true, + writable: true, + value: function (adUnitPath, sizes, divId) { + const slot = originalDefineSlot.apply(this, arguments) + try { + __ts_record_define_slot(adUnitPath, sizes, divId) + } catch (error) { + __ts_warn("define_slot_capture_failed", error) + } + return slot + } + }) + } catch (error) { + __ts_warn("define_slot_wrap_failed", error) + } + } + return googletag +} + +// Wrap an existing global or intercept a later assignment of it. +function __ts_install(name, wrap) { + if (window[name]) { + try { + wrap(window[name]) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + return + } + let internal + Object.defineProperty(window, name, { + configurable: true, + // A real `window.googletag` is an ordinary enumerable global; matching that + // keeps `Object.keys(window)` identical with and without the collector. + enumerable: true, + get() { + return internal + }, + set(value) { + internal = value + try { + internal = wrap(value) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + } + }) +} + +__ts_install("googletag", __ts_wrap_googletag) + +// On-demand DOM + getSlots scrape, invoked by the collector after settle/scroll. +window.__tsCollectAdTemplateEvidence = function () { + try { + const seen = new Set(__ts_ev.dom_ids.map((entry) => entry.dom_id)) + for (const element of document.querySelectorAll("[id]")) { + const id = __ts_text(element.id) + if (id.endsWith("-container")) continue + if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) + seen.add(id) + } + } + const googletag = window.googletag + if (googletag && typeof googletag.pubads === "function") { + const pubads = googletag.pubads() + const slots = typeof pubads.getSlots === "function" ? pubads.getSlots() : [] + for (const slot of slots) { + try { + const path = typeof slot.getAdUnitPath === "function" ? slot.getAdUnitPath() : "" + const divId = typeof slot.getSlotElementId === "function" ? slot.getSlotElementId() : "" + const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] + const sizes = [] + for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break + let pair = null + if ( + size && + typeof size.getWidth === "function" && + typeof size.getHeight === "function" + ) { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + const width = + size && typeof size.getWidth === "function" + ? size.getWidth() + : Array.isArray(size) + ? size[0] + : undefined + const height = + size && typeof size.getHeight === "function" + ? size.getHeight() + : Array.isArray(size) + ? size[1] + : undefined + __ts_warn_ignored_size(width, height) + } + } + const exists = __ts_ev.gpt_slots.some( + (entry) => entry.gam_unit_path === __ts_text(path) && entry.div_id === __ts_text(divId) + ) + if (!exists) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(path), + div_id: __ts_text(divId), + sizes, + phase: __ts_phase() + }) + } + } catch (error) { + __ts_warn("gpt_scrape_failed", error) + } + } + } + } catch (error) { + __ts_warn("collect_failed", error) + } + return __ts_ev +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs new file mode 100644 index 000000000..0e2b51371 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -0,0 +1,1014 @@ +//! Browser-backed `ts audit ad-templates verify` orchestration. +//! +//! For each URL: collect live evidence through an [`AuditCollector`], match +//! configured slots against the **final** (post-redirect) path, evaluate the +//! runtime gate, compare evidence, and assemble the stable §8 wire result. The +//! orchestration is collector-agnostic so it is fully tested with an in-memory +//! fake collector, with no Chrome dependency. + +use std::io::{self, Write}; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, CreativeOpportunitiesConfig, evaluate_ad_stack_gate, +}; + +use crate::ad_templates::compare::{ + BrowserAdEvidence, EvidencePhase, ExtraEvidence, RuntimeGateSummary, SlotEvidence, SlotResult, + SlotStatus as CompareStatus, compare_page_evidence, +}; +use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, normalize_path_or_url}; +use crate::ad_templates::output::{ + ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, + GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, + VerificationReport, Warning, escape_terminal_text, +}; +use crate::commands::audit::AuditAdTemplatesVerifyArgs; +use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, AuditCollector, BrowserCollectRequest, build_ad_template_init_script, +}; +use crate::run::RunOutcome; + +/// Verifies configured ad-template slots against live page evidence. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading fails, or when verification +/// surfaces a page-level error or a `--strict` failure (after writing output). +pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result { + args.browser.validate()?; + validate_cookie_scope(&args.urls, &args.cookies)?; + let loaded = crate::app_config::load_settings(&args.config)?; + let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); + let report = build_report( + &collector, + loaded.settings.creative_opportunities.as_ref(), + loaded.settings.auction.enabled, + &args.urls, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, + &args.cookies, + )?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + if args.json { + write_json(&mut out, &report)?; + } else { + write_human(&mut out, &report)?; + } + + if report.pages.iter().any(|page| page.error.is_some()) { + Err("ad-template verification reported problems".to_string()) + } else if report.ok { + Ok(RunOutcome::Success) + } else { + Ok(RunOutcome::AssertionFailed) + } +} + +fn validate_cookie_scope(urls: &[url::Url], cookies: &[(String, String)]) -> Result<(), String> { + if cookies.is_empty() { + return Ok(()); + } + let origins: std::collections::BTreeSet = urls + .iter() + .map(|url| url.origin().ascii_serialization()) + .collect(); + if origins.len() > 1 { + return Err( + "--cookie may be used only when every verification URL has one origin; split this run so credentials are never copied to another origin" + .to_string(), + ); + } + Ok(()) +} + +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + +/// Builds the verification report for `urls` using `collector`. +/// +/// `creative` is the effective `[creative_opportunities]` config (if any) and +/// `auction_enabled` is the `[auction].enabled` kill switch. +fn build_report( + collector: &dyn AuditCollector, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, + urls: &[url::Url], + options: VerifyOptions, + cookies: &[(String, String)], +) -> Result { + let init_script = build_init_script(creative)?; + + let requests: Vec<_> = urls + .iter() + .map(|url| BrowserCollectRequest { + url: url.clone(), + init_scripts: vec![init_script.clone()], + scroll: options.scroll, + collect_ad_evidence: true, + cookies: cookies.to_vec(), + }) + .collect(); + let collected_pages = collector.collect_pages(&requests); + + let mut pages = Vec::with_capacity(urls.len()); + let mut any_error = false; + let mut any_strict_fail = false; + + for (url, collected) in urls.iter().zip(collected_pages) { + match collected { + Err(message) => { + any_error = true; + pages.push(error_page(url, &message)); + } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } + Ok(collected) => { + let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); + if options.strict && strict_failed { + any_strict_fail = true; + } + pages.push(page); + } + } + } + + let ok = !(any_error || (options.strict && any_strict_fail)); + Ok(VerificationReport { + ok, + strict: options.strict, + pages, + warnings: Vec::new(), + }) +} + +/// The URL without its fragment, for comparisons the server can observe. +pub(super) fn without_fragment(url: &url::Url) -> url::Url { + let mut url = url.clone(); + url.set_fragment(None); + url +} + +/// Whether navigation left the requested URL's origin (scheme, host, or port). +/// +/// A same-host default-port `http:80` to `https:443` redirect is *not* a change: +/// the host is the cookie boundary, and that upgrade is the ordinary canonical +/// redirect. Host changes, port changes, and HTTPS downgrades all are. +pub(super) fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + if requested.host_str() != final_url.host_str() { + return true; + } + + match (requested.scheme(), final_url.scheme()) { + ("http", "https") => { + requested.port_or_known_default() != Some(80) + || final_url.port_or_known_default() != Some(443) + } + (requested_scheme @ ("http" | "https"), final_scheme) + if requested_scheme == final_scheme => + { + requested.port_or_known_default() != final_url.port_or_known_default() + } + // Refuse HTTPS downgrades and any unexpected scheme transition. + _ => true, + } +} + +/// Builds the read-only collector init script from the configured slots. +fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Result { + let config = AdTemplateCollectorConfig { + div_prefixes: creative + .map(|creative| { + creative + .slot + .iter() + .map(|slot| slot.resolved_div_id().to_string()) + .collect() + }) + .unwrap_or_default(), + }; + build_ad_template_init_script(&config) +} + +/// Assembles a successful page result, returning the wire `PageJson` and whether +/// the page would fail `--strict`. +fn build_page( + requested: &url::Url, + collected: &crate::commands::audit::collector::CollectedPage, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, +) -> (PageJson, bool) { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + let final_url = &collected.final_url; + let final_path = normalize_path_or_url(final_url.as_str()).unwrap_or_else(|_| "/".into()); + + let expected = creative + .map(|creative| expected_slots_for_path(&final_path, creative).slots) + .unwrap_or_default(); + let matched = !expected.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: matched, + consent_allows_auction: None, + auction_enabled, + // Absent creative opportunities block here as they do at runtime. + ad_templates_enabled: creative.is_some_and(|creative| creative.enabled), + }); + + let evidence = collected.ad_evidence.clone().unwrap_or_else(empty_evidence); + let result = compare_page_evidence( + &expected, + &evidence, + RuntimeGateSummary::from_expected(gate.expected), + ); + let strict_failed = result.strict_failed(); + + let mut warnings: Vec = collected.warnings.to_vec(); + warnings.extend(evidence.warnings.iter().map(|warning| Warning { + code: format!("page_{}", warning.code), + message: warning.message.clone(), + })); + // Fragments never reach the server, so a fragment-only difference is not a + // redirect and slots match on the path either way. + if without_fragment(requested) != without_fragment(final_url) { + warnings.push(Warning { + code: "redirected".to_string(), + message: format!("navigation redirected from {requested} to {final_url}"), + }); + } + + let slots = expected + .iter() + .zip(result.slots.iter()) + .map(|(expected_slot, slot_result)| to_slot_json(expected_slot, slot_result)) + .collect(); + let extra_evidence = result.extra_evidence.iter().map(to_extra_json).collect(); + + let page = PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: Some(final_path), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::from( + result.runtime_ad_stack_expected, + )), + gates: Some(to_gates( + matched, + auction_enabled, + creative.is_some_and(|creative| creative.enabled), + )), + matched_slot_count: Some(expected.len()), + slots, + extra_evidence, + warnings, + }; + (page, strict_failed) +} + +/// Builds a page-level navigation-failure result (spec §8 `navigation_failed`). +fn error_page(requested: &url::Url, message: &str) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: None, + requested_path, + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: message.to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +fn empty_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: Vec::new(), + gpt_slots: Vec::new(), + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } +} + +fn to_gates(matched: bool, auction_enabled: bool, ad_templates_enabled: bool) -> Gates { + let pass_if = |cond: bool| { + if cond { + GateState::Pass + } else { + GateState::Fail + } + }; + Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: pass_if(matched), + auction_enabled: pass_if(auction_enabled), + ad_templates_enabled: pass_if(ad_templates_enabled), + // Live consent is not provable from a browser navigation in Phase 1. + consent_allows_auction: GateState::Unknown, + } +} + +fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { + SlotJson { + id: result.id.clone(), + status: to_status(result.status), + phase: result.phase.map(to_phase), + configured: ConfiguredJson { + div_id: expected.div_id.clone(), + gam_unit_path: expected.gam_unit_path.clone(), + formats: expected + .formats + .iter() + .map(|format| FormatJson { + width: format.width, + height: format.height, + media_type: media_type_label(&format.media_type).to_string(), + }) + .collect(), + providers: expected.providers.clone(), + }, + evidence: to_slot_evidence(&result.evidence), + warnings: result.warnings.clone(), + } +} + +fn to_slot_evidence(evidence: &SlotEvidence) -> SlotEvidenceJson { + SlotEvidenceJson { + dom_id: evidence.dom_id.clone(), + gpt: evidence.gpt.as_ref().map(|gpt| GptEvidenceJson { + gam_unit_path: gpt.gam_unit_path.clone(), + div_id: gpt.div_id.clone(), + sizes: gpt.sizes.iter().map(|&(w, h)| [w, h]).collect(), + }), + } +} + +fn to_extra_json(extra: &ExtraEvidence) -> ExtraEvidenceJson { + ExtraEvidenceJson { + kind: extra.kind.clone(), + phase: to_phase(extra.phase), + dom_id: extra.dom_id.clone(), + gam_unit_path: extra.gam_unit_path.clone(), + sizes: extra.sizes.iter().map(|&(w, h)| [w, h]).collect(), + reason: extra.reason.clone(), + } +} + +fn to_status(status: CompareStatus) -> SlotStatus { + match status { + CompareStatus::Confirmed => SlotStatus::Confirmed, + CompareStatus::Partial => SlotStatus::Partial, + CompareStatus::Missing => SlotStatus::Missing, + CompareStatus::Unconfirmable => SlotStatus::Unconfirmable, + } +} + +fn to_phase(phase: EvidencePhase) -> EvidencePhaseJson { + match phase { + EvidencePhase::InitialLoad => EvidencePhaseJson::InitialLoad, + EvidencePhase::Scroll => EvidencePhaseJson::Scroll, + } +} + +fn media_type_label(media_type: &MediaType) -> &'static str { + match media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + } +} + +fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report) + .map_err(|error| format!("failed to serialize verification report: {error}"))?; + writeln!(out, "{json}").map_err(write_err) +} + +fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + + for warning in &report.warnings { + write_warning(out, "", warning)?; + } + for page in &report.pages { + writeln!(out, "url: {}", escape_terminal_text(&page.url)).map_err(write_err)?; + if let Some(error) = &page.error { + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; + continue; + } + if let Some(path) = &page.path { + writeln!(out, " path: {}", escape_terminal_text(path)).map_err(write_err)?; + } + if let Some(expected) = page.runtime_ad_stack_expected { + writeln!(out, " runtime ad stack: {}", runtime_label(expected)).map_err(write_err)?; + } + if let Some(count) = page.matched_slot_count { + writeln!(out, " matched slots: {count}").map_err(write_err)?; + } + if let Some(gates) = &page.gates { + writeln!(out, " gates: {}", gates_label(gates)).map_err(write_err)?; + } + for slot in &page.slots { + writeln!( + out, + " slot {}: {}", + escape_terminal_text(&slot.id), + status_label(slot.status) + ) + .map_err(write_err)?; + for warning in &slot.warnings { + write_warning(out, " ", warning)?; + } + } + for extra in &page.extra_evidence { + writeln!( + out, + " extra {} evidence: div={} gam={} sizes={:?} ({})", + escape_terminal_text(&extra.kind), + escape_terminal_text(extra.dom_id.as_deref().unwrap_or("-")), + escape_terminal_text(extra.gam_unit_path.as_deref().unwrap_or("-")), + extra.sizes, + escape_terminal_text(&extra.reason), + ) + .map_err(write_err)?; + } + for warning in &page.warnings { + write_warning(out, " ", warning)?; + } + } + writeln!(out, "ok: {}", report.ok).map_err(write_err) +} + +fn status_label(status: SlotStatus) -> &'static str { + match status { + SlotStatus::Confirmed => "confirmed", + SlotStatus::Partial => "partial", + SlotStatus::Missing => "missing", + SlotStatus::Unconfirmable => "unconfirmable", + } +} + +fn runtime_label(expected: RuntimeAdStackExpectedJson) -> &'static str { + match expected { + RuntimeAdStackExpectedJson::Yes => "yes", + RuntimeAdStackExpectedJson::No => "no", + RuntimeAdStackExpectedJson::Unknown => "unknown", + } +} + +fn gate_label(gate: GateState) -> &'static str { + match gate { + GateState::Pass => "pass", + GateState::Fail => "fail", + GateState::Unknown => "unknown", + } +} + +fn gates_label(gates: &Gates) -> String { + format!( + "method_get={} navigation={} not_prefetch={} not_bot={} matched_slots={} auction_enabled={} consent={}", + gate_label(gates.method_get), + gate_label(gates.navigation), + gate_label(gates.not_prefetch), + gate_label(gates.not_bot), + gate_label(gates.matched_slots), + gate_label(gates.auction_enabled), + gate_label(gates.consent_allows_auction), + ) +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn write_err(error: io::Error) -> String { + format!("failed to write command output: {error}") +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::collections::HashMap; + + use super::*; + use crate::ad_templates::compare::{DomEvidence, GptSlotEvidence}; + use crate::commands::audit::collector::CollectedPage; + + struct FakeCollector { + pages: HashMap>, + batch_calls: Cell, + } + + impl FakeCollector { + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse final URL"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { + pages, + batch_calls: Cell::new(0), + } + } + + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages + .insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + self.batch_calls.set(self.batch_calls.get() + 1); + requests + .iter() + .cloned() + .map(|request| self.collect_page(request)) + .collect() + } + } + + fn news_config() -> CreativeOpportunitiesConfig { + let toml = "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + config + } + + fn confirmed_news_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { + dom_id: "ad-atf-0".to_string(), + phase: EvidencePhase::InitialLoad, + }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn report_for( + collector: &dyn AuditCollector, + auction_enabled: bool, + strict: bool, + urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, + ) -> VerificationReport { + let config = news_config(); + let parsed: Vec = urls + .iter() + .map(|url| url::Url::parse(url).expect("should parse URL")) + .collect(); + build_report( + collector, + Some(&config), + auction_enabled, + &parsed, + options, + &[], + ) + .expect("typed collector configuration should serialize") + } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, false, &["https://www.example.com/"]); + let json = serde_json::to_value(&report).expect("should serialize"); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + assert_eq!(json["pages"][0]["slots"][0]["status"], "confirmed"); + let warnings = json["pages"][0]["warnings"] + .as_array() + .expect("should have warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("should have slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + + #[test] + fn same_host_http_to_https_upgrade_is_accepted() { + let collector = FakeCollector::page( + "http://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["http://www.example.com/news/story"], + ); + + assert!(report.ok, "a default-port HTTPS upgrade should be accepted"); + } + + #[test] + fn downgrade_and_port_changes_are_rejected() { + for (requested, final_url) in [ + ( + "https://www.example.com/news/story", + "http://www.example.com/news/story", + ), + ( + "https://www.example.com:8443/news/story", + "https://www.example.com:9443/news/story", + ), + ( + "http://www.example.com:8080/news/story", + "https://www.example.com:8443/news/story", + ), + ] { + let collector = FakeCollector::page(requested, final_url, confirmed_news_evidence()); + let report = report_for(&collector, true, true, &[requested]); + assert!(!report.ok, "redirect {requested} -> {final_url} must fail"); + } + } + + #[test] + fn confirmed_page_is_ok_in_default_mode() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!(report.ok, "confirmed page should be ok"); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn verifier_surfaces_injected_collector_warnings() { + let mut evidence = confirmed_news_evidence(); + evidence.warnings.push(Warning { + code: "fluid_size_ignored".to_string(), + message: "a fluid size could not be compared".to_string(), + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!( + report.pages[0] + .warnings + .iter() + .any(|warning| warning.code == "page_fluid_size_ignored"), + "collector warning should be visible in the page report" + ); + } + + #[test] + fn human_output_includes_runtime_and_extra_evidence_diagnostics() { + let mut evidence = confirmed_news_evidence(); + evidence.gpt_slots.push(GptSlotEvidence { + gam_unit_path: "/123/publisher/extra".to_string(), + div_id: "ad-extra-0".to_string(), + sizes: vec![(728, 90)], + phase: EvidencePhase::InitialLoad, + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + let mut output = Vec::new(); + + write_human(&mut output, &report).expect("should write human report"); + let output = String::from_utf8(output).expect("should be UTF-8 output"); + + assert!(output.contains("runtime ad stack: unknown")); + assert!(output.contains("matched slots: 1")); + assert!(output.contains("gates: method_get=pass")); + assert!(output.contains("extra gpt evidence")); + } + + #[test] + fn strict_missing_slot_fails() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + !report.ok, + "strict mode with a missing slot should not be ok" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + // auction disabled -> runtime expected No -> strict does not fail on missing. + let report = report_for( + &collector, + false, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + report.ok, + "missing slot must not fail strict when auction is disabled" + ); + assert_eq!( + report.pages[0].runtime_ad_stack_expected, + Some(RuntimeAdStackExpectedJson::No) + ); + } + + #[test] + fn multi_url_page_error_sets_ok_false() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ) + .with_error("https://www.example.com/broken", "navigation failed"); + let report = report_for( + &collector, + true, + false, + &[ + "https://www.example.com/news/story", + "https://www.example.com/broken", + ], + ); + + assert!(!report.ok, "a page-level error sets ok=false"); + assert_eq!( + collector.batch_calls.get(), + 1, + "all verifier URLs should use one collector batch" + ); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][1]["error"]["code"], "navigation_failed"); + assert!(json["pages"][1]["final_url"].is_null()); + } + + #[test] + fn supplied_cookies_are_rejected_for_multiple_origins() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://b.example/y").expect("should parse second URL"), + ]; + + let error = validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect_err("should not replicate one cookie across origins"); + + assert!( + error.contains("one origin"), + "the refusal should explain cookie scope, got {error}" + ); + } + + #[test] + fn supplied_cookies_are_allowed_for_same_origin_urls() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://a.example/y").expect("should parse second URL"), + ]; + + validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect("same-origin URLs share the intended cookie scope"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs new file mode 100644 index 000000000..fd79155f0 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -0,0 +1,1194 @@ +//! Chrome/Chromium-backed implementation of [`AuditCollector`] using +//! `chromiumoxide` (CDP). +//! +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. + +use std::time::Duration; + +use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::handler::viewport::Viewport; +use chromiumoxide::page::Page; +use futures::StreamExt as _; + +use crate::ad_templates::compare::BrowserAdEvidence; +use crate::ad_templates::output::Warning; +use crate::commands::audit::browser_scroll::{self, CDP_OPERATION_TIMEOUT}; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, BrowserProfile, CollectedPage, + PAGE_SETTLE_MAX_MS, PAGE_SETTLE_QUIET_MS, +}; + +/// Candidate Chrome/Chromium executable names searched on `PATH`. +pub(crate) const CHROME_NAMES: &[&str] = &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", + "Google Chrome", + "Google Chrome for Testing", +]; + +/// Poll interval while waiting for the page network to settle, in milliseconds. +const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Hard cap per decoded evidence list, so a hostile page cannot inflate CLI +/// memory. +/// +/// Must equal `__ts_max_entries` in `ad_template_collector.js`. The collector +/// already caps each list, but the evidence object lives on `window`, so a page +/// that appends to it directly is bounded here instead. Anything the collector +/// itself dropped is reported as an `evidence_truncated` warning. +const MAX_EVIDENCE_ENTRIES: usize = 128; +/// Hard cap on the UTF-8 JSON payload before CDP transfers it back to Rust. +const MAX_EVIDENCE_PAYLOAD_BYTES: usize = 1024 * 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Page-settle timing thresholds. +#[derive(Debug, Clone, Copy)] +struct SettleConfig { + /// Quiet window with no new resources marking the page settled. + quiet: Duration, + /// Hard cap on total settle time. + max: Duration, +} + +/// Immutable browser/session settings shared by every URL in one audit batch. +struct BrowserSessionOptions<'a> { + chrome: &'a std::path::Path, + profile_dir: &'a std::path::Path, + settle: SettleConfig, + accept_invalid_certs: bool, + headful: bool, + assume_consent: bool, + proxy: Option<&'a str>, + profile: BrowserProfile, +} + +/// A `chromiumoxide`-backed page collector launching a local Chrome/Chromium. +#[derive(Debug, Clone)] +pub struct BrowserCollector { + /// Explicit Chrome/Chromium executable override (else `$CHROME`, else auto-detect). + chrome: Option, + /// Quiet window marking the page settled. + settle_quiet: Duration, + /// Hard cap on settling. + settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, + /// Run visible Chrome rather than new headless Chrome. + headful: bool, + /// Install the standard consent API stub before publisher scripts. + assume_consent: bool, + /// Optional browser proxy endpoint. + proxy: Option, + /// Device viewport/user-agent profile. + profile: BrowserProfile, +} + +impl Default for BrowserCollector { + fn default() -> Self { + Self::new() + } +} + +impl BrowserCollector { + /// Creates a collector with default tuning and auto-detected Chrome. + #[must_use] + pub fn new() -> Self { + Self { + chrome: None, + settle_quiet: Duration::from_millis(PAGE_SETTLE_QUIET_MS), + settle_max: Duration::from_millis(PAGE_SETTLE_MAX_MS), + accept_invalid_certs: false, + headful: false, + assume_consent: true, + proxy: None, + profile: BrowserProfile::Desktop, + } + } + + /// Creates a collector from operator-supplied browser options. + #[must_use] + pub fn from_opts(opts: &BrowserOpts) -> Self { + Self { + chrome: opts.chrome.clone(), + settle_quiet: Duration::from_millis(opts.settle_quiet_ms), + settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, + headful: opts.headful, + assume_consent: !opts.no_assume_consent, + proxy: opts.browser_proxy.clone(), + profile: opts.profile, + } + } +} + +/// Pre-document consent behavior shared with the generation crawler. +pub(crate) const CONSENT_STUB_SCRIPT: &str = include_str!("consent_stub.js"); + +/// Shared browser launch inputs used by both audit collectors. +pub(crate) struct BrowserLaunchOptions<'a> { + pub(crate) chrome: &'a std::path::Path, + pub(crate) profile_dir: &'a std::path::Path, + pub(crate) headful: bool, + pub(crate) proxy: Option<&'a str>, + pub(crate) accept_invalid_certs: bool, + pub(crate) viewport: Viewport, + pub(crate) user_agent: Option<&'a str>, +} + +/// Builds the common Chrome configuration for all browser-backed audits. +pub(crate) fn build_browser_config( + options: BrowserLaunchOptions<'_>, +) -> Result { + let mut builder = BrowserConfig::builder() + .chrome_executable(options.chrome) + .user_data_dir(options.profile_dir); + if !options.accept_invalid_certs { + builder = builder.respect_https_errors(); + } + if let Some(proxy) = options.proxy { + let endpoint = if proxy.contains("://") { + proxy.to_string() + } else { + format!("http://{proxy}") + }; + builder = builder + .arg(("proxy-server", endpoint.as_str())) + .arg(("proxy-bypass-list", "<-loopback>")); + } + builder = if options.headful { + builder.with_head() + } else { + builder.new_headless_mode() + }; + builder = builder + .window_size(options.viewport.width, options.viewport.height) + .viewport(options.viewport); + if let Some(user_agent) = options.user_agent { + builder = builder.arg(("user-agent", user_agent)); + } + builder + .build() + .map_err(|error| format!("failed to build browser config: {error}")) +} + +fn browser_profile(profile: BrowserProfile) -> (Viewport, Option<&'static str>) { + match profile { + BrowserProfile::Desktop => ( + Viewport { + width: 1280, + height: 800, + device_scale_factor: Some(1.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + }, + None, + ), + BrowserProfile::Mobile => ( + Viewport { + width: 390, + height: 844, + device_scale_factor: Some(3.0), + emulating_mobile: true, + is_landscape: false, + has_touch: true, + }, + Some( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ), + ), + } +} + +/// Resolves the Chrome/Chromium executable to launch. +/// +/// Precedence: explicit `--chrome` override, then the `CHROME` environment +/// variable, then auto-detection on `PATH` and standard install locations. +pub(crate) fn resolve_chrome( + override_path: Option<&std::path::Path>, +) -> Result { + if let Some(path) = override_path { + return if path.is_file() { + Ok(path.to_path_buf()) + } else { + Err(format!( + "--chrome path does not point to a file: {}", + path.display() + )) + }; + } + if let Ok(env_path) = std::env::var("CHROME") { + let path = std::path::PathBuf::from(&env_path); + return if path.is_file() { + Ok(path) + } else { + Err(format!("CHROME={env_path} does not point to a file")) + }; + } + find_chrome() +} + +/// Builds a host-only cookie that applies to every path on `url`'s host. +/// +/// Scoped by origin rather than by the full URL: only the origin is load-bearing +/// for a host-only cookie, and a full URL would carry the path, query, and any +/// `user:password@` into CDP and into this function's error message. +pub(crate) fn host_cookie(name: &str, value: &str, url: &url::Url) -> Result { + let origin = url.origin(); + if !origin.is_tuple() { + return Err(format!( + "cannot scope cookie `{name}` because the audited URL has no host" + )); + } + let mut cookie = CookieParam::new(name.to_string(), value.to_string()); + cookie.url = Some(origin.ascii_serialization()); + cookie.path = Some("/".to_string()); + cookie.secure = Some(url.scheme() == "https"); + Ok(cookie) +} + +fn format_cookie_install_error(name: &str, _error: impl std::fmt::Display) -> String { + // Do not forward the CDP error: a browser implementation may include the + // rejected cookie value in its diagnostic. + format!("failed to set cookie `{name}`") +} + +/// Installs host-only, root-scoped cookies before a page has an origin. +pub(crate) async fn set_browser_cookies( + browser: &Browser, + cookies: &[(String, String)], + url: &url::Url, +) -> Result<(), String> { + for (name, value) in cookies { + let cookie = host_cookie(name, value, url)?; + browser + .set_cookies(vec![cookie]) + .await + .map_err(|error| format_cookie_install_error(name, error))?; + } + Ok(()) +} + +/// Auto-detects a Chrome/Chromium executable. +/// +/// Searches `PATH` by common names first, then well-known per-OS install +/// locations (e.g. the macOS `.app` bundle, which is not on `PATH`). +fn find_chrome() -> Result { + if let Some(path) = CHROME_NAMES.iter().find_map(|name| which::which(name).ok()) { + return Ok(path); + } + if let Some(path) = well_known_chrome_paths() + .into_iter() + .find(|path| path.is_file()) + { + return Ok(path); + } + Err(format!( + "could not find Chrome/Chromium on PATH or in standard install locations (looked for: {})", + CHROME_NAMES.join(", ") + )) +} + +/// Well-known absolute Chrome/Chromium install locations for the host OS. +fn well_known_chrome_paths() -> Vec { + let mut paths = Vec::new(); + + #[cfg(target_os = "macos")] + { + const APPS: &[&str] = &[ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "Chromium.app/Contents/MacOS/Chromium", + ]; + for app in APPS { + paths.push(std::path::PathBuf::from(format!("/Applications/{app}"))); + if let Ok(home) = std::env::var("HOME") { + paths.push(std::path::PathBuf::from(format!( + "{home}/Applications/{app}" + ))); + } + } + } + + #[cfg(target_os = "linux")] + { + for path in [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + #[cfg(target_os = "windows")] + { + for path in [ + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + paths +} + +impl AuditCollector for BrowserCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.collect_pages(std::slice::from_ref(&request)) + .into_iter() + .next() + .expect("should return one result for one browser request") + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + if requests.is_empty() { + return Vec::new(); + } + // HTTP(S) scheme is enforced by the CLI value parser before we get here. + let chrome = match resolve_chrome(self.chrome.as_deref()) { + Ok(chrome) => chrome, + Err(error) => return vec![Err(error); requests.len()], + }; + let profile = match tempfile::tempdir() { + Ok(profile) => profile, + Err(error) => { + let error = format!("failed to create browser profile dir: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let error = format!("failed to build browser runtime: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let settle = SettleConfig { + quiet: self.settle_quiet, + max: self.settle_max, + }; + + let accept_invalid_certs = self.accept_invalid_certs; + let headful = self.headful; + let assume_consent = self.assume_consent; + let proxy = self.proxy.clone(); + let browser_profile = self.profile; + let request_count = requests.len(); + let requests = requests.to_vec(); + let result = runtime.block_on(async move { + let options = BrowserSessionOptions { + chrome: &chrome, + profile_dir: profile.path(), + settle, + accept_invalid_certs, + headful, + assume_consent, + proxy: proxy.as_deref(), + profile: browser_profile, + }; + collect(requests, &options).await + }); + match result { + Ok(results) => results, + Err(error) => vec![Err(error); request_count], + } + } +} + +/// Drives a single page collection on the current-thread runtime. +async fn collect( + requests: Vec, + options: &BrowserSessionOptions<'_>, +) -> Result>, String> { + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let (viewport, user_agent) = browser_profile(options.profile); + let config = build_browser_config(BrowserLaunchOptions { + chrome: options.chrome, + profile_dir: options.profile_dir, + headful: options.headful, + proxy: options.proxy, + accept_invalid_certs: options.accept_invalid_certs, + viewport, + user_agent, + })?; + + let (mut browser, mut handler) = Browser::launch(config) + .await + .map_err(|error| format!("failed to launch browser: {error}"))?; + + // Drive the CDP event loop for the duration of the session. + let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); + + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + results.push( + collect_with_browser(&browser, request, options.settle, options.assume_consent).await, + ); + } + + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; + handler_task.abort(); + + Ok(results) +} + +async fn collect_with_browser( + browser: &Browser, + request: BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + set_browser_cookies(browser, &request.cookies, &request.url).await?; + + // Open a blank page first so init scripts are installed before the real + // document loads (evaluate-on-new-document applies to subsequent navigations). + let page = browser + .new_page("about:blank") + .await + .map_err(|error| format!("failed to open browser page: {error}"))?; + + let result = collect_open_page(&page, &request, settle_config, assume_consent).await; + let close_result = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, page.close()).await; + + match (result, close_result) { + (Err(error), _) => Err(error), + (Ok(mut collected), Err(_)) => { + collected.warnings.push(Warning { + code: "page_close_timeout".to_string(), + message: "timed out closing the browser tab after collection".to_string(), + }); + Ok(collected) + } + (Ok(mut collected), Ok(Err(error))) => { + collected.warnings.push(Warning { + code: "page_close_failed".to_string(), + message: format!("failed to close the browser tab after collection: {error}"), + }); + Ok(collected) + } + (Ok(collected), Ok(Ok(_))) => Ok(collected), + } +} + +/// Collects from an open tab. The caller owns tab teardown so every return path, +/// including an error from this function, closes the page before continuing. +async fn collect_open_page( + page: &Page, + request: &BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + let mut warnings = Vec::new(); + + if assume_consent { + page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) + .await + .map_err(|error| format!("failed to install consent init script: {error}"))?; + warnings.push(Warning { + code: "consent_stub_active".to_string(), + message: "audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution".to_string(), + }); + } + page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") + .await + .map_err(|error| format!("failed to increase resource timing buffer: {error}"))?; + + for script in &request.init_scripts { + page.evaluate_on_new_document(script.clone()) + .await + .map_err(|error| format!("failed to install init script: {error}"))?; + } + + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) + .await + .map_err(|_| format!("navigation to {} timed out", request.url))? + .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; + match tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(Warning { + code: "navigation_wait_failed".to_string(), + message: format!( + "navigation load event could not be read ({error}); continuing with settled page evidence" + ), + }), + Err(_) => warnings.push(Warning { + code: "navigation_wait_timeout".to_string(), + message: format!( + "navigation did not fire its load event within {} seconds; continuing with settled page evidence", + NAVIGATION_TIMEOUT.as_secs() + ), + }), + } + + settle(page, settle_config, &mut warnings).await; + + if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + if tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ), + ) + .await + .is_err() + { + warnings.push(Warning { + code: "ad_evidence_snapshot_timeout".to_string(), + message: "timed out snapshotting ad evidence before scroll".to_string(), + }); + } + } + // Mark subsequent observations as scroll-phase for the verifier's + // injected evidence collector before shared scrolling begins. + eval_discard(page, "window.__tsScrollPhase = true", &mut warnings).await; + warnings.extend( + browser_scroll::scroll_page(page) + .await + .into_iter() + .map(|failure| Warning { + code: failure.code().to_string(), + message: failure.to_string(), + }), + ); + settle(page, settle_config, &mut warnings).await; + } + + let final_url_text = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.url()) + .await + .map_err(|_| "timed out reading final page URL".to_string())? + .map_err(|error| format!("failed to read final page URL: {error}"))? + .ok_or_else(|| "browser page URL was empty after navigation".to_string())?; + let final_url = url::Url::parse(&final_url_text).map_err(|error| { + format!("browser returned invalid final URL `{final_url_text}`: {error}") + })?; + let title = match tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.get_title()).await { + Ok(Ok(title)) => title.unwrap_or_default(), + Ok(Err(error)) => { + warnings.push(Warning { + code: "page_title_failed".to_string(), + message: format!("failed to read page title: {error}"), + }); + String::new() + } + Err(_) => { + warnings.push(Warning { + code: "page_title_timeout".to_string(), + message: "timed out reading page title".to_string(), + }); + String::new() + } + }; + let script_count = eval_usize(page, "document.querySelectorAll('script').length") + .await + .unwrap_or_else(|message| { + warnings.push(Warning { + code: "script_count_failed".to_string(), + message, + }); + 0 + }); + let resource_count = resource_count(page).await.unwrap_or_else(|message| { + warnings.push(Warning { + code: "resource_count_failed".to_string(), + message, + }); + 0 + }); + + if resource_count >= 250 { + warnings.push(Warning { + code: "resource_timing_heavy".to_string(), + message: format!("page recorded {resource_count} network resources"), + }); + } + + if let Ok(Ok(frames)) = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.frames()).await + && frames.len() > 1 + { + warnings.push(Warning { + code: "child_frames_not_inspected".to_string(), + message: format!( + "ad-template evidence inspected only the main frame; {} child frame(s) were present", + frames.len() - 1 + ), + }); + } + + let ad_evidence = if request.collect_ad_evidence { + extract_ad_evidence(page, &mut warnings).await + } else { + None + }; + + Ok(CollectedPage { + final_url, + title, + script_count, + resource_count, + warnings, + ad_evidence, + }) +} + +/// Waits for the page network to go quiet after navigation or scroll. +/// +/// Polls the resource-entry count and returns once it stays unchanged for a +/// quiet window, or when the hard cap elapses — so ad-heavy pages finish loading +/// before evidence is read, without hanging on pages that never go idle. +async fn settle(page: &Page, config: SettleConfig, warnings: &mut Vec) { + let start = std::time::Instant::now(); + let mut last = None; + let mut quiet_since = None; + + loop { + if start.elapsed() >= config.max { + warnings.push(Warning { + code: "settle_timeout".to_string(), + message: "page did not settle before the configured maximum wait".to_string(), + }); + return; + } + + let ready_state = match eval_string(page, "document.readyState").await { + Ok(state) => state, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let current = match resource_count(page).await { + Ok(count) => count, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let ready = matches!(ready_state.as_str(), "interactive" | "complete"); + if ready && last == Some(current) { + let quiet_start = quiet_since.get_or_insert_with(std::time::Instant::now); + if quiet_start.elapsed() >= config.quiet { + return; + } + } else { + quiet_since = None; + } + last = Some(current); + + let remaining_max = config.max.saturating_sub(start.elapsed()); + let remaining_quiet = quiet_since + .map(|quiet_start| config.quiet.saturating_sub(quiet_start.elapsed())) + .unwrap_or(config.quiet); + let sleep_for = Duration::from_millis(SETTLE_POLL_MS) + .min(remaining_max) + .min(remaining_quiet.max(Duration::from_millis(1))); + tokio::time::sleep(sleep_for).await; + } +} + +/// Reads the number of resource timing entries observed so far. +async fn resource_count(page: &Page) -> Result { + eval_usize(page, "performance.getEntriesByType('resource').length").await +} + +async fn eval_discard(page: &Page, expression: impl Into, warnings: &mut Vec) { + if let Err(failure) = browser_scroll::evaluate(page, expression).await { + warnings.push(Warning { + code: failure.code().to_string(), + message: failure.to_string(), + }); + } +} + +async fn eval_usize(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +async fn eval_string(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +/// Reads and decodes `window.__tsAdTemplateEvidence`, warning (not failing) on a +/// decode error. +async fn extract_ad_evidence( + page: &Page, + warnings: &mut Vec, +) -> Option { + // Serialize and size-check in the page so a hostile publisher-controlled + // evidence object cannot force an unbounded CDP response and Rust decode. + let evaluation = tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate(format!( + r#"(() => {{ + const evidence = typeof window.__tsCollectAdTemplateEvidence === 'function' + ? window.__tsCollectAdTemplateEvidence() + : (window.__tsAdTemplateEvidence || null) + if (evidence === null) return {{ kind: 'absent' }} + try {{ + const json = JSON.stringify(evidence) + const bytes = new TextEncoder().encode(json).byteLength + if (bytes > {MAX_EVIDENCE_PAYLOAD_BYTES}) return {{ kind: 'too_large' }} + return {{ kind: 'evidence', json }} + }} catch (error) {{ + return {{ + kind: 'serialization_failed', + message: String(error).slice(0, 512), + }} + }} + }})()"# + )), + ) + .await; + + let envelope = match evaluation { + Ok(Ok(result)) => match result.into_value::() { + Ok(envelope) => Some(envelope), + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence envelope: {error}"), + }); + return None; + } + }, + Ok(Err(error)) => { + warnings.push(Warning { + code: "ad_evidence_read_failed".to_string(), + message: format!("failed to read ad-template evidence: {error}"), + }); + return None; + } + Err(_) => { + warnings.push(Warning { + code: "ad_evidence_read_timeout".to_string(), + message: "timed out reading ad-template evidence".to_string(), + }); + return None; + } + }; + + match envelope { + Some(envelope) => decode_ad_evidence_envelope(envelope, warnings), + None => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + } +} + +#[derive(Debug, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum EvidenceEnvelope { + Absent, + TooLarge, + Evidence { json: String }, + SerializationFailed { message: String }, +} + +fn decode_ad_evidence_envelope( + envelope: EvidenceEnvelope, + warnings: &mut Vec, +) -> Option { + match envelope { + EvidenceEnvelope::Absent => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + EvidenceEnvelope::TooLarge => { + warnings.push(Warning { + code: "ad_evidence_too_large".to_string(), + message: format!( + "ad-template evidence exceeded the {MAX_EVIDENCE_PAYLOAD_BYTES}-byte limit" + ), + }); + None + } + EvidenceEnvelope::SerializationFailed { message } => { + warnings.push(Warning { + code: "ad_evidence_encode_failed".to_string(), + message: format!("failed to serialize ad-template evidence in the page: {message}"), + }); + None + } + EvidenceEnvelope::Evidence { json } => { + match serde_json::from_str::(&json) { + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence: {error}"), + }); + None + } + } + } + } +} + +/// Whether a Chrome/Chromium fixture is available for browser-backed tests. +/// +/// Skips optional local runs, but makes the scripted/CI contract fail loudly. +/// Shared with the generation collector's tests so the contract has one +/// definition. +#[cfg(test)] +pub(crate) fn browser_fixture_available() -> bool { + if resolve_chrome(None).is_ok() { + return true; + } + assert!( + std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), + "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" + ); + false +} + +#[cfg(test)] +mod tests { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + use std::sync::mpsc; + + use super::*; + use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, build_ad_template_init_script, + }; + + const AD_TEMPLATE_COLLECTOR_JS: &str = include_str!("ad_template_collector.js"); + + #[test] + fn rust_and_javascript_evidence_entry_caps_match() { + // Parse the declared value rather than matching the whole line, so JS + // punctuation or spacing cannot false-alarm on a still-correct cap. + let declared = AD_TEMPLATE_COLLECTOR_JS + .lines() + .find_map(|line| line.trim().strip_prefix("const __ts_max_entries =")) + .and_then(|value| value.trim().trim_end_matches(';').parse::().ok()) + .expect("should declare __ts_max_entries in the collector script"); + + assert_eq!( + declared, MAX_EVIDENCE_ENTRIES, + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" + ); + } + + #[test] + fn well_known_chrome_paths_are_known_for_this_os() { + // macOS/Linux/Windows each have candidate paths; guards the cfg branches. + assert!( + !well_known_chrome_paths().is_empty(), + "supported OSes should list candidate Chrome install paths" + ); + } + + #[test] + fn oversized_ad_evidence_is_an_explicit_warning() { + let mut warnings = Vec::new(); + let evidence = decode_ad_evidence_envelope(EvidenceEnvelope::TooLarge, &mut warnings); + + assert!(evidence.is_none()); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "ad_evidence_too_large"); + } + + #[test] + fn supplied_cookie_is_host_only_and_root_scoped() { + let url = + url::Url::parse("https://publisher.example/news/story").expect("should parse test URL"); + let cookie = host_cookie("clearance", "token", &url).expect("should build cookie"); + + assert!(cookie.domain.is_none(), "host-only cookies omit Domain"); + assert_eq!(cookie.path.as_deref(), Some("/")); + assert_eq!( + cookie.url.as_deref(), + Some("https://publisher.example"), + "the origin scopes a host-only cookie before first navigation" + ); + assert_eq!(cookie.secure, Some(true), "HTTPS cookies must be Secure"); + } + + #[test] + fn cookie_install_error_identifies_name_without_a_value() { + let error = format_cookie_install_error( + "datadome", + "invalid cookie value operator-secret-cookie-value", + ); + + assert_eq!(error, "failed to set cookie `datadome`"); + assert!(!error.contains("operator-secret-cookie-value")); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn supplied_cookie_reaches_first_navigation() { + if !browser_fixture_available() { + return; + } + + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + let (request_tx, request_rx) = mpsc::channel(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + request_tx + .send(String::from_utf8_lossy(&request).into_owned()) + .expect("should send captured request"); + + let body = b"cookie fixture"; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .expect("should write fixture headers"); + stream.write_all(body).expect("should write fixture body"); + }); + + let collector = BrowserCollector { + settle_quiet: Duration::from_millis(100), + settle_max: Duration::from_secs(1), + ..BrowserCollector::new() + }; + collector + .collect_page(BrowserCollectRequest { + url: url::Url::parse(&format!("http://{address}/")) + .expect("should parse fixture URL"), + init_scripts: Vec::new(), + scroll: false, + collect_ad_evidence: false, + cookies: vec![("clearance".to_string(), "token".to_string())], + }) + .expect("cookie should be installed before first navigation"); + + let request = request_rx + .recv_timeout(Duration::from_secs(5)) + .expect("fixture should receive the first navigation"); + assert!( + request.lines().any(|line| { + line.split_once(':').is_some_and(|(name, value)| { + name.eq_ignore_ascii_case("cookie") + && value + .trim() + .split(';') + .any(|cookie| cookie.trim() == "clearance=token") + }) + }), + "first navigation should carry the supplied cookie; request was {request:?}" + ); + } + + /// A self-contained page that stubs just enough of GPT (no network) for the + /// collector to observe a defined slot via the wrapped `defineSlot` and the + /// `getSlots()` scrape. + const GPT_FIXTURE: &str = r#" + + + +
+ + + +"#; + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_gpt_slot_from_local_fixture() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/browser_collector.rs deleted file mode 100644 index 87a2ccc2c..000000000 --- a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use chromiumoxide::ArcHttpRequest; -use chromiumoxide::browser::{Browser, BrowserConfig}; -use futures::StreamExt as _; -use serde::Deserialize; -use tempfile::TempDir; -use tokio::runtime::Builder; -use tokio::time::{sleep, timeout}; -use url::Url; -use which::which; - -use crate::commands::audit::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, -}; -use crate::error::{CliResult, report_error}; - -const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); -const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); -const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; -const RESOURCE_TIMING_BUFFER_WARNING: &str = - "browser resource timing buffer reached its default size; some network assets may be missing"; - -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; - -impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { - let runtime = Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Tokio runtime for browser audit: {error}" - )) - })?; - - runtime.block_on(collect_page_via_browser_async(target_url)) - } -} - -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { - let chrome_executable = find_browser_executable()?; - let user_data_dir = TempDir::new().map_err(|error| { - report_error(format!( - "failed to create temporary browser profile for audit: {error}" - )) - })?; - let config = BrowserConfig::builder() - .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .new_headless_mode() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; - - let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { - report_error(format!( - "failed to launch Chrome/Chromium for audit: {error}" - )) - })?; - - let handler_task = tokio::spawn(async move { - while let Some(event) = handler.next().await { - if event.is_err() { - break; - } - } - }); - - let result = collect_page_from_browser(&mut browser, target_url).await; - - let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) - .await - .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { - report_error(format!("failed to close browser after audit: {error}")) - }) - }); - if close_result.is_err() { - handler_task.abort(); - } - let _ = handler_task.await; - - match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), - } -} - -async fn collect_page_from_browser( - browser: &mut Browser, - target_url: &Url, -) -> CliResult { - let page = browser.new_page("about:blank").await.map_err(|error| { - report_error(format!("failed to create browser page for audit: {error}")) - })?; - - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; - - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; - - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); - } - if !wait_for_page_settle(&page).await? { - warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - } - - let final_url = page - .url() - .await - .map_err(|error| report_error(format!("failed to read final page URL: {error}")))? - .ok_or_else(|| report_error("browser page URL was empty after navigation"))?; - let page_title = page - .get_title() - .await - .map_err(|error| report_error(format!("failed to read page title: {error}")))?; - let html = page - .content() - .await - .map_err(|error| report_error(format!("failed to read rendered page HTML: {error}")))?; - - let script_tags: Vec = page - .evaluate( - r#"() => Array.from(document.scripts).map((script) => ({ - src: script.src || null, - inline_text: script.src ? null : (script.textContent || null), - }))"#, - ) - .await - .map_err(|error| report_error(format!("failed to read rendered script tags: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode rendered script tag data: {error}" - )) - })?; - - let network_requests: Vec = page - .evaluate( - r#"() => performance.getEntriesByType('resource').map((entry) => ({ - url: entry.name, - initiator_type: entry.initiatorType || null, - }))"#, - ) - .await - .map_err(|error| { - report_error(format!( - "failed to read browser performance resource entries: {error}" - )) - })? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode browser performance resource data: {error}" - )) - })?; - - if let Some(warning) = resource_timing_buffer_warning(network_requests.len()) { - warnings.push(warning.to_string()); - } - - Ok(CollectedPage { - requested_url: target_url.to_string(), - final_url, - page_title: page_title.filter(|title| !title.trim().is_empty()), - html, - script_tags: script_tags - .into_iter() - .map(|script| CollectedScriptTag { - src: script.src, - inline_text: script.inline_text.filter(|text| !text.trim().is_empty()), - }) - .collect(), - network_requests: network_requests - .into_iter() - .map(|entry| CollectedRequest { - url: entry.url, - resource_type: entry.initiator_type, - }) - .collect(), - warnings, - }) -} - -async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { - let mut elapsed = Duration::ZERO; - let mut previous_count = None; - let mut stable_for = Duration::ZERO; - - while elapsed < SETTLE_MAX_WAIT { - let ready_state: String = page - .evaluate("document.readyState") - .await - .map_err(|error| report_error(format!("failed to read document ready state: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!("failed to decode document ready state: {error}")) - })?; - let resource_count: usize = page - .evaluate("performance.getEntriesByType('resource').length") - .await - .map_err(|error| report_error(format!("failed to read resource count: {error}")))? - .into_value() - .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - - if ready_state == "complete" { - if previous_count == Some(resource_count) { - stable_for += SETTLE_POLL_INTERVAL; - } else { - stable_for = Duration::ZERO; - } - - if stable_for >= SETTLE_QUIET_PERIOD { - return Ok(true); - } - } - - previous_count = Some(resource_count); - sleep(SETTLE_POLL_INTERVAL).await; - elapsed += SETTLE_POLL_INTERVAL; - } - - Ok(false) -} - -fn validate_navigation_response(navigation_response: ArcHttpRequest) -> CliResult> { - let request = navigation_response - .ok_or_else(|| report_error("browser audit did not capture the main document response"))?; - - if let Some(failure_text) = &request.failure_text { - return Err(report_error(format!( - "main document request failed: {failure_text}" - ))); - } - - let response = request.response.as_ref().ok_or_else(|| { - report_error("browser audit did not capture the main document HTTP response") - })?; - - if is_successful_navigation_status(response.status) { - return Ok(None); - } - - Ok(Some(format!( - "audit request returned HTTP {} {} for `{}`; results may be partial", - response.status, response.status_text, response.url - ))) -} - -fn is_successful_navigation_status(status: i64) -> bool { - (200..400).contains(&status) -} - -fn resource_timing_buffer_warning(resource_count: usize) -> Option<&'static str> { - (resource_count >= RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD) - .then_some(RESOURCE_TIMING_BUFFER_WARNING) -} - -fn find_browser_executable() -> CliResult { - for candidate in browser_executable_path_candidates() { - if let Ok(path) = which(candidate) { - return Ok(path); - } - } - - for candidate in browser_executable_fallbacks() { - let candidate_path = Path::new(candidate); - if candidate_path.is_file() { - return Ok(candidate_path.to_path_buf()); - } - } - - Err(report_error( - "Chrome/Chromium was not found on PATH or in the standard local install locations checked by `ts audit`. Install a local Chrome or Chromium binary before running `ts audit`.", - )) -} - -fn browser_executable_path_candidates() -> &'static [&'static str] { - &[ - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "chrome", - "Google Chrome", - "Google Chrome for Testing", - ] -} - -fn browser_executable_fallbacks() -> &'static [&'static str] { - #[cfg(target_os = "macos")] - { - &[ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - ] - } - - #[cfg(target_os = "linux")] - { - &[ - "/usr/bin/google-chrome", - "/usr/bin/google-chrome-stable", - "/usr/bin/chromium", - "/usr/bin/chromium-browser", - "/snap/bin/chromium", - ] - } - - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - &[] - } -} - -#[derive(Debug, Deserialize)] -struct BrowserScriptTag { - src: Option, - inline_text: Option, -} - -#[derive(Debug, Deserialize)] -struct BrowserPerformanceEntry { - url: String, - initiator_type: Option, -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use chromiumoxide::cdp::browser_protocol::network::{Headers, RequestId, Response}; - use chromiumoxide::cdp::browser_protocol::security::SecurityState; - use chromiumoxide::handler::http::HttpRequest; - - use super::*; - - #[test] - fn successful_navigation_status_allows_redirects_but_rejects_errors() { - assert!(is_successful_navigation_status(200)); - assert!(is_successful_navigation_status(302)); - assert!(is_successful_navigation_status(399)); - assert!(!is_successful_navigation_status(199)); - assert!(!is_successful_navigation_status(400)); - assert!(!is_successful_navigation_status(500)); - } - - #[test] - fn navigation_response_returns_warning_for_http_error_status() { - let warning = - validate_navigation_response(navigation_response_with_status(403, "Forbidden")) - .expect("should validate navigation response") - .expect("should return warning for HTTP error status"); - - assert_eq!( - warning, - "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", - "should warn and continue when the main document returns an HTTP error" - ); - } - - #[test] - fn resource_timing_buffer_warning_starts_at_threshold() { - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD - 1), - None, - "should not warn before the resource timing buffer threshold" - ); - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD), - Some(RESOURCE_TIMING_BUFFER_WARNING), - "should warn when the resource timing buffer reaches the threshold" - ); - } - - #[test] - fn browser_path_candidates_include_common_names() { - let candidates = browser_executable_path_candidates(); - - assert!(candidates.contains(&"google-chrome")); - assert!(candidates.contains(&"chromium")); - assert!(candidates.contains(&"Google Chrome for Testing")); - } - - fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { - let mut request = - HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); - request.response = Some( - Response::builder() - .url("https://example.com/") - .status(status) - .status_text(status_text) - .headers(Headers::default()) - .mime_type("text/html") - .charset("utf-8") - .connection_reused(false) - .connection_id(1.0) - .encoded_data_length(0.0) - .security_state(SecurityState::Secure) - .build() - .expect("should build navigation response"), - ); - - Some(Arc::new(request)) - } -} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs b/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs new file mode 100644 index 000000000..17b05a491 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs @@ -0,0 +1,83 @@ +//! Shared deterministic browser scrolling for audit commands. + +use std::time::Duration; + +use chromiumoxide::Page; + +const SCROLL_STEP_DELAY: Duration = Duration::from_millis(250); +/// Bound for each CDP operation after navigation. +pub(crate) const CDP_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +/// A best-effort browser scroll operation that could not be completed. +#[derive(Debug, derive_more::Display)] +pub(crate) enum ScrollFailure { + /// Chrome rejected the page evaluation. + #[display("browser page evaluation failed: {_0}")] + Evaluation(String), + /// Chrome did not complete the page evaluation within the operation bound. + #[display("browser page evaluation timed out")] + Timeout, +} + +impl core::error::Error for ScrollFailure {} + +impl ScrollFailure { + /// Stable warning code used by structured audit output. + pub(crate) const fn code(&self) -> &'static str { + match self { + Self::Evaluation(_) => "page_evaluation_failed", + Self::Timeout => "page_evaluation_timeout", + } + } +} + +/// Scrolls a page through deterministic fractions to trigger lazy content. +pub(crate) async fn scroll_page(page: &chromiumoxide::Page) -> Vec { + let mut failures = Vec::new(); + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, \ + document.documentElement.scrollHeight) * {fraction}))" + ); + if let Err(failure) = evaluate(page, script).await { + failures.push(failure); + } + tokio::time::sleep(SCROLL_STEP_DELAY).await; + } + if let Err(failure) = evaluate(page, "window.scrollTo(0, 0)").await { + failures.push(failure); + } + failures +} + +/// Evaluates a browser expression with the shared operation bound and errors. +pub(crate) async fn evaluate( + page: &Page, + expression: impl Into, +) -> Result<(), ScrollFailure> { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression.into())) + .await + .map_err(|_| ScrollFailure::Timeout)? + .map(|_| ()) + .map_err(|error| ScrollFailure::Evaluation(error.to_string())) +} + +#[cfg(test)] +mod tests { + use super::ScrollFailure; + + #[test] + fn scroll_failures_have_stable_messages() { + assert_eq!( + ScrollFailure::Evaluation("execution context was destroyed".to_string()).to_string(), + "browser page evaluation failed: execution context was destroyed" + ); + assert_eq!( + ScrollFailure::Timeout.to_string(), + "browser page evaluation timed out" + ); + + fn assert_error() {} + assert_error::(); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index 314ae54fc..322b96e89 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -1,41 +1,356 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::{Args, ValueEnum}; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Default quiet window for generation's browser collector. +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for generation's browser collector. +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; +/// Default quiet window for `ts audit page` and `ts audit ad-templates verify`. +/// +/// [`BrowserOpts`] and `BrowserCollector::new` must agree, or a collector built +/// in code drifts from the parsed flags without anything failing. +pub(crate) const PAGE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// See [`PAGE_SETTLE_QUIET_MS`] for why this is shared rather than duplicated. +pub(crate) const PAGE_SETTLE_MAX_MS: u64 = 10_000; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Browser device profile used for viewport and user-agent emulation. + #[arg(long = "browser-profile", value_enum, default_value_t = BrowserProfile::Desktop)] + pub profile: BrowserProfile, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = PAGE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = PAGE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Browser options for generation, whose device selection is controlled by +/// `--profiles` rather than the verifier's singular `--browser-profile`. +#[derive(Debug, Clone, Args)] +pub struct GenerateBrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then auto-detection. + #[arg(long)] + pub chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg(long, default_value_t = GENERATE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Shared budget in milliseconds for the initial, post-scroll, and GPT settle waits. + /// + /// Starts after navigation; scrolling consumes this budget. GPT polling + /// precedes metadata extraction so those reads cannot consume its budget. + /// Navigation and browser operations have separate timeouts, so this is not + /// a total page deadline. An exhausted budget still takes one GPT snapshot; + /// two consecutive empty GPT polls end the wait early regardless of the budget. + #[arg(long, default_value_t = GENERATE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as the evidence it writes config from, so an + /// invalid certificate could mean an impersonator is harvesting the session + /// and fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Defaults mirroring the `#[arg(default_value_t)]` values above, so a path that +/// builds these options in code (the legacy `ts audit ` form) behaves like +/// the parsed command. +impl Default for GenerateBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: GENERATE_SETTLE_QUIET_MS, + settle_max_ms: GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, + } + } +} + +impl GenerateBrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + +/// Browser device profile shared by page audits and ad-template verification. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum BrowserProfile { + /// Desktop Chrome at 1280×800. + #[default] + Desktop, + /// Mobile-sized viewport with a mobile user agent. + Mobile, +} + +impl BrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -pub(crate) struct CollectedPage { - pub(crate) requested_url: String, - pub(crate) final_url: String, - pub(crate) page_title: Option, - pub(crate) html: String, - pub(crate) script_tags: Vec, - pub(crate) network_requests: Vec, - pub(crate) warnings: Vec, +fn validate_settle_window(quiet_ms: u64, max_ms: u64) -> Result<(), String> { + if quiet_ms > max_ms { + return Err(format!( + "--settle-quiet-ms ({quiet_ms}) cannot exceed --settle-max-ms ({max_ms})" + )); + } + Ok(()) +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of ` + +"#; + + const DELAYED_GPT_FIXTURE: &str = r#" + + +
+
+ + +"#; + + /// A registry whose second burst lands on a real timer rather than on + /// observation, so two consecutive identical reads can straddle the gap. + /// + /// The first burst is gated on the first `getSlots()` call — otherwise it + /// would complete during the settle wait, long before polling starts — but + /// the gap that follows is genuine wall-clock time, which is what makes + /// this fixture able to detect a criterion that exits on a single matching + /// pair. + const BATCHED_GPT_FIXTURE: &str = r#" + + +
+
+ + +"#; + + /// Local HTTP server that serves one fixture document for a browser test. + /// + /// Chrome opens several sockets per navigation: the document request, socket + /// pool preconnects that close without sending anything, and speculative + /// `/favicon.ico`, `/robots.txt`, and `/sitemap.xml` fetches. The server must + /// therefore keep accepting connections for as long as the test runs, must + /// serve them concurrently so a silent preconnect cannot stall the document + /// request, and must treat a connection that carries no request as normal. + /// Serving a single connection instead loses the accept race and fails the + /// navigation with `net::ERR_CONNECTION_REFUSED`. + struct GptFixtureServer { + url: Url, + address: SocketAddr, + shutdown: Arc, + acceptor: Option>, + } + + impl GptFixtureServer { + fn url(&self) -> &Url { + &self.url + } + + fn address(&self) -> SocketAddr { + self.address + } + } + + impl Drop for GptFixtureServer { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::Relaxed); + // Wake the blocking accept so shutdown can join the listener thread. + let _ = TcpStream::connect(self.address); + if let Some(acceptor) = self.acceptor.take() { + let _ = acceptor.join(); + } + } + } + + /// Serves `html` for `GET /` until the returned server is dropped. + fn gpt_fixture_server(html: &'static str) -> GptFixtureServer { + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + let shutdown = Arc::new(AtomicBool::new(false)); + let acceptor_shutdown = Arc::clone(&shutdown); + let acceptor = std::thread::spawn(move || { + while !acceptor_shutdown.load(Ordering::Relaxed) { + match listener.accept() { + Ok((stream, _)) => { + if acceptor_shutdown.load(Ordering::Relaxed) { + return; + } + std::thread::spawn(move || serve_gpt_fixture_connection(stream, html)); + } + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(_) => return, + } + } + }); + + GptFixtureServer { + url: Url::parse(&format!("http://{address}/")).expect("should parse fixture URL"), + address, + shutdown, + acceptor: Some(acceptor), + } + } + + /// Answers one fixture connection, ignoring sockets that carry no request. + fn serve_gpt_fixture_connection(mut stream: TcpStream, html: &'static str) { + if stream + .set_read_timeout(Some(Duration::from_secs(10))) + .is_err() + { + return; + } + + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + match stream.read(&mut chunk) { + // A preconnect socket closes without a request; that is not a failure. + Ok(0) => return, + Ok(chunk_len) => request.extend_from_slice(&chunk[..chunk_len]), + Err(_) => return, + } + if request.len() > 16 * 1024 { + return; + } + } + + if !request.starts_with(b"GET / HTTP") { + let _ = write!( + stream, + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + return; + } + + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + html.len(), + html, + ); + } + + /// Requests a path from a fixture server, failing if no response arrives in time. + fn fixture_response(address: SocketAddr, path: &str) -> String { + let mut stream = TcpStream::connect(address).expect("should connect to the fixture server"); + // A fixture that stalls behind another connection must fail this read + // rather than deliver the document late. + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("should bound the fixture client read"); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: fixture.example.com\r\n\r\n" + ) + .expect("should send the fixture request"); + let mut response = String::new(); + stream + .read_to_string(&mut response) + .expect("should read the fixture response before the client timeout"); + response + } + + #[test] + fn fixture_server_serves_the_document_after_a_socket_that_sends_no_request() { + let fixture = gpt_fixture_server(DELAYED_GPT_FIXTURE); + + // Chrome's socket-pool preconnect opens a socket and closes it without + // sending anything, and it can win the accept race with the navigation. + drop(TcpStream::connect(fixture.address()).expect("should open a preconnect socket")); + + let response = fixture_response(fixture.address(), "/"); + assert!( + response.starts_with("HTTP/1.1 200 OK"), + "should still answer the document request: {response}" + ); + assert!( + response.contains("ad-z-delayed-0"), + "should serve the fixture document body: {response}" + ); + } + + #[test] + fn fixture_server_serves_the_document_while_a_silent_socket_stays_open() { + let fixture = gpt_fixture_server(DELAYED_GPT_FIXTURE); + + // Held open, sending nothing: serving connections sequentially would + // block the document request behind this socket's read timeout. + let _silent = TcpStream::connect(fixture.address()).expect("should open a silent socket"); + + let response = fixture_response(fixture.address(), "/"); + assert!( + response.starts_with("HTTP/1.1 200 OK"), + "should answer the document request without waiting for the silent socket: {response}" + ); + } + + #[test] + fn fixture_server_answers_every_speculative_browser_request() { + let fixture = gpt_fixture_server(LAZY_GPT_FIXTURE); + + // Chrome follows the document with /favicon.ico, /robots.txt and + // /sitemap.xml probes on separate connections. + for path in ["/favicon.ico", "/robots.txt", "/sitemap.xml"] { + let response = fixture_response(fixture.address(), path); + assert_eq!( + response, + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "should return an empty 404 for {path}" + ); + } + let response = fixture_response(fixture.address(), "/"); + assert!( + response.starts_with("HTTP/1.1 200 OK") && response.ends_with(LAZY_GPT_FIXTURE), + "should still serve the document after speculative requests" + ); + } + + fn gpt_slot(unit_path: &str, div_id: &str) -> CollectedGptSlot { + CollectedGptSlot { + gam_unit_path: unit_path.to_string(), + div_id: div_id.to_string(), + sizes: vec![(300, 250)], + } + } + + #[test] + fn empty_gpt_reading_voids_the_stability_streak() { + let previous = vec![gpt_slot("/123/a", "ad-a-0")]; + + assert_eq!( + gpt_registry_reading(Some(&previous), &[]), + GptRegistryReading::Empty, + "an empty read should discard the earlier non-empty reading" + ); + assert_eq!( + gpt_registry_reading(None, &[]), + GptRegistryReading::Empty, + "an empty read with no history should still classify as empty" + ); + } + + #[test] + fn first_nonempty_gpt_reading_is_a_change() { + let current = vec![gpt_slot("/123/a", "ad-a-0")]; + + assert_eq!( + gpt_registry_reading(None, ¤t), + GptRegistryReading::Changed, + "the first non-empty read has nothing to repeat" + ); + } + + #[test] + fn identical_gpt_reading_repeats() { + let slots = vec![gpt_slot("/123/a", "ad-a-0"), gpt_slot("/123/b", "ad-b-0")]; + + assert_eq!( + gpt_registry_reading(Some(&slots), &slots), + GptRegistryReading::Repeated, + "an unchanged registry should classify as repeated" + ); + } + + #[test] + fn gpt_reading_stability_is_order_sensitive() { + let previous = vec![gpt_slot("/123/a", "ad-a-0"), gpt_slot("/123/b", "ad-b-0")]; + let reordered = vec![gpt_slot("/123/b", "ad-b-0"), gpt_slot("/123/a", "ad-a-0")]; + let grown = vec![ + gpt_slot("/123/a", "ad-a-0"), + gpt_slot("/123/b", "ad-b-0"), + gpt_slot("/123/c", "ad-c-0"), + ]; + + assert_eq!( + gpt_registry_reading(Some(&previous), &reordered), + GptRegistryReading::Changed, + "the same slot set in a different order is not yet stable" + ); + assert_eq!( + gpt_registry_reading(Some(&previous), &grown), + GptRegistryReading::Changed, + "a later registration burst should reset the streak" + ); + } + + #[tokio::test(start_paused = true)] + async fn registry_poll_stops_after_two_empty_readings() { + let mut reads = 0; + let mut warnings = Vec::new(); + let start = tokio::time::Instant::now(); + let slots = poll_gpt_registry( + || { + reads += 1; + std::future::ready(Ok(Vec::new())) + }, + Duration::from_millis(750), + Duration::from_secs(12), + &mut warnings, + ) + .await; + assert!(slots.is_empty(), "should retain an empty registry"); + assert_eq!(reads, 2, "should stop after consecutive empty snapshots"); + assert_eq!( + start.elapsed(), + SETTLE_POLL_INTERVAL, + "should avoid spending the full budget" + ); + assert!( + warnings.is_empty(), + "should leave the empty-state diagnostic to the caller" + ); + } + + #[tokio::test(start_paused = true)] + async fn registry_poll_reads_once_with_zero_budget() { + let expected = vec![gpt_slot("/123/a", "ad-a")]; + let mut reads = 0; + let mut warnings = Vec::new(); + let slots = poll_gpt_registry( + || { + reads += 1; + std::future::ready(Ok(expected.clone())) + }, + Duration::ZERO, + Duration::ZERO, + &mut warnings, + ) + .await; + assert_eq!(reads, 1, "should always collect an initial snapshot"); + assert_eq!( + slots, expected, + "should retain actual evidence at zero budget" + ); + } + + #[tokio::test(start_paused = true)] + async fn registry_poll_resets_empty_streak_and_preserves_latest_evidence() { + let first = vec![gpt_slot("/123/a", "ad-a")]; + let latest = vec![gpt_slot("/123/b", "ad-b")]; + let mut readings = [ + Vec::new(), + first, + Vec::new(), + latest.clone(), + Vec::new(), + Vec::new(), + ] + .into_iter(); + let mut warnings = Vec::new(); + let result = poll_gpt_registry( + || { + std::future::ready(Ok(readings + .next() + .expect("should stop at consecutive empties"))) + }, + Duration::from_millis(750), + Duration::from_secs(12), + &mut warnings, + ) + .await; + assert_eq!( + result, latest, + "should retain the latest snapshot through empty readings" + ); + assert!( + readings.next().is_none(), + "should reset each empty streak on a nonempty reading" + ); + assert!(warnings.is_empty(), "should not report budget exhaustion"); + } + + #[tokio::test(start_paused = true)] + async fn registry_poll_requires_dwell_after_a_later_registration_burst() { + let first = vec![gpt_slot("/123/a", "ad-a")]; + let latest = vec![gpt_slot("/123/a", "ad-a"), gpt_slot("/123/b", "ad-b")]; + let start = tokio::time::Instant::now(); + let mut warnings = Vec::new(); + let result = poll_gpt_registry( + || { + std::future::ready(Ok(if start.elapsed() < Duration::from_millis(500) { + first.clone() + } else { + latest.clone() + })) + }, + Duration::from_millis(750), + Duration::from_secs(12), + &mut warnings, + ) + .await; + assert_eq!( + result, latest, + "should include the second registration burst" + ); + assert_eq!( + start.elapsed(), + Duration::from_millis(1500), + "should reset the dwell after registration changes" + ); + assert!(warnings.is_empty(), "should stabilize within the budget"); + } + + #[tokio::test(start_paused = true)] + async fn registry_poll_expiry_preserves_changing_evidence() { + let mut reads = 0; + let mut warnings = Vec::new(); + let result = poll_gpt_registry( + || { + reads += 1; + std::future::ready(Ok(vec![gpt_slot("/123/a", &format!("ad-{reads}"))])) + }, + Duration::from_millis(750), + Duration::from_millis(600), + &mut warnings, + ) + .await; + assert_eq!(reads, 3, "should stop at the budget without an extra read"); + assert_eq!( + result, + [gpt_slot("/123/a", "ad-3")], + "should retain the most recent evidence" + ); + assert_eq!(warnings.len(), 1, "should report partial evidence once"); + assert!( + warnings[0].contains("600ms budget"), + "should identify budget exhaustion" + ); + } + + #[tokio::test(start_paused = true)] + async fn registry_poll_read_failure_preserves_evidence_and_reports_the_cause() { + let expected = vec![gpt_slot("/123/a", "ad-a")]; + let mut readings = [Ok(expected.clone()), Err("read unavailable".to_string())].into_iter(); + let mut warnings = Vec::new(); + let result = poll_gpt_registry( + || std::future::ready(readings.next().expect("should stop on read failure")), + Duration::from_millis(750), + Duration::from_secs(12), + &mut warnings, + ) + .await; + assert_eq!( + result, expected, + "should preserve evidence on a later read failure" + ); + assert_eq!( + warnings, + ["read unavailable"], + "should distinguish transport failure from expiry" + ); + } + + #[test] + fn expired_stability_budget_warns_only_for_partial_evidence() { + let mut partial_warnings = Vec::new(); + let partial = expired_gpt_registry( + vec![gpt_slot("/123/a", "ad-a-0")], + Duration::from_millis(750), + Duration::from_millis(12_000), + &mut partial_warnings, + ); + + assert_eq!(partial.len(), 1, "the latest snapshot should be returned"); + assert_eq!( + partial_warnings, + [ + "GPT slot registration did not hold still for 750ms within the 12000ms budget; \ + using the latest snapshot of 1 slot(s), so results may be partial; raise \ + `--settle-max-ms` to wait longer" + ], + "a partial snapshot should name the slot count, the dwell, and the budget" + ); + + let mut empty_warnings = Vec::new(); + let empty = expired_gpt_registry( + Vec::new(), + Duration::from_millis(750), + Duration::from_millis(12_000), + &mut empty_warnings, + ); + + assert!(empty.is_empty(), "an empty registry should stay empty"); + assert!( + empty_warnings.is_empty(), + "an empty registry is reported by the GPT state diagnostic instead" + ); + } + + #[test] + fn successful_navigation_status_allows_redirects_but_rejects_errors() { + assert!(is_successful_navigation_status(200)); + assert!(is_successful_navigation_status(302)); + assert!(is_successful_navigation_status(399)); + assert!(!is_successful_navigation_status(199)); + assert!(!is_successful_navigation_status(400)); + assert!(!is_successful_navigation_status(500)); + } + + #[test] + fn navigation_response_returns_warning_for_http_error_status() { + let warning = + validate_navigation_response(navigation_response_with_status(403, "Forbidden")) + .expect("should validate navigation response") + .expect("should return warning for HTTP error status"); + + assert_eq!( + warning, + "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", + "should warn and continue when the main document returns an HTTP error" + ); + } + + #[test] + fn navigation_response_reports_chromium_request_failure() { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.failure_text = Some("net::ERR_BLOCKED_BY_ORB".to_string()); + + let error = validate_navigation_response(Some(Arc::new(request))) + .expect_err("should reject Chromium request failures"); + + assert_eq!( + error, "main document request failed: net::ERR_BLOCKED_BY_ORB", + "the crawl should retain the browser failure for its final skipped-page note" + ); + } + + #[test] + fn resource_timing_buffer_warning_starts_at_threshold() { + assert_eq!( + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE - 1), + None, + "should not warn before the resource timing buffer threshold" + ); + assert_eq!( + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE), + Some(RESOURCE_TIMING_BUFFER_WARNING), + "should warn when the resource timing buffer reaches the threshold" + ); + } + + #[test] + fn browser_path_candidates_include_common_names() { + let candidates = crate::commands::audit::browser::CHROME_NAMES; + + assert!(candidates.contains(&"google-chrome")); + assert!(candidates.contains(&"chromium")); + assert!(candidates.contains(&"Google Chrome for Testing")); + } + + #[test] + fn browser_run_reports_close_error_before_wait_error() { + let result = combine_browser_run_results( + Ok(()), + Ok(()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve teardown error"), + "close failed", + "the close failure is the first teardown failure" + ); + } + + #[test] + fn browser_run_reports_wait_error_when_close_succeeds() { + let result = + combine_browser_run_results(Ok(()), Ok(()), Ok(()), Err("wait failed".to_string())); + + assert_eq!( + result.expect_err("should preserve wait error"), + "wait failed", + "a wait failure must not be mislabeled as a close failure" + ); + } + + #[test] + fn browser_run_preserves_collection_error_over_later_failures() { + let result = combine_browser_run_results( + Err("collection failed".to_string()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve first browser run error"), + "collection failed" + ); + } + + #[test] + fn browser_run_reports_finalization_progress_before_teardown_errors() { + let result = combine_browser_run_results( + Ok(()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve finalization progress error"), + "finalization progress failed" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn progress_failure_still_finalizes_browser_session() { + if !browser_fixture_available() { + return; + } + + let collector = BrowserAuditCollector::default(); + let target = Url::parse("http://127.0.0.1:9/").expect("should parse fixture URL"); + let mut phases = Vec::new(); + let error = collector + .collect_pages( + &[target], + &[], + &mut |progress| match progress { + CollectionProgress::Launching => { + phases.push("launching"); + Ok(()) + } + CollectionProgress::Loading { .. } => { + phases.push("loading"); + Err(report_error("simulated progress failure")) + } + CollectionProgress::Planning => { + phases.push("planning"); + Ok(()) + } + CollectionProgress::Finalizing => { + phases.push("finalizing"); + Ok(()) + } + }, + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure after browser teardown"); + + let rendered_error = format!("{error:?}"); + assert!( + rendered_error.contains("simulated progress failure"), + "should preserve progress failure, got {rendered_error}" + ); + assert_eq!(phases, ["launching", "loading", "finalizing"]); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_lazy_gpt_slot_only_when_scroll_is_enabled() { + if !browser_fixture_available() { + return; + } + + let unscrolled_fixture = gpt_fixture_server(LAZY_GPT_FIXTURE); + let without_scroll = BrowserAuditCollector::default() + .collect_page(unscrolled_fixture.url(), &[]) + .expect("should collect without scrolling"); + let scrolled_fixture = gpt_fixture_server(LAZY_GPT_FIXTURE); + let with_scroll = BrowserAuditCollector::default() + .with_scroll(true) + .collect_page(scrolled_fixture.url(), &[]) + .expect("should collect with scrolling"); + + assert!( + without_scroll.gpt_slots.is_empty(), + "lazy GPT slot should not exist before scrolling" + ); + assert!( + with_scroll + .gpt_slots + .iter() + .any(|slot| { slot.gam_unit_path == "/123/lazy" && slot.div_id == "ad-lazy-0" }), + "scrolling should trigger and collect the lazy GPT slot" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn waits_for_delayed_gpt_registry_to_stabilize_in_definition_order() { + if !browser_fixture_available() { + return; + } + + let delayed_fixture = gpt_fixture_server(DELAYED_GPT_FIXTURE); + let collected = BrowserAuditCollector::default() + .collect_page(delayed_fixture.url(), &[]) + .expect("should collect delayed GPT registry"); + + assert_eq!( + collected.gpt_slots, + vec![ + CollectedGptSlot { + gam_unit_path: "/123/z-delayed".to_string(), + div_id: "ad-z-delayed-0".to_string(), + sizes: vec![(300, 250)], + }, + CollectedGptSlot { + gam_unit_path: "/123/a-delayed".to_string(), + div_id: "ad-a-delayed-0".to_string(), + sizes: vec![(728, 90)], + }, + ], + "collector should wait for stable registration without reordering slots" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn exhausted_page_settle_budget_still_collects_one_gpt_snapshot() { + if !browser_fixture_available() { + return; + } + + for scroll in [false, true] { + let fixture = gpt_fixture_server(BATCHED_GPT_FIXTURE); + // A quiet window equal to the maximum cannot finish inside that + // maximum, so the initial settle spends the entire shared budget. + let options = GenerateBrowserOpts { + settle_quiet_ms: 600, + settle_max_ms: 600, + ..GenerateBrowserOpts::default() + }; + let collected = BrowserAuditCollector::default() + .with_browser_options(&options) + .with_scroll(scroll) + .collect_page(fixture.url(), &[]) + .expect("should collect a snapshot after the shared budget expires"); + + assert_eq!( + collected.gpt_slots, + [gpt_slot("/123/first-batch", "ad-first-batch-0")], + "should take one snapshot without restarting the GPT budget (scroll={scroll})" + ); + assert!( + collected + .warnings + .iter() + .any(|warning| warning.contains("within the 0ms budget")), + "should report exhausted remaining GPT budget (scroll={scroll})" + ); + assert_eq!( + collected.warnings.iter().any(|warning| { + warning.contains("settle budget was exhausted before the post-scroll settle") + }), + scroll, + "should report skipped post-scroll settling only when scrolling (scroll={scroll})" + ); + assert!( + !collected.warnings.iter().any(|warning| { + warning.contains("timed out while waiting for the page to settle after scroll") + }), + "should not report a timeout for a wait that never ran (scroll={scroll})" + ); + } + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn slow_metadata_does_not_consume_gpt_settle_budget() { + if !browser_fixture_available() { + return; + } + + let fixture = gpt_fixture_server(BATCHED_GPT_FIXTURE); + let mut url = fixture.url().clone(); + url.set_fragment(Some("slow-title")); + // The 4s title read exceeds this budget on its own. GPT's second + // batch appears only after polling starts, so a single late snapshot + // cannot substitute for giving the registry its remaining dwell time. + let options = GenerateBrowserOpts { + settle_quiet_ms: 750, + settle_max_ms: 3500, + ..GenerateBrowserOpts::default() + }; + let collected = BrowserAuditCollector::default() + .with_browser_options(&options) + .collect_page(&url, &[]) + .expect("should collect GPT slots and slow metadata"); + + assert_eq!( + collected.page_title.as_deref(), + Some("Slow metadata fixture"), + "should still extract metadata after stabilizing the registry" + ); + assert_eq!( + collected.gpt_slots.len(), + 2, + "should retain both registration batches despite slow metadata extraction" + ); + assert!( + !collected + .warnings + .iter() + .any(|warning| warning.contains("GPT slot registration did not hold still")), + "should let the registry stabilize before reading slow metadata: {:?}", + collected.warnings + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn waits_through_a_timer_driven_gpt_registration_gap() { + if !browser_fixture_available() { + return; + } + + let batched_fixture = gpt_fixture_server(BATCHED_GPT_FIXTURE); + let collected = BrowserAuditCollector::default() + .collect_page(batched_fixture.url(), &[]) + .expect("should collect batched GPT registry"); + + assert_eq!( + collected.gpt_slots, + vec![ + CollectedGptSlot { + gam_unit_path: "/123/first-batch".to_string(), + div_id: "ad-first-batch-0".to_string(), + sizes: vec![(300, 250)], + }, + CollectedGptSlot { + gam_unit_path: "/123/second-batch".to_string(), + div_id: "ad-second-batch-0".to_string(), + sizes: vec![(728, 90)], + }, + ], + "the dwell window should outlast a gap between registration bursts" + ); + } + + fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.response = Some( + Response::builder() + .url("https://example.com/") + .status(status) + .status_text(status_text) + .headers(Headers::default()) + .mime_type("text/html") + .charset("utf-8") + .connection_reused(false) + .connection_id(1.0) + .encoded_data_length(0.0) + .security_state(SecurityState::Secure) + .build() + .expect("should build navigation response"), + ); + + Some(Arc::new(request)) + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs new file mode 100644 index 000000000..dc23af09c --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -0,0 +1,366 @@ +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::error::CliResult; + +/// Warning recorded on a page collected with the audit consent stub installed. +/// +/// A whole-run fact rather than a property of one page, so consumers report it +/// once and unscoped instead of once per page and per profile. +pub(crate) const CONSENT_STUB_WARNING: &str = "consent_stub_active: audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution"; + +/// A user-visible phase reached while collecting browser audit evidence. +#[derive(Debug, Clone, Copy)] +pub(crate) enum CollectionProgress<'a> { + /// The browser process is about to launch. + Launching, + /// A page navigation is about to begin. + Loading { + /// One-based position of this attempted page in the crawl. + current: usize, + /// Total pages when planning has completed, or `None` for the root. + total: Option, + /// Target page; renderers must omit credentials, query, and fragment. + url: &'a Url, + }, + /// Follow-up pages are being selected from the collected root page. + Planning, + /// The browser session is being closed and its process reaped. + Finalizing, +} + +/// Sink invoked synchronously when browser collection reaches a visible phase. +/// +/// Returning an error stops new collection work. An already-launched browser +/// must still be finalized, closed, and waited on before that error is returned. +pub(crate) type ProgressSink<'a> = + &'a mut dyn for<'event> FnMut(CollectionProgress<'event>) -> CliResult<()>; + +/// Sink invoked once per collected page during a batch crawl. +/// +/// Receives the per-page outcome so a failed page can be folded into the run as +/// a warning rather than aborting it; returning `Err` stops the crawl. +pub(crate) type PageSink<'a> = + &'a mut dyn FnMut(&Url, CliResult) -> CliResult; + +/// Plans follow-up URLs from the successfully collected root page. +pub(crate) type RootPlanner<'a> = &'a mut dyn FnMut(&Url, &CollectedPage) -> CliResult>; + +/// Whether a batch crawl should keep going after a page. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ControlFlow { + /// Collect the next target. + Continue, + /// Stop the crawl without an error (budget reached, challenge rate exceeded). + /// + /// What this can prevent depends on the collector: a sequential one loads no + /// further pages, while the browser collector has already finished + /// navigating by the time it folds, so there it only stops the fold. + Stop, +} + +pub(crate) trait AuditCollector { + /// Collects a live page. `cookies` are `(name, value)` pairs set on the + /// browser context before navigation (scoped to `target_url`) so an existing + /// session — e.g. a valid bot-protection clearance cookie — can carry the + /// audit past an origin challenge. + fn collect_page( + &self, + target_url: &Url, + cookies: &[(String, String)], + ) -> CliResult; + + /// Collects several pages in one session, handing each result to `on_page`. + /// + /// The default implementation loops over [`collect_page`](Self::collect_page), + /// which keeps every existing implementor working unchanged. The browser + /// collector overrides it to reuse one Chrome instance and profile across the + /// crawl — a fresh launch per page dominates the cost of a multi-page run, + /// and a shared profile carries bot-protection clearance cookies site-wide. + /// + /// Collectors may buffer results until the browser session closes so CPU-heavy + /// HTML analysis cannot starve a single-threaded CDP event pump. The sink API + /// keeps that buffering policy private and lets simple collectors stream. + /// + /// # Errors + /// + /// Returns an error when `on_page` does, or when the session itself cannot + /// be established. Individual page failures are delivered to `on_page`. + fn collect_pages( + &self, + targets: &[Url], + cookies: &[(String, String)], + on_progress: ProgressSink<'_>, + on_page: PageSink<'_>, + ) -> CliResult<()> { + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 1, + total: Some(targets.len()), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } + + /// Collects a root and follow-up URLs planned from it in one logical crawl. + /// + /// The browser implementation overrides this so planning happens while the + /// root's browser/profile remains open. Simple collectors retain equivalent + /// behavior through the default implementation. + fn collect_site( + &self, + root: &Url, + cookies: &[(String, String)], + on_progress: ProgressSink<'_>, + planner: RootPlanner<'_>, + on_page: PageSink<'_>, + ) -> CliResult<()> { + on_progress(CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + // A root failure is reported through `on_page` rather than returned, so + // the caller sees the reason as a per-page note exactly as it does from + // the browser collector. With no root page there is nothing to plan + // from, so the crawl ends here. + let root_page = match self.collect_page(root, cookies) { + Ok(page) => page, + Err(error) => { + on_page(root, Err(error))?; + return Ok(()); + } + }; + on_progress(CollectionProgress::Planning)?; + let targets = planner(root, &root_page)?; + if on_page(root, Ok(root_page))? == ControlFlow::Stop { + return Ok(()); + } + let total = targets.len() + 1; + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 2, + total: Some(total), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedPage { + pub(crate) requested_url: String, + pub(crate) final_url: String, + pub(crate) page_title: Option, + pub(crate) html: String, + pub(crate) script_tags: Vec, + pub(crate) network_requests: Vec, + /// Slots read from the live GPT registry (`googletag.pubads().getSlots()`). + /// + /// Populated at `defineSlot` time, so this captures configured slots even + /// when the ad request never fires (consent-gated or iframe-issued). + #[serde(default)] + pub(crate) gpt_slots: Vec, + /// Same-origin `a[href]` targets read from the hydrated DOM, absolutized. + /// + /// Read from the live DOM rather than the served HTML on purpose: an + /// app-router page keeps its link graph in the framework payload, so parsing + /// the raw markup finds only a fraction of the site's sections. + #[serde(default)] + pub(crate) links: Vec, + /// Sitemap `` entries discovered from `robots.txt`, when fetched. + /// + /// Empty unless sitemap discovery ran (root page only). + #[serde(default)] + pub(crate) sitemap_locs: Vec, + pub(crate) warnings: Vec, +} + +/// A same-origin link observed in the hydrated DOM. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedLink { + /// Absolute URL of the link target. + pub(crate) url: String, + /// Whether the anchor sits inside site navigation (`nav`, `header`, + /// `[role="navigation"]`). Nav links are the publisher's own declaration of + /// its taxonomy, so they rank above body links when choosing sections. + pub(crate) in_nav: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::{cli_error, report_error}; + + struct ProgressCollector; + + impl AuditCollector for ProgressCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + if target_url.path() == "/broken" { + return cli_error("simulated page failure"); + } + Ok(CollectedPage { + requested_url: target_url.to_string(), + final_url: target_url.to_string(), + page_title: None, + html: String::new(), + script_tags: Vec::new(), + network_requests: Vec::new(), + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }) + } + } + + fn record_progress(event: CollectionProgress<'_>) -> String { + match event { + CollectionProgress::Launching => "launching".to_string(), + CollectionProgress::Loading { + current, + total, + url, + } => format!( + "loading:{current}/{}:{}", + total.map_or_else(|| "?".to_string(), |total| total.to_string()), + url.path() + ), + CollectionProgress::Planning => "planning".to_string(), + CollectionProgress::Finalizing => "finalizing".to_string(), + } + } + + #[test] + fn default_collect_site_reports_root_planning_and_offset_followups() { + let collector = ProgressCollector; + let root = Url::parse("https://publisher.example/").expect("should parse root URL"); + let news = Url::parse("https://publisher.example/news").expect("should parse news URL"); + let broken = + Url::parse("https://publisher.example/broken").expect("should parse broken URL"); + let mut events = Vec::new(); + let mut outcomes = Vec::new(); + + collector + .collect_site( + &root, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(vec![news.clone(), broken.clone()]), + &mut |url, result| { + outcomes.push((url.path().to_string(), result.is_ok())); + Ok(ControlFlow::Continue) + }, + ) + .expect("should collect site despite one page outcome failing"); + + assert_eq!( + events, + [ + "loading:1/?:/", + "planning", + "loading:2/3:/news", + "loading:3/3:/broken", + ] + ); + assert_eq!( + outcomes, + [ + ("/".to_string(), true), + ("/news".to_string(), true), + ("/broken".to_string(), false) + ] + ); + } + + #[test] + fn default_collect_pages_reports_a_fixed_total() { + let collector = ProgressCollector; + let targets = [ + Url::parse("https://publisher.example/").expect("should parse root URL"), + Url::parse("https://publisher.example/broken").expect("should parse broken URL"), + ]; + let mut events = Vec::new(); + + collector + .collect_pages( + &targets, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(ControlFlow::Continue), + ) + .expect("should deliver failed page as an outcome"); + + assert_eq!(events, ["loading:1/2:/", "loading:2/2:/broken"]); + } + + #[test] + fn default_collection_stops_when_progress_fails() { + let collector = ProgressCollector; + let targets = [Url::parse("https://publisher.example/").expect("should parse root URL")]; + + let error = collector + .collect_pages( + &targets, + &[], + &mut |_| Err(report_error("simulated progress failure")), + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure"); + + assert!(format!("{error:?}").contains("simulated progress failure")); + } +} + +/// A single slot read from the page's live GPT registry. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedGptSlot { + /// The GAM ad-unit path (`slot.getAdUnitPath()`). + pub(crate) gam_unit_path: String, + /// The slot's div element id (`slot.getSlotElementId()`). + pub(crate) div_id: String, + /// Numeric `[width, height]` sizes (`slot.getSizes()`, fluid entries dropped). + pub(crate) sizes: Vec<(u32, u32)>, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedScriptTag { + pub(crate) src: Option, + pub(crate) inline_text: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedRequest { + pub(crate) url: String, + pub(crate) resource_type: Option, +} + +impl CollectedPage { + pub(crate) fn requested_url(&self) -> Result { + Url::parse(&self.requested_url) + } + + pub(crate) fn final_url(&self) -> Result { + Url::parse(&self.final_url) + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs new file mode 100644 index 000000000..76d57cf9e --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -0,0 +1,864 @@ +//! Pure crawl planning: turn discovered links and sitemap entries into the +//! bounded set of pages worth loading in a browser. +//! +//! The goal is deliberately *not* site coverage. Ad slots repeat per site +//! section, and the generated config needs one glob pair per section +//! (`/news` and `/news/*`), so one representative page per section is enough. +//! That keeps the crawl proportional to the publisher's taxonomy (a dozen +//! sections) rather than its catalog (tens of thousands of articles). +//! +//! Two sources feed the plan and each supplies a half the other cannot: +//! +//! - **Navigation links** give section *landing* paths (`/news`), which +//! sitemaps routinely omit, and are the publisher's own taxonomy declaration. +//! - **Sitemap entries** give a real *article* per section (`/news/story-abc`), +//! which is where in-content slots live, and reveal sections hidden behind a +//! navigation overflow menu. + +use std::collections::BTreeMap; + +use url::Url; + +use super::collector::CollectedLink; + +/// Path segments that are never a content section worth sampling. +/// +/// These carry either no ad stack at all or an unrepresentative one, and +/// crawling them spends budget that a real section needs. +const NOISE_SEGMENTS: &[&str] = &[ + "about", + "about-us", + "account", + "author", + "cart", + "contact", + "editorial-policy", + "login", + "logout", + "newsletter", + "page", + "press", + "privacy", + "register", + "search", + "sitemap", + "subscribe", + "terms", +]; + +/// File extensions that are assets rather than pages. +const NON_PAGE_EXTENSIONS: &[&str] = &[ + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".svg", ".ico", ".css", ".js", ".json", + ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", +]; + +/// ISO 639-1 alpha-2 language codes, sorted for binary search. +/// +/// Country codes are deliberately absent: `/us` and `/tv` are section roots on +/// plenty of publishers, and only the language form appears as a URL locale +/// prefix on its own. +const ISO_639_1_CODES: &[&str] = &[ + "aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av", "ay", "az", "ba", "be", "bg", "bh", + "bi", "bm", "bn", "bo", "br", "bs", "ca", "ce", "ch", "co", "cr", "cs", "cu", "cv", "cy", "da", + "de", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff", "fi", "fj", "fo", "fr", + "fy", "ga", "gd", "gl", "gn", "gu", "gv", "ha", "he", "hi", "ho", "hr", "ht", "hu", "hy", "hz", + "ia", "id", "ie", "ig", "ii", "ik", "io", "is", "it", "iu", "ja", "jv", "ka", "kg", "ki", "kj", + "kk", "kl", "km", "kn", "ko", "kr", "ks", "ku", "kv", "kw", "ky", "la", "lb", "lg", "li", "ln", + "lo", "lt", "lu", "lv", "mg", "mh", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "na", "nb", + "nd", "ne", "ng", "nl", "nn", "no", "nr", "nv", "ny", "oc", "oj", "om", "or", "os", "pa", "pi", + "pl", "ps", "pt", "qu", "rm", "rn", "ro", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", + "sl", "sm", "sn", "so", "sq", "sr", "ss", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "ti", + "tk", "tl", "tn", "to", "tr", "ts", "tt", "tw", "ty", "ug", "uk", "ur", "uz", "ve", "vi", "vo", + "wa", "wo", "xh", "yi", "yo", "za", "zh", "zu", +]; + +/// Filenames that name a directory's index document rather than a page of their +/// own, so a link to one is treated as a link to the parent directory. +const DIRECTORY_INDEX_NAMES: &[&str] = &[ + "index.html", + "index.htm", + "index.php", + "default.html", + "default.htm", + "default.php", + "home.html", + "home.htm", + "home.php", +]; + +/// Bounds on how much of a site a single run will load. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct CrawlBudget { + /// Maximum number of sections to sample. + pub(crate) max_sections: usize, + /// Maximum number of pages to load in total, including the root. + pub(crate) max_pages: usize, +} + +impl Default for CrawlBudget { + fn default() -> Self { + Self { + max_sections: 8, + max_pages: 17, + } + } +} + +/// One section selected for sampling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PlannedSection { + /// The path segment at [`CrawlPlan::section_segment`] identifying the section. + pub(super) segment: String, + /// The section landing page, when one was observed. + pub(super) landing: Option, + /// A representative content page inside the section, when one was observed. + pub(super) article: Option, +} + +impl PlannedSection { + /// The pages to load for this section, landing first. + fn targets(&self) -> impl Iterator { + self.landing.iter().chain(self.article.iter()) + } +} + +/// The bounded outcome of planning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct CrawlPlan { + /// Sections selected for sampling, highest confidence first. + pub(super) sections: Vec, + /// Sections found but dropped because the budget was already spent. + pub(super) dropped_sections: Vec, + /// Human-readable notes about how the plan was reached. + pub(super) notes: Vec, + /// Path segment used to distinguish sections in this crawl. + pub(super) section_segment: usize, +} + +impl CrawlPlan { + /// Page URLs to load, in crawl order. The root is *not* included — the + /// caller has already collected it in order to plan at all. + pub(super) fn targets(&self) -> Vec { + self.sections + .iter() + .flat_map(PlannedSection::targets) + .cloned() + .collect() + } +} + +/// Evidence gathered about one candidate section before ranking. +#[derive(Debug, Default)] +struct SectionCandidate { + landing: Option, + article: Option, + in_nav: bool, + in_sitemap: bool, + link_count: usize, +} + +impl SectionCandidate { + /// Confidence ordering: corroborated by both sources beats either alone, + /// and navigation beats a sitemap-only hit because navigation is the + /// publisher's own statement of what its sections are. + fn rank(&self) -> u8 { + match (self.in_nav, self.in_sitemap) { + (true, true) => 3, + (true, false) => 2, + (false, true) => 1, + (false, false) => 0, + } + } +} + +/// Plans the crawl from the root page's links and any sitemap entries. +/// +/// `root` bounds the crawl: every candidate must share its origin, which also +/// stops a hostile or misconfigured `robots.txt` from redirecting the crawl (and +/// the operator's cookies) at an unrelated host. +pub(super) fn plan_crawl( + root: &Url, + links: &[CollectedLink], + sitemap_locs: &[String], + budget: CrawlBudget, +) -> CrawlPlan { + let section_segment = usize::from(root_is_locale_prefix(root)); + let mut candidates: BTreeMap = BTreeMap::new(); + let mut notes = Vec::new(); + + for link in links { + let Some(url) = same_origin_page_url(root, &link.url, section_segment) else { + continue; + }; + let Some(segment) = section_at(&url, section_segment) else { + continue; + }; + let entry = candidates.entry(segment).or_default(); + entry.in_nav |= link.in_nav; + entry.link_count += 1; + record_url(entry, &url, section_segment); + } + + let mut sitemap_pages = 0_usize; + for loc in sitemap_locs { + let Some(url) = same_origin_page_url(root, loc, section_segment) else { + continue; + }; + let Some(segment) = section_at(&url, section_segment) else { + continue; + }; + sitemap_pages += 1; + let entry = candidates.entry(segment).or_default(); + entry.in_sitemap = true; + record_url(entry, &url, section_segment); + } + + if !sitemap_locs.is_empty() { + notes.push(format!( + "sitemap contributed {sitemap_pages} same-origin page(s) across {} section(s)", + candidates.values().filter(|c| c.in_sitemap).count() + )); + } + if links.iter().all(|link| !link.in_nav) && !links.is_empty() { + notes.push( + "no navigation links were found; sections were inferred from body links only" + .to_string(), + ); + } + + // Rank before truncating: confidence first, then how heavily the section is + // linked, then the segment name so runs are reproducible. + let mut ranked: Vec<(String, SectionCandidate)> = candidates.into_iter().collect(); + ranked.sort_by(|(left_segment, left), (right_segment, right)| { + right + .rank() + .cmp(&left.rank()) + .then(right.link_count.cmp(&left.link_count)) + .then(left_segment.cmp(right_segment)) + }); + + let mut sections = Vec::new(); + let mut dropped_sections = Vec::new(); + // The root page is already collected and counts against the page budget. + let mut pages_used = 1_usize; + for (segment, candidate) in ranked { + let planned = PlannedSection { + segment: segment.clone(), + landing: candidate.landing, + article: candidate.article, + }; + let cost = planned.targets().count(); + if cost == 0 { + continue; + } + if sections.len() >= budget.max_sections || pages_used + cost > budget.max_pages { + dropped_sections.push(segment); + continue; + } + pages_used += cost; + sections.push(planned); + } + + if !dropped_sections.is_empty() { + let shown = dropped_sections + .iter() + .take(10) + .cloned() + .collect::>() + .join(", "); + let remainder = dropped_sections.len().saturating_sub(10); + let suffix = if remainder == 0 { + String::new() + } else { + format!(", and {remainder} more") + }; + notes.push(format!( + "budget reached: {} section(s) not sampled ({shown}{suffix}); raise --max-sections/--max-pages to include them", + dropped_sections.len(), + )); + } + + CrawlPlan { + sections, + dropped_sections, + notes, + section_segment, + } +} + +/// Files a URL as the section's landing page or its representative article. +/// +/// The first candidate of each kind wins, so a run is stable given stable input. +fn record_url(entry: &mut SectionCandidate, url: &Url, section_segment: usize) { + if segment_count(url) == section_segment + 1 { + if entry.landing.is_none() { + entry.landing = Some(url.clone()); + } + } else if entry.article.is_none() { + entry.article = Some(url.clone()); + } +} + +/// Parses `raw` against `root` and keeps it only if it is a same-origin page. +/// +/// Rejects other origins, non-HTTP schemes, asset extensions, and paginated or +/// utility paths. Query and fragment are dropped so `/news?page=2` and +/// `/news#top` collapse onto `/news`. +fn same_origin_page_url(root: &Url, raw: &str, section_segment: usize) -> Option { + let mut url = root.join(raw).ok()?; + if !matches!(url.scheme(), "http" | "https") || url.origin() != root.origin() { + return None; + } + url.set_query(None); + url.set_fragment(None); + + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); + if NON_PAGE_EXTENSIONS + .iter() + .any(|extension| path.ends_with(extension)) + { + return None; + } + // A section reachable only through its index document is still that section: + // `/news/index.html` is `/news`. Rejecting the URL outright loses the + // section; dropping the filename keeps it. + if path + .split('/') + .rfind(|part| !part.is_empty()) + .is_some_and(|last| DIRECTORY_INDEX_NAMES.contains(&last)) + { + url.path_segments_mut().ok()?.pop(); + } + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.is_empty() { + return None; + } + if NOISE_SEGMENTS.contains(&segments.get(section_segment).copied().unwrap_or_default()) { + return None; + } + if section_segment > 0 { + let root_path = percent_decode_for_filtering(root.path()).to_ascii_lowercase(); + let root_segments: Vec<&str> = root_path + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + if !segments.starts_with(&root_segments) { + return None; + } + } + // `/news/page/2` is the same inventory as `/news`, so it is not a second + // sample worth spending a page load on. + if segments.contains(&"page") { + return None; + } + Some(url) +} + +/// The non-empty path segment at `index`, percent-decoded and lowercased. +fn section_at(url: &Url, index: usize) -> Option { + percent_decode_for_filtering(url.path()) + .split('/') + .filter(|part| !part.is_empty()) + .nth(index) + .map(str::to_ascii_lowercase) +} + +/// Whether the requested root is nothing but a locale prefix, which puts +/// sections one segment deeper than usual. +fn root_is_locale_prefix(root: &Url) -> bool { + let segments: Vec<&str> = root + .path() + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + matches!(segments.as_slice(), [locale] if is_locale_segment(locale)) +} + +/// Whether a root's single path segment is a locale prefix (`/en`, `/en-gb`) +/// rather than a content section. +/// +/// The language half must be a real ISO 639-1 code. Accepting any two letters +/// read ordinary section roots — `/tv`, `/ai`, `/us` — as locales, which shifts +/// `section_segment` by one: article slugs then become "sections" and the +/// containment check below discards the root's real siblings. +fn is_locale_segment(segment: &str) -> bool { + let segment = segment.to_ascii_lowercase(); + match segment.as_bytes() { + [_, _] => is_language_code(&segment), + [_, _, b'-', c, d] => { + is_language_code(&segment[..2]) && c.is_ascii_alphabetic() && d.is_ascii_alphabetic() + } + _ => false, + } +} + +/// Whether `segment` is an ISO 639-1 alpha-2 language code. +fn is_language_code(segment: &str) -> bool { + ISO_639_1_CODES.binary_search(&segment).is_ok() +} + +/// Decodes percent escapes solely for normalized path classification. +fn percent_decode_for_filtering(path: &str) -> String { + let bytes = path.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' + && index + 2 < bytes.len() + && let (Some(high), Some(low)) = + (hex_value(bytes[index + 1]), hex_value(bytes[index + 2])) + { + decoded.push((high << 4) | low); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + String::from_utf8_lossy(&decoded).into_owned() +} + +/// Converts one ASCII hexadecimal digit to its numeric value. +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +/// Count of non-empty path segments. +fn segment_count(url: &Url) -> usize { + url.path() + .split('/') + .filter(|part| !part.is_empty()) + .count() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root() -> Url { + Url::parse("https://publisher.example/").expect("valid root") + } + + fn nav(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + } + } + + fn body(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: false, + } + } + + fn segments(plan: &CrawlPlan) -> Vec<&str> { + plan.sections + .iter() + .map(|section| section.segment.as_str()) + .collect() + } + + #[test] + fn pairs_a_landing_page_with_an_article_from_the_sitemap() { + let plan = plan_crawl( + &root(), + &[nav("/news")], + &["https://publisher.example/news/story-abc".to_string()], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); + let section = &plan.sections[0]; + assert_eq!( + section.landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news") + ); + assert_eq!( + section.article.as_ref().map(Url::as_str), + Some("https://publisher.example/news/story-abc") + ); + assert_eq!(plan.targets().len(), 2, "should load landing then article"); + } + + #[test] + fn cross_origin_candidates_are_dropped() { + // Guards both the sitemap (a `Sitemap:` directive can point anywhere) + // and links: the crawl carries operator cookies, so it must not leave + // the requested origin. + let plan = plan_crawl( + &root(), + &[CollectedLink { + url: "https://tracker.example/news".to_string(), + in_nav: true, + }], + &["https://other.example/deals/x".to_string()], + CrawlBudget::default(), + ); + + assert!( + plan.sections.is_empty(), + "no off-origin section should survive, got {:?}", + segments(&plan) + ); + } + + #[test] + fn utility_paths_and_assets_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/about-us"), + nav("/search"), + nav("/editorial-policy"), + nav("/logo.png"), + nav("/feed.xml"), + nav("/news/page/2"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the real content section should remain" + ); + } + + #[test] + fn query_and_fragment_collapse_onto_one_landing_page() { + let plan = plan_crawl( + &root(), + &[nav("/news?utm_source=x"), nav("/news#top"), nav("/news")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); + assert_eq!( + plan.sections[0].landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news"), + "tracking query and fragment should be stripped" + ); + } + + #[test] + fn nav_and_sitemap_corroboration_outranks_either_alone() { + let plan = plan_crawl( + &root(), + &[nav("/features"), body("/reviews")], + &[ + "https://publisher.example/features/story".to_string(), + "https://publisher.example/deals/x".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan)[0], + "features", + "nav + sitemap should rank first, got {:?}", + segments(&plan) + ); + } + + #[test] + fn budget_truncates_and_reports_what_was_dropped() { + let links: Vec = ["a", "b", "c", "d"] + .iter() + .map(|segment| nav(&format!("/{segment}"))) + .collect(); + + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 2, + max_pages: 17, + }, + ); + + assert_eq!(plan.sections.len(), 2, "section cap should be honoured"); + assert_eq!( + plan.dropped_sections.len(), + 2, + "sections past the budget should be reported as dropped" + ); + assert!( + plan.notes + .iter() + .any(|note| note.contains("budget reached")), + "dropping sections must be reported, not silent: {:?}", + plan.notes + ); + } + + #[test] + fn page_budget_counts_the_already_collected_root() { + // max_pages = 3 leaves room for exactly one landing+article pair on top + // of the root page the caller already loaded. + let plan = plan_crawl( + &root(), + &[nav("/news"), nav("/deals")], + &[ + "https://publisher.example/news/a".to_string(), + "https://publisher.example/deals/b".to_string(), + ], + CrawlBudget { + max_sections: 8, + max_pages: 3, + }, + ); + + assert_eq!( + plan.targets().len(), + 2, + "root + 2 pages fills max_pages = 3" + ); + assert_eq!( + plan.dropped_sections.len(), + 1, + "the section past the budget should be reported as dropped" + ); + } + + #[test] + fn body_only_links_still_yield_sections_with_a_note() { + let plan = plan_crawl( + &root(), + &[body("/news"), body("/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["deals", "news"]); + assert!( + plan.notes + .iter() + .any(|note| note.contains("no navigation links")), + "a nav-less page should say so: {:?}", + plan.notes + ); + } + + #[test] + fn empty_input_plans_nothing_rather_than_panicking() { + let plan = plan_crawl(&root(), &[], &[], CrawlBudget::default()); + + assert!( + plan.sections.is_empty(), + "no input means no sections to sample" + ); + assert!( + plan.targets().is_empty(), + "no sections means nothing to load" + ); + } + + #[test] + fn locale_root_plans_sections_from_the_second_segment() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/en/deals")], + &[ + "https://publisher.example/en/news/story".to_string(), + "https://publisher.example/en/deals/item".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "a locale root puts sections one segment deeper" + ); + assert_eq!(segments(&plan), ["deals", "news"]); + assert_eq!( + plan.targets().len(), + 4, + "each section contributes a landing page and an article" + ); + } + + #[test] + fn encoded_noise_and_page_extensions_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/%70rivacy"), + nav("/index.html"), + nav("/archive.htm"), + nav("/story.php"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["archive.htm", "news", "story.php"], + "only directory-index documents should be excluded by extension" + ); + } + + #[test] + fn a_two_letter_section_root_is_not_read_as_a_locale() { + // `/tv`, `/ai` and `/us` are section roots, not locales. Reading them as + // locales moves the section segment to 1, so article slugs become + // "sections" and the root's real siblings are discarded. + for root_path in ["/tv", "/ai", "/us"] { + let section_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[ + nav(&format!("{root_path}/story-one")), + nav(&format!("{root_path}/story-two")), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "`{root_path}` should be a section root, not a locale prefix" + ); + assert_eq!( + segments(&plan), + [root_path.trim_start_matches('/')], + "articles below `{root_path}` should stay one section" + ); + } + } + + #[test] + fn a_real_language_prefix_is_still_read_as_a_locale() { + for root_path in ["/en", "/fr", "/pt-br"] { + let locale_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav(&format!("{root_path}/news"))], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "`{root_path}` is a locale prefix, so sections start one segment in" + ); + assert_eq!(segments(&plan), ["news"]); + } + } + + #[test] + fn a_section_reachable_only_by_its_index_document_collapses_to_the_parent() { + let plan = plan_crawl( + &root(), + &[ + nav("/news/index.html"), + nav("/deals/index.php"), + nav("/sport/home.htm"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["deals", "news", "sport"], + "an index document names its section rather than disqualifying it" + ); + let targets: Vec = plan + .targets() + .iter() + .map(|url| url.path().to_string()) + .collect(); + assert_eq!( + targets, + ["/deals", "/news", "/sport"], + "the parent directory is what gets loaded" + ); + } + + #[test] + fn locale_root_rejects_candidates_outside_its_path_prefix() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/fr/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "a locale-root crawl must not mix another locale on the same origin" + ); + } + + #[test] + fn a_section_root_does_not_treat_article_slugs_as_sections() { + let section_root = Url::parse("https://publisher.example/news").expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[nav("/news/story-one"), nav("/news/story-two")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "a generic one-segment root is a section, not necessarily a locale" + ); + assert_eq!( + segments(&plan), + ["news"], + "articles below a section root should remain one section" + ); + } + + #[test] + fn dropped_section_note_is_capped() { + let links: Vec<_> = (0..15) + .map(|index| nav(&format!("/section-{index:02}"))) + .collect(); + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 0, + max_pages: 1, + }, + ); + let note = plan + .notes + .iter() + .find(|note| note.contains("budget reached")) + .expect("should report dropped sections"); + + assert!(note.contains("and 5 more")); + assert!(!note.contains("section-14")); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs new file mode 100644 index 000000000..f7c0e71df --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -0,0 +1,802 @@ +//! Cross-page slot evidence: what each slot looked like on every page it was +//! observed on. +//! +//! A single page cannot distinguish a literal ad-unit path from a templated one, +//! so inference needs the *set* of observations per slot rather than one +//! snapshot. This module accumulates that set and is deliberately the only place +//! that reconciles a slot seen more than once: +//! +//! - **Formats union.** A size that appears only on article pages (a 300x600 +//! rail, say) must survive alongside the homepage's sizes. Taking the first +//! page's formats would silently narrow the slot. +//! - **Unit paths are kept, not collapsed.** Divergence across pages is the +//! signal inference reads; discarding it is what makes templating impossible. +//! - **Network ids must agree.** Two different GAM networks in one crawl means +//! the pages are not one property, and writing either one would be a guess. +//! +//! Slots are keyed on the *normalized div stem* produced by +//! [`discover_gpt_slots`](super::gpt_slots::discover_gpt_slots), because raw GPT +//! div ids carry per-render framework hashes and would otherwise look like a new +//! slot on every page. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::gpt_slots::DiscoveredSlots; +use crate::error::{CliResult, cli_error}; + +/// One observation of a slot on one page. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct EvidenceRow { + /// The page path the slot was observed on, normalized (leading `/`, no + /// query or fragment). + pub(super) path: String, + /// The literal GAM ad-unit path the live page used for this slot. + pub(super) unit_path: String, +} + +/// Everything observed about one slot across the crawl. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SlotEvidence { + /// Config slot id derived from the div stem. + pub(super) id: String, + /// Normalized div stem, used as the runtime `div_id` prefix. + pub(super) div_id: String, + /// Union of every pixel size observed for this slot, smallest first. + pub(super) formats: BTreeSet<(u32, u32)>, + /// Whether any page carrying this slot showed header-bidding signals. + pub(super) has_prebid: bool, + /// Distinct `(path, unit_path)` observations, in a stable order. + pub(super) rows: BTreeSet, +} + +impl SlotEvidence { + /// The distinct literal unit paths observed for this slot. + pub(super) fn unit_paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.unit_path.as_str()).collect() + } + + /// The distinct page paths this slot was observed on. + pub(super) fn paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.path.as_str()).collect() + } +} + +/// Slots grouped by the shape that would make them one placement: an identical +/// ad-unit path and an identical format set. +type SlotsByShape<'a> = BTreeMap<(String, Vec<(u32, u32)>), Vec<&'a SlotEvidence>>; + +/// Several observed slots that are really one placement under volatile div ids. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct FragmentGroup { + /// The volatile div ids observed, in evidence order. + pub(super) div_ids: Vec, + /// The ad-unit path every fragment shared. + pub(super) unit_path: String, + /// The stable prefix the ids share, when they share a useful one. + /// + /// Offered to the operator as a starting point only. It is deliberately not + /// written as a `div_id`: the shared prefix reaches only as far as the + /// *observed* tokens happen to agree, so it would keep matching this crawl's + /// ids and stop matching the next render's. + pub(super) suggested_prefix: Option, +} + +/// Whether no two slots were ever seen on the same page. +fn pages_are_disjoint(slots: &[&SlotEvidence]) -> bool { + for (index, slot) in slots.iter().enumerate() { + let pages = slot.paths(); + if slots[index + 1..] + .iter() + .any(|other| other.paths().intersection(&pages).next().is_some()) + { + return false; + } + } + true +} + +/// The longest prefix the div ids share, trimmed back to a separator. +/// +/// Trimming matters: the raw common prefix usually ends mid-token (the leading +/// digits of a timestamp two fragments happen to share), which is worse than +/// useless as a suggestion. Cutting at the last `-` or `_` yields the part a +/// human would recognise as the placement's name. +fn shared_div_prefix(slots: &[&SlotEvidence]) -> Option { + let mut prefix: &str = slots.first()?.div_id.as_str(); + for slot in &slots[1..] { + let mut shared_end = 0; + for ((byte_index, left), right) in prefix.char_indices().zip(slot.div_id.chars()) { + if left != right { + break; + } + shared_end = byte_index + left.len_utf8(); + } + prefix = &prefix[..shared_end]; + } + let trimmed = prefix.trim_end_matches(|ch: char| ch != '-' && ch != '_'); + let candidate = trimmed.trim_end_matches(['-', '_']); + (!candidate.is_empty()).then(|| candidate.to_string()) +} + +/// Slot evidence accumulated across every collected page. +#[derive(Debug, Clone, Default)] +pub(super) struct EvidenceTable { + slots: BTreeMap, + /// Div stems in first-seen order, so generated config keeps crawl order + /// rather than alphabetical order. + order: Vec, + network_ids: BTreeSet, + /// Every page path folded in, including those that yielded no slots. + pages: BTreeSet, + /// Page paths that produced no slot evidence at all. + empty_pages: BTreeSet, + /// Page paths that produced slot evidence on at least one selected profile. + non_empty_pages: BTreeSet, + /// Div stems any page refused as ambiguous, unioned across the crawl. + /// + /// The verdict has to outlive the page that reached it. Article pages carry + /// several in-content units and refuse the shared prefix; a landing page + /// carries one and would otherwise contribute it as a usable slot, so the + /// written config would depend on which pages the crawl happened to sample. + ambiguous_stems: BTreeSet, + /// Normalized div IDs refused from generation but observed live. + refused_div_ids: BTreeSet, +} + +impl EvidenceTable { + /// Folds one page's discovered slots into the table. + /// + /// `path` is the page's normalized request path; it is what page patterns + /// and `{section}` derivation are computed from later, so it must be the + /// post-redirect path actually audited. + pub(super) fn fold_page(&mut self, path: &str, discovered: &DiscoveredSlots) { + self.pages.insert(path.to_string()); + if let Some(network_id) = &discovered.gam_network_id { + self.network_ids.insert(network_id.clone()); + } + if !discovered.had_slot_evidence { + if !self.non_empty_pages.contains(path) { + self.empty_pages.insert(path.to_string()); + } + return; + } + self.non_empty_pages.insert(path.to_string()); + self.empty_pages.remove(path); + self.ambiguous_stems + .extend(discovered.ambiguous_stems.iter().cloned()); + self.refused_div_ids + .extend(discovered.refused_div_ids.iter().cloned()); + + for slot in &discovered.slots { + let entry = self.slots.entry(slot.div_id.clone()).or_insert_with(|| { + self.order.push(slot.div_id.clone()); + SlotEvidence { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + formats: BTreeSet::new(), + has_prebid: false, + rows: BTreeSet::new(), + } + }); + // Union rather than replace: a size seen only on one page type is + // still a size this slot serves. + entry.formats.extend(slot.formats.iter().copied()); + entry.has_prebid |= slot.has_prebid; + entry.rows.insert(EvidenceRow { + path: path.to_string(), + unit_path: slot.gam_unit_path.clone(), + }); + } + } + + /// Slots in first-seen order, excluding stems any page refused as ambiguous. + pub(super) fn slots(&self) -> impl Iterator { + self.order + .iter() + .filter(|div_id| !self.ambiguous_stems.contains(*div_id)) + .filter_map(|div_id| self.slots.get(div_id)) + } + + /// Every normalized div ID observed, including all refused evidence. + pub(super) fn observed_div_ids(&self) -> impl Iterator { + self.order + .iter() + .map(String::as_str) + .chain(self.ambiguous_stems.iter().map(String::as_str)) + .chain(self.refused_div_ids.iter().map(String::as_str)) + } + + /// Normalized div IDs observed as concrete live elements. + /// + /// Unlike [`EvidenceTable::observed_div_ids`], this excludes identifiers + /// that exist only as refused ambiguity or volatility evidence. Prefix + /// routing must not treat those inferred stems as literal DOM elements. + pub(super) fn observed_literals(&self) -> impl Iterator { + self.order + .iter() + .filter(|div_id| !self.ambiguous_stems.contains(*div_id)) + .filter(|div_id| { + !self.refused_div_ids.contains(*div_id) || self.slots.contains_key(*div_id) + }) + .map(String::as_str) + } + + /// Number of usable distinct slots observed. + pub(super) fn slot_count(&self) -> usize { + self.slots().count() + } + + /// Every page path folded in, whether or not it yielded slots. + pub(super) fn pages(&self) -> &BTreeSet { + &self.pages + } + + /// Page paths that produced no slot evidence. + /// + /// A high proportion of these is the signature of a bot challenge serving + /// interstitials instead of the real site, which is worth refusing to write + /// from rather than persisting a half-empty config. + pub(super) fn empty_pages(&self) -> &BTreeSet { + &self.empty_pages + } + + /// Whether any slot was observed at all, ambiguous ones included. + /// + /// Deliberately not `slot_count() == 0`: a crawl that saw only ambiguous + /// placements did observe an ad stack, and the caller distinguishes "this + /// page has no slots" from "every slot found was refused". + pub(super) fn is_empty(&self) -> bool { + self.slots.is_empty() + } + + /// Groups of slots that are one slot wearing a different div id per page. + /// + /// Some ad stacks build div ids from a per-render token — a timestamp, a + /// framework id — so the same placement arrives under a new key on every + /// page. Written verbatim those ids never match at runtime, and the + /// fragmentation also starves template inference, which needs to see one + /// slot more than once. + /// + /// Detection is by evidence rather than by guessing at token shapes, because + /// each stack invents its own. Candidates share an identical ad-unit path and + /// identical formats; what separates a fragmented slot from two legitimate + /// siblings on the same unit is **co-occurrence**. Real siblings appear + /// together on a page; fragments of one slot never do, because each page + /// produces exactly one of them. + pub(super) fn fragmented_slots(&self) -> Vec { + let mut by_shape: SlotsByShape<'_> = BTreeMap::new(); + for slot in self.slots() { + // Only slots pinned to exactly one unit path can be compared this + // way; a slot whose unit varies is inference's problem, not this one. + let units = slot.unit_paths(); + if units.len() != 1 { + continue; + } + let unit = (*units.iter().next().expect("should have one unit path")).to_string(); + let formats: Vec<(u32, u32)> = slot.formats.iter().copied().collect(); + by_shape.entry((unit, formats)).or_default().push(slot); + } + + by_shape + .into_iter() + .filter(|(_, slots)| slots.len() > 1) + .filter(|(_, slots)| pages_are_disjoint(slots)) + .filter_map(|((unit_path, _), slots)| { + let suggested_prefix = shared_div_prefix(&slots); + (suggested_prefix.is_some() || slots.len() >= 3).then(|| FragmentGroup { + div_ids: slots.iter().map(|slot| slot.div_id.clone()).collect(), + unit_path, + suggested_prefix, + }) + }) + .collect() + } + + /// The single GAM network id observed across the crawl. + /// + /// # Errors + /// + /// Returns an error when pages disagreed. Two networks in one crawl means + /// the pages are not one property (a syndicated subdomain, a child network, + /// an off-origin redirect that slipped through), and picking either would be + /// a guess that silently bids against the wrong inventory. + pub(super) fn network_id(&self) -> CliResult> { + let mut found = self.network_ids.iter(); + let Some(first) = found.next() else { + return Ok(None); + }; + if self.network_ids.len() > 1 { + let all: Vec<&str> = self.network_ids.iter().map(String::as_str).collect(); + return cli_error(format!( + "the crawled pages reported more than one GAM network id ({}); \ + they do not appear to be one property, so no network id can be \ + chosen safely. Audit a single property, or pass explicit URLs", + all.join(", ") + )); + } + Ok(Some(first.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// One live slot as `(unit path, div id, sizes)`. + type SlotFixture<'a> = (&'a str, &'a str, &'a [(u32, u32)]); + + fn page(slots: &[SlotFixture<'_>], has_prebid: bool) -> DiscoveredSlots { + let registry: Vec = slots + .iter() + .map(|(unit_path, div_id, sizes)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: sizes.to_vec(), + }) + .collect(); + discover_gpt_slots(®istry, &[], has_prebid) + } + + #[test] + fn formats_union_across_pages_instead_of_first_seen_winning() { + // The 300x600 rail only ever renders on article pages. Keeping the + // homepage's format list alone would silently narrow the slot. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-rail", &[(300, 250)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-rail", &[(300, 600)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.formats.iter().copied().collect::>(), + [(300, 250), (300, 600)], + "both pages' sizes should survive" + ); + assert_eq!(table.slot_count(), 1, "one div stem is one slot"); + } + + #[test] + fn divergent_unit_paths_are_preserved_as_separate_rows() { + // This divergence is the entire signal template inference reads. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.unit_paths().into_iter().collect::>(), + ["/123/site/home", "/123/site/news"], + "both observed unit paths must be retained" + ); + assert_eq!( + slot.paths().into_iter().collect::>(), + ["/", "/news/story"] + ); + } + + #[test] + fn repeated_identical_observations_collapse() { + let mut table = EvidenceTable::default(); + let observed = page(&[("/123/site/home", "ad-header", &[(728, 90)])], false); + table.fold_page("/", &observed); + table.fold_page("/", &observed); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!(slot.rows.len(), 1, "the same page twice is one observation"); + } + + #[test] + fn prebid_is_sticky_once_any_page_shows_it() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], true), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert!( + slot.has_prebid, + "a slot proven to run prebid on any page runs prebid" + ); + } + + #[test] + fn slots_keep_first_seen_order_not_alphabetical_order() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page( + &[ + ("/123/site/home", "zeta-slot", &[(728, 90)]), + ("/123/site/home", "alpha-slot", &[(300, 250)]), + ], + false, + ), + ); + + let ids: Vec<&str> = table.slots().map(|slot| slot.div_id.as_str()).collect(); + assert_eq!( + ids, + ["zeta-slot", "alpha-slot"], + "generated config should follow crawl order" + ); + } + + #[test] + fn refused_only_div_ids_are_observed_but_not_literals() { + let mut discovered = page(&[("/123/site/home", "ad-x-stable", &[(300, 250)])], false); + discovered.refused_div_ids.insert("ad-x".to_string()); + let mut table = EvidenceTable::default(); + table.fold_page("/", &discovered); + + assert_eq!( + table.observed_div_ids().collect::>(), + ["ad-x-stable", "ad-x"], + "the staleness view should retain refused evidence" + ); + assert_eq!( + table.observed_literals().collect::>(), + ["ad-x-stable"], + "prefix routing should use only concrete live-element evidence" + ); + } + + #[test] + fn later_refusal_preserves_a_written_literal() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-x", &[(300, 250)])], false), + ); + let mut refused = DiscoveredSlots { + had_slot_evidence: true, + ..DiscoveredSlots::default() + }; + refused.refused_div_ids.insert("ad-x".to_string()); + table.fold_page("/news", &refused); + + assert_eq!( + table.observed_literals().collect::>(), + ["ad-x"], + "should retain a concrete slot even when another page refuses its stem" + ); + assert_eq!( + table.slot_count(), + 1, + "should still write the concrete slot" + ); + assert_eq!( + table.observed_div_ids().collect::>(), + BTreeSet::from(["ad-x"]), + "the refused stem should remain available to staleness accounting" + ); + } + + #[test] + fn one_placement_under_per_render_div_ids_is_detected() { + // Each page yields a new key for the same placement: same unit, same + // formats, never co-occurring. The tokens here deliberately do *not* + // match the digit-led shape `discover_gpt_slots` refuses on sight, so + // this exercises the evidence-based detector that catches the stacks + // whose token shape cannot be recognized from one observation. + let mut table = EvidenceTable::default(); + for (path, div) in [ + ("/features/a", "ex_slot_ce6Bj0uc8sL0aa_overlay_1"), + ("/news/b", "ex_slot_aoYmv4RQyN3nbb_overlay_1"), + ("/deals/c", "ex_slot_mYPDB3tz8cpBcc_overlay_1"), + ] { + table.fold_page( + path, + &page(&[("/99/site_Overlay", div, &[(300, 250)])], false), + ); + } + + let groups = table.fragmented_slots(); + + assert_eq!(groups.len(), 1, "the three fragments should form one group"); + assert_eq!(groups[0].div_ids.len(), 3); + assert_eq!(groups[0].unit_path, "/99/site_Overlay"); + assert_eq!( + groups[0].suggested_prefix.as_deref(), + Some("ex_slot"), + "the suggestion should be trimmed back off the volatile token" + ); + } + + #[test] + fn an_ambiguous_stem_stays_refused_on_every_page() { + // The article page carries two in-content units and refuses the shared + // prefix; the landing page carries one. Folding the landing page must + // not resurrect a prefix that cannot resolve to one element site-wide. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ( + "/123/site/news", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + ( + "/123/site/news", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ], + false, + ), + ); + table.fold_page( + "/", + &page( + &[( + "/123/site/home", + "ad-in_content-1c0de08e5a2f4d6f9b3a7e5c8d1f2a4b-in_content-0", + &[(300, 250)], + )], + false, + ), + ); + + assert_eq!( + table.slots().count(), + 0, + "a stem refused on one page must stay refused, got {:?}", + table.slots().map(|slot| &slot.div_id).collect::>() + ); + assert_eq!( + table.slot_count(), + 0, + "the count should match what is written" + ); + assert!( + !table.is_empty(), + "the crawl did observe an ad stack, so this is not an empty result" + ); + assert!( + table.observed_literals().next().is_none(), + "a globally ambiguous stem must not remain a literal-routing candidate" + ); + } + + #[test] + fn genuine_siblings_on_one_unit_are_not_treated_as_fragments() { + // Two real in-content positions can share a unit path and formats. What + // distinguishes them from fragments is that they appear *together* on a + // page, so refusing to write them would lose real inventory. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ("/99/site/news", "ad-in_content-1", &[(300, 250)]), + ("/99/site/news", "ad-in_content-2", &[(300, 250)]), + ], + false, + ), + ); + + assert!( + table.fragmented_slots().is_empty(), + "co-occurring slots are siblings, not fragments" + ); + } + + #[test] + fn slots_differing_in_formats_are_not_fragments() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "slot-aaaa", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "slot-bbbb", &[(728, 90)])], false), + ); + + assert!( + table.fragmented_slots().is_empty(), + "a differing format set means these are different placements" + ); + } + + #[test] + fn a_slot_seen_alone_is_never_a_fragment() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "only-slot", &[(300, 250)])], false), + ); + + assert!(table.fragmented_slots().is_empty()); + } + + #[test] + fn fragments_with_no_shared_prefix_report_none() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "alpha-1111", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "beta-2222", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + + assert!( + groups.is_empty(), + "two unrelated placements are too ambiguous to classify as fragments" + ); + } + + #[test] + fn three_disjoint_same_shape_ids_are_fragment_evidence_without_a_prefix() { + let mut table = EvidenceTable::default(); + for (path, div_id) in [("/a", "alpha"), ("/b", "bravo"), ("/c", "charlie")] { + table.fold_page(path, &page(&[("/99/site/x", div_id, &[(300, 250)])], false)); + } + + let groups = table.fragmented_slots(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].suggested_prefix, None); + } + + #[test] + fn unicode_shared_prefix_uses_a_utf8_boundary() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "ünicode-ad-a", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "ünicode-ad-b", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].suggested_prefix.as_deref(), Some("ünicode-ad")); + } + + #[test] + fn a_later_non_empty_profile_clears_the_empty_page_marker() { + let mut table = EvidenceTable::default(); + table.fold_page("/news", &page(&[], false)); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + + assert!(table.empty_pages().is_empty()); + } + + #[test] + fn a_later_empty_profile_does_not_re_mark_a_non_empty_page() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + table.fold_page("/news", &page(&[], false)); + + assert!( + table.empty_pages().is_empty(), + "emptiness is a page-level fact across all selected profiles" + ); + } + + #[test] + fn conflicting_network_ids_are_a_hard_error() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/111/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/222/site/news", "ad-header", &[(728, 90)])], false), + ); + + let error = table + .network_id() + .expect_err("two networks in one crawl should not resolve"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("111") && rendered.contains("222"), + "the error should name both observed ids, got {rendered}" + ); + } + + #[test] + fn agreeing_network_ids_resolve_to_one_value() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + assert_eq!( + table.network_id().expect("agreeing ids should resolve"), + Some("123".to_string()) + ); + } + + #[test] + fn pages_without_slots_are_recorded_for_challenge_detection() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page("/blocked", &page(&[], false)); + + assert_eq!( + table + .empty_pages() + .iter() + .map(String::as_str) + .collect::>(), + ["/blocked"], + "a slot-less page must be visible to the caller, not silently dropped" + ); + assert_eq!( + table.pages().len(), + 2, + "every folded page should be counted" + ); + } + + #[test] + fn collision_only_page_is_not_classified_as_empty() { + let discovered = page( + &[ + ("/123/site/home", "ad-x-aaaaaaaaaaaaaaaa-0", &[(300, 250)]), + ("/123/site/home", "ad-x-bbbbbbbbbbbbbbbb-1", &[(300, 250)]), + ], + false, + ); + let mut table = EvidenceTable::default(); + + table.fold_page("/collision-only", &discovered); + + assert!(discovered.had_slot_evidence); + assert!(discovered.slots.is_empty()); + assert!( + table.empty_pages().is_empty(), + "intentionally omitted GPT evidence must not look like a bot challenge" + ); + } + + #[test] + fn empty_table_resolves_no_network_id_rather_than_erroring() { + let table = EvidenceTable::default(); + + assert!(table.is_empty()); + assert_eq!(table.network_id().expect("empty is not a conflict"), None); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs new file mode 100644 index 000000000..746cb6f74 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -0,0 +1,1588 @@ +//! Reconstructs `[creative_opportunities]` slots from a live page's GPT state. +//! +//! Two complementary sources feed the reconstruction: +//! +//! 1. The **live GPT registry** (`googletag.pubads().getSlots()`) is the primary +//! source. It exposes each defined slot's ad-unit path, div id, and sizes +//! directly, and is populated at `defineSlot` time — so it captures slots even +//! when the ad request never fires (consent-gated stacks, iframe-issued +//! requests). It carries no per-slot header-bidding signal, so Prebid is +//! inferred from page-level detection. +//! 2. Captured **`gampad/ads` requests** are a fallback for any div the registry +//! did not report. Each request URL encodes the ad-unit path (`iu_parts`), div +//! id (`dids`), sizes (`prev_iu_szs`), and targeting (`prev_scp`, which does +//! carry a per-slot Prebid signal). +//! +//! Neither source executes the page's ad-stack logic ourselves; both read state +//! the page's own GPT/Prebid setup produced. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::LazyLock; + +use regex::Regex; +use trusted_server_core::creative_opportunities::validate_slot_id; +use url::Url; + +use crate::commands::audit::generate::collector::{CollectedGptSlot, CollectedRequest}; + +/// A hyphen-delimited hex hash *segment* (16+ hex chars bounded by `-` or end), +/// e.g. the UUID GPT embeds in `ad-in_content--in_content-0`. Marks the +/// start of ephemeral div-id noise, like the React `_R_` hash. The trailing +/// boundary avoids truncating a legit token that merely starts with hex-like +/// characters (only `start()` of the match is used). +static HEX_HASH_SEGMENT: LazyLock = + LazyLock::new(|| Regex::new(r"-[0-9a-f]{16,}(?:-|$)").expect("should compile hex hash regex")); +static UUID_SEGMENT: LazyLock = LazyLock::new(|| { + Regex::new(r"-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?:-|$)") + .expect("should compile UUID regex") +}); + +/// Matches a React `useId` token, which changes on every render. +/// +/// React emits these in both cases — `_R_3f_` from a server render and `_r_0_` +/// from a client one — so matching only the uppercase form leaves the lowercase +/// variant in the stem. That is not merely untidy: the suffix differs per +/// render, so one logical slot fragments into a new key on every page, which +/// both breaks runtime div matching and starves template inference of the +/// repeated observations it needs. +/// +/// The uppercase form is distinctive enough to match bare, and its hash is +/// included so the match spans the whole ephemeral token — [`normalize_div_stem`] +/// only reads the match *start*, but [`ephemeral_marker_residue`] excises the +/// match, and a residue that still carried the hash would make two renders of one +/// element look like two elements. The lowercase form is anchored (`_r_`, a short +/// alphanumeric run, `_`) so an ordinary id that merely contains `_r_` keeps its +/// full stem. +static REACT_USE_ID: LazyLock = LazyLock::new(|| { + Regex::new(r"_R_[0-9a-z]*_?|_r_[0-9a-z]{1,8}_").expect("should compile react id regex") +}); + +/// Hosts that serve GPT `gampad/ads` requests. +const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; + +/// Common GPT div-id prefix stripped when deriving a slot id. +const GPT_DIV_PREFIX: &str = "div-gpt-ad-"; + +/// Minimum width/height for a format to be treated as a real creative size. +/// +/// GPT encodes fluid/native aspect-ratio markers (e.g. `4x1`, `8x1`) alongside +/// pixel sizes in `prev_iu_szs`; those are not banner dimensions, so they are +/// dropped from the drafted `formats`. +const MIN_FORMAT_DIMENSION: u32 = 50; + +/// A slot reconstructed from a single GPT ad request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DiscoveredSlot { + /// Slot id derived from the div id (GPT prefix stripped). + pub(crate) id: String, + /// The HTML div id that holds the creative. + pub(crate) div_id: String, + /// The full GAM ad-unit path (e.g. `/123/desktop/homepage/leaderboard`). + pub(crate) gam_unit_path: String, + /// Candidate creative sizes as `(width, height)` pixel pairs. + pub(crate) formats: Vec<(u32, u32)>, + /// Whether the slot's targeting shows Prebid/header-bidding signals. + pub(crate) has_prebid: bool, +} + +/// The result of scanning captured requests for GPT slots. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct DiscoveredSlots { + /// GAM network id shared by the discovered slots, if any were found. + pub(crate) gam_network_id: Option, + /// Whether the page exposed any otherwise usable slot evidence, including + /// ambiguous placements that were intentionally omitted from `slots`. + pub(crate) had_slot_evidence: bool, + /// The reconstructed slots, deduplicated by div id in first-seen order. + pub(crate) slots: Vec, + /// Div stems refused because several live elements normalized onto them. + /// + /// Carried separately from `slots` because the verdict is a property of the + /// *site*, not of this page: another page that happens to render only one + /// member of the group must not resurrect the ambiguous prefix. + pub(crate) ambiguous_stems: BTreeSet, + /// Normalized div IDs refused from generation but still observed live. + pub(crate) refused_div_ids: BTreeSet, + /// Diagnostics for placements whose normalized stable stems collided. + pub(crate) warnings: Vec, +} + +/// Reconstructs GPT slots from the page's live registry and ad requests. +/// +/// The live registry (`googletag.pubads().getSlots()`) is the primary source: it +/// carries the authoritative path/div/size for every defined slot and is present +/// even when the ad request never fires. Captured `gampad/ads` requests are a +/// fallback for any div the registry did not report, and also supply per-slot +/// Prebid signals. Slots are deduplicated by div id in first-seen order. +/// +/// `page_has_prebid` marks registry slots as Prebid-enabled when the page as a +/// whole was detected running Prebid (the registry alone carries no such signal). +pub(crate) fn discover_gpt_slots( + registry: &[CollectedGptSlot], + requests: &[CollectedRequest], + page_has_prebid: bool, +) -> DiscoveredSlots { + let mut slots = Vec::new(); + let mut warnings = Vec::new(); + let mut ambiguous_stems = BTreeSet::new(); + let mut refused_div_ids = BTreeSet::new(); + let mut gam_network_id = None; + let mut had_slot_evidence = false; + let mut registry_residues: BTreeMap> = BTreeMap::new(); + // Stems refused outright, so the request fallback cannot re-add them. Kept + // apart from `registry_residues` so a later registry entry cannot read a + // refused stem as a one-member collision group. + let mut refused_stems: BTreeSet = BTreeSet::new(); + + for entry in registry { + let Some(slot) = slot_from_registry(entry, page_has_prebid) else { + continue; + }; + had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); + } + if let Some(prefix) = volatile_prefix_before_placement(&entry.div_id) { + refused_stems.insert(slot.div_id.clone()); + refused_div_ids.insert(slot.div_id); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); + continue; + } + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut registry_residues, slot, &entry.div_id) + { + warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); + } + } + + let registry_stems: BTreeSet = registry_residues + .keys() + .cloned() + .chain(refused_stems) + .collect(); + let mut request_residues: BTreeMap> = BTreeMap::new(); + for request in requests { + let Some((network_id, slot, raw_div)) = parse_gampad_request(&request.url) else { + continue; + }; + had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = Some(network_id); + } + if registry_stems.contains(&slot.div_id) { + continue; + } + if let Some(prefix) = volatile_prefix_before_placement(&raw_div) { + refused_div_ids.insert(slot.div_id); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); + continue; + } + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut request_residues, slot, &raw_div) + { + warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); + } + } + make_slot_ids_unique(&mut slots); + + DiscoveredSlots { + gam_network_id, + had_slot_evidence, + slots, + ambiguous_stems, + refused_div_ids, + warnings, + } +} + +/// Adds one source-local slot unless two distinct *elements* share its stem. +/// +/// Sharing a stem is not by itself ambiguity: one element re-rendered under a +/// fresh framework token is exactly what normalization exists to absorb, and it +/// produces two raw ids that collapse onto one stem. Ambiguity is two elements, +/// which [`ephemeral_marker_residue`] separates from two renders of one. +/// +/// The first distinct residue removes the tentatively accepted slot and returns +/// its stem for one diagnostic. Repeats and later collision members stay +/// suppressed and return `None`. +fn push_slot_refusing_collisions( + slots: &mut Vec, + seen_residues: &mut BTreeMap>, + slot: DiscoveredSlot, + raw_div: &str, +) -> Option { + let normalized = slot.div_id.clone(); + let residue = ephemeral_marker_residue(raw_div); + match seen_residues.get_mut(&normalized) { + None => { + seen_residues.insert(normalized, BTreeSet::from([residue])); + slots.push(slot); + None + } + Some(residues) if residues.contains(&residue) => None, + Some(residues) => { + let became_ambiguous = residues.len() == 1; + residues.insert(residue); + if became_ambiguous { + slots.retain(|entry| entry.div_id != normalized); + Some(normalized) + } else { + None + } + } + } +} + +/// Operator-facing text for a stem several live elements normalized onto. +fn ambiguous_collision_warning(prefix: &str) -> String { + format!( + "skipped ambiguous div-id prefix `{prefix}`: multiple active elements normalized to it, \ + but the runtime can resolve a prefix to only one active element and exact div ids change \ + across renders; expose distinct stable div ids in publisher markup before configuring \ + these placements" + ) +} + +/// The stable prefix of a div id whose per-render token precedes more of the id. +/// +/// Some ad stacks build ids as `__` — a +/// millisecond timestamp plus a random suffix sitting *before* the part that +/// distinguishes one placement from the next. Such an id can be written neither +/// literally (the token changes on the next render) nor as a prefix: the only +/// stable prefix stops at the token, and that prefix reaches every placement in +/// the family, while the runtime resolves a prefix to a single element. So the +/// slot is refused from a single observation, without waiting for a second +/// placement to prove the collision. +/// +/// The shape decides, not the vendor: any segment that is a long digit run +/// followed by more alphanumerics counts, so a new stack with the same layout +/// needs no code change. A token in *trailing* position is deliberately not this +/// case — everything before it still identifies the element — and is left to +/// normalization and the same-page collision check. +fn volatile_prefix_before_placement(div_id: &str) -> Option { + let div_id = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut start = 0_usize; + for (index, character) in div_id.char_indices() { + if character != '_' && character != '-' { + continue; + } + if is_per_render_token(&div_id[start..index]) { + let prefix = div_id[..start].trim_end_matches(['_', '-']); + // A delimiter is one byte, so the remainder starts just past it. + return (!prefix.is_empty() && !div_id[index + 1..].is_empty()) + .then(|| prefix.to_string()); + } + start = index + character.len_utf8(); + } + None +} + +/// Whether one div-id segment is a per-render token: a long leading digit run +/// followed by alphanumerics, or a shorter counter paired with a long random +/// suffix. +/// +/// Both halves are required. Eight-digit values need at least eight suffix +/// characters with a random-looking shape; this avoids treating calendar labels +/// followed by stable words as generated ids while still catching single-case +/// hashes and mixed alphanumeric tokens. A bare digit run is how publishers +/// write stable placement indices, and a token with a non-alphanumeric character +/// is some other structure than a generated id. +fn is_per_render_token(segment: &str) -> bool { + let leading_digits = segment.bytes().take_while(u8::is_ascii_digit).count(); + let suffix_length = segment.len().saturating_sub(leading_digits); + let suffix = &segment[leading_digits..]; + ((leading_digits >= 10 && suffix_length >= 1) + || (leading_digits >= 8 && suffix_length >= 8 && looks_random_suffix(suffix))) + && segment.bytes().all(|byte| byte.is_ascii_alphanumeric()) +} + +/// Whether a long suffix has structural signals of generated randomness. +fn looks_random_suffix(value: &str) -> bool { + let digit_count = value.bytes().filter(u8::is_ascii_digit).count(); + let letter_count = value.bytes().filter(u8::is_ascii_alphabetic).count(); + if digit_count >= 4 && letter_count >= 4 { + return true; + } + + let distinct = distinct_ascii_bytes(value); + if value.bytes().all(|byte| byte.is_ascii_hexdigit()) && distinct >= 4 { + return true; + } + // Single-case runs look generated only when they also lack the vowel spread + // of a word. ALL-CAPS placement labels such as `BILLBOARD` are ordinary + // publisher markup, while `zzqxwvkm` is a hash, and case alone cannot tell + // them apart in either direction. Vowel-free placement abbreviations can + // still be refused: these short tokens provide no reliable distinction + // between a consonant-only label and a generated suffix. + let single_case = value.bytes().all(|byte| byte.is_ascii_uppercase()) + || value.bytes().all(|byte| byte.is_ascii_lowercase()); + if single_case && distinct >= 4 && !has_vowel_structure(value) { + return true; + } + + has_random_case_alternation(value) && !has_wordlike_camel_segments(value) +} + +/// Whether letters are spread with the vowel density of a word. +/// +/// English placement labels run roughly 30-50% vowels and hashes cluster far +/// below, so a quarter-of-the-letters floor separates `SKYSCRAPER` from +/// `zzqxwvkm` without reading letter case. +fn has_vowel_structure(value: &str) -> bool { + let letters = value.bytes().filter(u8::is_ascii_alphabetic).count(); + let vowels = value + .bytes() + .filter(|byte| byte.is_ascii_alphabetic() && is_ascii_vowel(*byte)) + .count(); + letters > 0 && vowels * 4 >= letters +} + +/// Number of distinct byte values in a candidate token. +/// +/// A four-word bitset covers every byte without the 256-byte scan a flag array +/// would need for tokens this short. +fn distinct_ascii_bytes(value: &str) -> usize { + let mut seen = [0_u64; 4]; + let mut distinct = 0_usize; + for byte in value.bytes() { + let word = usize::from(byte >> 6); + let bit = 1_u64 << (byte & 0b0011_1111); + if seen[word] & bit == 0 { + seen[word] |= bit; + distinct += 1; + } + } + distinct +} + +/// Whether every CamelCase component contains a vowel-like letter. +/// +/// This distinguishes short word sequences such as `TopUsNewsAd` and +/// `MyAdUnitXy` from dense random alternation such as `AbCdEfGh`. +fn has_wordlike_camel_segments(value: &str) -> bool { + let mut segment_has_vowel = false; + for (index, byte) in value.bytes().enumerate() { + if index > 0 && byte.is_ascii_uppercase() { + if !segment_has_vowel { + return false; + } + segment_has_vowel = is_ascii_vowel(byte); + } else { + segment_has_vowel |= is_ascii_vowel(byte); + } + } + segment_has_vowel +} + +/// Whether an ASCII letter is a vowel, treating `y` as vowel-like for labels. +const fn is_ascii_vowel(byte: u8) -> bool { + matches!( + byte.to_ascii_lowercase(), + b'a' | b'e' | b'i' | b'o' | b'u' | b'y' + ) +} + +/// Whether letter case alternates densely enough to resemble a random token. +fn has_random_case_alternation(value: &str) -> bool { + let mut previous = None; + let mut comparisons = 0_usize; + let mut transitions = 0_usize; + for uppercase in value.bytes().filter_map(|byte| { + byte.is_ascii_lowercase() + .then_some(false) + .or_else(|| byte.is_ascii_uppercase().then_some(true)) + }) { + if let Some(previous) = previous { + comparisons += 1; + transitions += usize::from(previous != uppercase); + } + previous = Some(uppercase); + } + transitions >= 3 && transitions.saturating_mul(3) >= comparisons.saturating_mul(2) +} + +/// Operator-facing text for a div-id family carrying a per-render token. +fn volatile_prefix_warning(prefix: &str) -> String { + format!( + "skipped volatile div-id family `{prefix}`: a per-render token sits before the placement \ + suffix, so exact div ids change across renders and no distinct stable element prefix is \ + available; expose distinct stable div ids in publisher markup before configuring these \ + placements" + ) +} + +/// Records `warning` unless the same text was already recorded for this page. +fn push_unique_warning(warnings: &mut Vec, warning: String) { + if !warnings.contains(&warning) { + warnings.push(warning); + } +} + +/// Converts a live-registry slot into a [`DiscoveredSlot`]. +/// +/// Returns `None` when the slot has no usable pixel size or its div id is a +/// multi-slot (SRA) concatenation rather than a single element. +fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option { + if is_multi_slot_div(&entry.div_id) { + return None; + } + if !is_usable_unit_path(&entry.gam_unit_path) { + return None; + } + let formats: Vec<(u32, u32)> = entry + .sizes + .iter() + .copied() + .filter(|(width, height)| *width >= MIN_FORMAT_DIMENSION && *height >= MIN_FORMAT_DIMENSION) + .collect(); + if formats.is_empty() { + return None; + } + let div_stem = normalize_div_stem(&entry.div_id); + // Normalization truncates at the first ephemeral marker, so a div id that is + // *entirely* ephemeral (`_R_9sl…`, or exactly `-container`) reduces to the + // empty string. An empty `div_id` override fails config load outright, and + // an empty prefix would bind the slot to the first id-bearing element on the + // page, so such a slot is unusable rather than merely imprecise. + if div_stem.is_empty() { + return None; + } + Some(DiscoveredSlot { + id: slot_id_from_div(&div_stem), + div_id: div_stem, + gam_unit_path: entry.gam_unit_path.clone(), + formats, + has_prebid: page_has_prebid, + }) +} + +/// Whether a div id is a GPT single-request (SRA) concatenation of multiple +/// slots (joined with `~`) rather than one element. +fn is_multi_slot_div(div_id: &str) -> bool { + div_id.contains('~') +} + +/// Whether a scraped GAM ad-unit path can be represented in config. +/// +/// `gam_unit_path` is a template: `{` and `}` delimit placeholders and +/// [`parse_unit_template`](trusted_server_core::creative_opportunities) offers no +/// escape syntax. A live path containing a brace would either fail config load +/// or, worse, be silently reinterpreted as a placeholder-bearing template. A +/// blank path is rejected for the same reason config load rejects it. +fn is_usable_unit_path(path: &str) -> bool { + !path.trim().is_empty() && !path.contains(['{', '}']) +} + +/// Strips ephemeral GPT div-id noise so the stored id is stable across renders. +/// +/// Removes a trailing `-container` wrapper, then truncates at the first ephemeral +/// marker — a React SSR hash (`_R_`) or a hex-UUID segment — since both +/// change on every page load. Truncating (rather than excising) keeps the result +/// a valid **prefix** of the live div id, which is how verify matches slots. +/// +/// `div-gpt-ad-leaderboard-1` (stable) is unchanged; `ad-header-0-_R_9sl…-container` +/// and `ad-header-0-_r_8_` → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` +/// → `ad-in_content`. +fn normalize_div_stem(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let cut = ephemeral_marker_ranges(stem) + .first() + .map_or(stem.len(), |range| range.start); + stem[..cut].trim_end_matches('-').to_string() +} + +/// Byte ranges of every ephemeral per-render marker in `stem`, in order and +/// without overlaps. +/// +/// A hex-hash candidate must contain at least one `a`-`f`; a run of 16+ digits +/// is how publishers write stable ids, not a hash. +fn ephemeral_marker_ranges(stem: &str) -> Vec> { + let mut ranges: Vec> = REACT_USE_ID + .find_iter(stem) + .chain(UUID_SEGMENT.find_iter(stem)) + .chain(HEX_HASH_SEGMENT.find_iter(stem).filter(|matched| { + matched + .as_str() + .bytes() + .any(|byte| matches!(byte, b'a'..=b'f')) + })) + .map(|matched| matched.range()) + .collect(); + ranges.sort_by_key(|range| range.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start < last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + +/// The parts of a raw div id that no ephemeral marker covered, NUL-joined. +/// +/// [`normalize_div_stem`] truncates at the first marker, so two ids differing +/// only *inside* a marker collapse onto one stem — the signature of one element +/// re-rendered. What the markers did not cover separates that from two elements: +/// `ad-header-0-_R_3f_` and `ad-header-0-_r_0_` leave the same residue (one +/// element, two renders), while `…-in_content-0` and `…-in_content-1` do not +/// (two siblings). A live div id cannot contain NUL, so joining on it cannot +/// make two different residues compare equal. +fn ephemeral_marker_residue(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut residue = String::with_capacity(stem.len()); + let mut previous = 0_usize; + for range in ephemeral_marker_ranges(stem) { + residue.push_str(&stem[previous..range.start]); + residue.push('\0'); + previous = range.end; + } + residue.push_str(&stem[previous..]); + residue +} + +/// Extracts the leading network id from a GAM ad-unit path (`//...`). +fn network_id_from_unit_path(path: &str) -> Option { + let segment = path.trim_start_matches('/').split('/').next()?; + (!segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| segment.to_string()) +} + +/// Parses a single `gampad/ads` request URL into `(network_id, slot)`. +/// +/// Returns `None` when the URL is not a GPT ad request or is missing the fields +/// needed to describe a slot (ad-unit path, div id, and at least one size). +fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot, String)> { + let url = Url::parse(raw_url).ok()?; + let host = url.host_str()?; + if !GAMPAD_HOSTS.contains(&host) || !url.path().ends_with("/gampad/ads") { + return None; + } + + let mut iu_parts = None; + let mut dids = None; + let mut sizes_raw = None; + let mut fallback_sizes_raw = None; + let mut scp = None; + for (key, value) in url.query_pairs() { + match key.as_ref() { + "iu_parts" => iu_parts = Some(value.into_owned()), + "dids" => dids = Some(value.into_owned()), + "prev_iu_szs" => sizes_raw = Some(value.into_owned()), + "pb_szs" => fallback_sizes_raw = Some(value.into_owned()), + "prev_scp" => scp = Some(value.into_owned()), + _ => {} + } + } + + let iu_parts = iu_parts?; + let mut parts = iu_parts.split(',').filter(|part| !part.is_empty()); + // Mirror the registry path's validation: a GAM network id is digits only. + // The percent-decoded query value is page-controlled and gets spliced into + // generated TOML, so reject anything else. + let network_id = parts + .next() + .filter(|segment| segment.bytes().all(|byte| byte.is_ascii_digit()))? + .to_string(); + let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); + if !is_usable_unit_path(&gam_unit_path) { + return None; + } + // A usable unit path needs the network id plus at least one path segment. + parts.next()?; + + let raw_div = dids?; + if raw_div.contains(',') { + return None; + } + let raw_div = raw_div.trim().to_string(); + if raw_div.is_empty() { + return None; + } + if is_multi_slot_div(&raw_div) { + return None; + } + let div_id = normalize_div_stem(&raw_div); + // See `slot_from_registry`: a fully ephemeral div id normalizes to nothing, + // which is neither a valid config value nor a usable runtime prefix. + if div_id.is_empty() { + return None; + } + + let formats = parse_sizes(sizes_raw.as_deref().or(fallback_sizes_raw.as_deref())?); + if formats.is_empty() { + return None; + } + + let id = slot_id_from_div(&div_id); + let has_prebid = scp.as_deref().is_some_and(scp_shows_prebid); + + Some(( + network_id, + DiscoveredSlot { + id, + div_id, + gam_unit_path, + formats, + has_prebid, + }, + raw_div, + )) +} + +/// Parses a GPT size list (e.g. `970x250|4x1|620x366`) into pixel pairs. +/// +/// Accepts `|` or `,` separators, ignores non-`WxH` tokens, and drops +/// fluid/native ratio markers below [`MIN_FORMAT_DIMENSION`]. +fn parse_sizes(raw: &str) -> Vec<(u32, u32)> { + let mut sizes = Vec::new(); + for token in raw.split(['|', ',']) { + let Some((width, height)) = token.trim().split_once('x') else { + continue; + }; + let (Ok(width), Ok(height)) = (width.parse::(), height.parse::()) else { + continue; + }; + if width < MIN_FORMAT_DIMENSION || height < MIN_FORMAT_DIMENSION { + continue; + } + if !sizes.contains(&(width, height)) { + sizes.push((width, height)); + } + } + sizes +} + +/// Derives a runtime-safe slot id from a div id. +/// +/// The common GPT prefix is stripped, invalid character runs become one +/// hyphen, and an all-invalid value falls back to `slot`. +fn slot_id_from_div(div_id: &str) -> String { + let candidate = div_id.strip_prefix(GPT_DIV_PREFIX).unwrap_or(div_id); + let mut id = String::with_capacity(candidate.len()); + let mut previous_was_hyphen = false; + for character in candidate.chars() { + if character.is_ascii_alphanumeric() || character == '_' { + id.push(character); + previous_was_hyphen = false; + } else if !id.is_empty() && !previous_was_hyphen { + id.push('-'); + previous_was_hyphen = true; + } + } + while id.ends_with('-') { + id.pop(); + } + if id.is_empty() { + id.push_str("slot"); + } + + if validate_slot_id(&id).is_ok() { + id + } else { + "slot".to_string() + } +} + +/// Adds deterministic numeric suffixes when sanitization produces duplicate ids. +fn make_slot_ids_unique(slots: &mut [DiscoveredSlot]) { + let mut used = BTreeSet::new(); + for slot in slots { + if used.insert(slot.id.clone()) { + continue; + } + + let base = slot.id.clone(); + let mut suffix = 2_usize; + loop { + let candidate = format!("{base}-{suffix}"); + if used.insert(candidate.clone()) { + slot.id = candidate; + break; + } + suffix += 1; + } + } +} + +/// Detects Prebid/header-bidding signals in a slot's `prev_scp` targeting. +fn scp_shows_prebid(scp: &str) -> bool { + url::form_urlencoded::parse(scp.as_bytes()).any(|(key, value)| { + let key = key.to_ascii_lowercase(); + let value = value.to_ascii_lowercase(); + (key == "test" && value == "prebid") + || (key == "tude" && value == "true") + || key.starts_with("prebid") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A sample GPT leaderboard ad request (truncated to the fields the + /// parser reads; values are otherwise unmodified live output). + const SAMPLE_LEADERBOARD: &str = "https://securepubads.g.doubleclick.net/gampad/ads?\ + gdfp_req=1&iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C8x1%7C620x366%7C325x508%7C325x204\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=ad-loc%3Dleaderboard-1%26baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid%26tude%3Dtrue\ + &pb_szs=970x250%7C620x366"; + const SHORT_VOLATILE_DIV: &str = "vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1"; + + fn request(url: &str) -> CollectedRequest { + CollectedRequest { + url: url.to_string(), + resource_type: Some("fetch".to_string()), + } + } + + /// Discovers slots from ad requests only (no live registry). + fn from_requests(requests: &[CollectedRequest]) -> DiscoveredSlots { + discover_gpt_slots(&[], requests, false) + } + + #[test] + fn parses_leaderboard_slot() { + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD)]); + + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_eq!(discovered.slots.len(), 1, "should find one slot"); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1", "should strip the GPT div prefix"); + assert_eq!(slot.div_id, "div-gpt-ad-leaderboard-1"); + assert_eq!( + slot.gam_unit_path, + "/123456789/desktop/homepage/leaderboard1" + ); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366), (325, 508), (325, 204)], + "should keep pixel sizes and drop 4x1/8x1 fluid markers" + ); + assert!(slot.has_prebid, "prev_scp test=prebid should flag prebid"); + } + + #[test] + fn prebid_detection_requires_a_targeting_key_not_a_substring() { + assert!(scp_shows_prebid("test=prebid")); + assert!(!scp_shows_prebid("noprebid=true")); + } + + #[test] + fn deduplicates_refreshed_slot_requests() { + // GPT refreshes the same slot; a second identical request must not + // produce a duplicate slot. + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD), request(SAMPLE_LEADERBOARD)]); + + assert_eq!( + discovered.slots.len(), + 1, + "repeat requests for the same div should collapse" + ); + } + + #[test] + fn ignores_non_gampad_requests() { + let discovered = from_requests(&[ + request("https://securepubads.g.doubleclick.net/tag/js/gpt.js"), + request("https://cdn.example.com/app.js"), + request("https://analytics.example.com/collect?iu_parts=1%2Cfoo&dids=x"), + ]); + + assert!( + discovered.slots.is_empty(), + "only doubleclick gampad/ads requests should yield slots" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn skips_requests_missing_sizes() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x", + )]); + + assert!( + discovered.slots.is_empty(), + "a slot with no usable size should be skipped" + ); + } + + #[test] + fn skips_requests_with_only_network_id() { + // iu_parts with just the network id yields no unit path segment. + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a bare network id is not a usable ad-unit path" + ); + } + + #[test] + fn skips_requests_with_non_numeric_network_id() { + // A page-controlled iu_parts value must not smuggle a non-numeric + // network id (it gets spliced into generated TOML). + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%22evil%2Cslot&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a non-numeric network id should be rejected" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn falls_back_to_pb_szs_when_prev_iu_szs_absent() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x&pb_szs=300x250%7C728x90", + )]); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!(discovered.slots[0].formats, vec![(300, 250), (728, 90)]); + } + + fn registry_slot(path: &str, div: &str, sizes: &[(u32, u32)]) -> CollectedGptSlot { + CollectedGptSlot { + gam_unit_path: path.to_string(), + div_id: div.to_string(), + sizes: sizes.to_vec(), + } + } + + #[test] + fn lowercase_react_use_id_suffixes_collapse_to_one_slot() { + // React emits `_r_0_` client-side and `_R_3f_` server-side, and the + // token changes per render. Leaving it in the stem fragments one slot + // into a new key on every page, which starves template inference. + for volatile in [ + "ad-header-0-_r_0_", + "ad-header-0-_r_8_", + "ad-header-0-_r_a_", + "ad-header-0-_R_3f_", + ] { + let registry = vec![registry_slot("/123/site/news", volatile, &[(728, 90)])]; + let discovered = discover_gpt_slots(®istry, &[], false); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "`{volatile}` should normalize to a stable stem" + ); + } + } + + #[test] + fn an_ordinary_id_containing_r_is_left_alone() { + // The React shape is anchored, so a legitimate id keeps its full stem. + let registry = vec![registry_slot("/123/site/news", "ad_r_rail", &[(300, 250)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].div_id, "ad_r_rail"); + } + + #[test] + fn registry_slot_with_brace_in_unit_path_is_skipped() { + // `gam_unit_path` is a template and there is no escape syntax, so a + // literal brace either fails config load or is silently reinterpreted as + // a placeholder. Neither is acceptable to persist. + let registry = vec![ + registry_slot("/123/home/{section}", "div-gpt-ad-a", &[(300, 250)]), + registry_slot("/123/home/ok", "div-gpt-ad-b", &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "the brace-bearing slot should be dropped, the clean one kept" + ); + assert_eq!(discovered.slots[0].gam_unit_path, "/123/home/ok"); + } + + #[test] + fn registry_slot_whose_div_id_is_entirely_ephemeral_is_skipped() { + // `_R_…` is a React SSR marker; normalizing truncates at it, leaving an + // empty stem. An empty div_id fails config load, and as a runtime prefix + // it would match the first id-bearing element on the page. + let registry = vec![registry_slot( + "/123/home/header", + "_R_9slkta7pd6", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a slot with no stable div stem should be dropped, got {:?}", + discovered.slots + ); + } + + #[test] + fn volatile_guid_div_id_still_normalizes_to_a_usable_prefix() { + // A GUID between two copies of the placement name must still yield a + // usable stable stem; only an entirely ephemeral id is dropped. + let registry = vec![registry_slot( + "/123456789/publisher/homepage", + "ad-in_content-0949b6c5726343bf8bbec2ac47b494b4-in_content-0", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!( + discovered.slots[0].div_id, "ad-in_content", + "the GUID and trailing index should be truncated to a stable prefix" + ); + } + + #[test] + fn reads_slots_from_live_registry() { + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250), (1, 1), (620, 366)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], true); + + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "network id should come from the unit path" + ); + assert_eq!(discovered.slots.len(), 1); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1"); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366)], + "should drop the 1x1 out-of-page marker" + ); + assert!( + slot.has_prebid, + "page-level prebid should mark registry slots" + ); + } + + #[test] + fn registry_wins_and_requests_fill_gaps() { + // The registry reports the leaderboard; a gampad request reports a + // different div that the registry missed. Both should appear once. + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250)], + )]; + let requests = vec![ + // Same div as the registry — must not duplicate. + request(SAMPLE_LEADERBOARD), + // A div the registry did not report — must be added. + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Cdesktop%2Chomepage%2Csidebar1&dids=div-gpt-ad-sidebar-1&prev_iu_szs=300x600", + ), + ]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + let ids: Vec<&str> = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect(); + assert_eq!( + ids, + vec!["leaderboard-1", "sidebar-1"], + "registry slot kept, request fills the missing div, no duplicate" + ); + } + + #[test] + fn registry_slot_without_pixel_sizes_is_skipped() { + let registry = vec![registry_slot("/123/fluid", "div-gpt-ad-fluid", &[(1, 1)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a registry slot with only fluid markers is not usable" + ); + } + + #[test] + fn normalizes_ephemeral_hash_and_container_and_dedups() { + // A framework-hashed div: the same placement appears as a hashed inner div, + // a `-container` wrapper, and re-rendered with a different hash. All must + // collapse to one stable stem. + let registry = vec![ + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_", + &[(728, 90)], + ), + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_-container", + &[(728, 90)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "hash + container variants collapse" + ); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "ephemeral React hash and -container are stripped to a stable stem" + ); + assert_eq!(discovered.slots[0].id, "ad-header-0"); + } + + #[test] + fn drops_sra_multi_slot_concatenations() { + let registry = vec![registry_slot( + "/987654321/homepage/header-0/fixed_bottom-0", + "ad-header-0-_R_9slin~ad-fixed_bottom-0-_R_ainp", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "tilde-joined SRA multi-slot divs are not real single elements" + ); + } + + #[test] + fn leaves_clean_div_ids_unchanged() { + assert_eq!( + normalize_div_stem("div-gpt-ad-leaderboard-1"), + "div-gpt-ad-leaderboard-1" + ); + } + + #[test] + fn sanitizes_page_controlled_div_ids_for_runtime_slot_ids() { + let registry = vec![registry_slot( + "/123456789/homepage/header", + "div-gpt-ad-header.main: 1", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "header-main-1"); + assert_eq!( + discovered.slots[0].div_id, "div-gpt-ad-header.main: 1", + "matching should retain the original normalized div stem" + ); + trusted_server_core::creative_opportunities::validate_slot_id(&discovered.slots[0].id) + .expect("generated id should pass runtime validation"); + } + + #[test] + fn uses_fallback_for_div_id_without_safe_slot_id_characters() { + let registry = vec![registry_slot( + "/123456789/homepage/fallback", + "div-gpt-ad-...", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "slot"); + } + + #[test] + fn makes_colliding_sanitized_slot_ids_unique() { + let registry = vec![ + registry_slot( + "/123456789/homepage/dotted", + "div-gpt-ad-header.main", + &[(728, 90)], + ), + registry_slot( + "/123456789/homepage/colon", + "div-gpt-ad-header:main", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + let ids = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + + #[test] + fn normalizes_react_and_hex_hashes_to_stable_prefixes() { + assert_eq!( + normalize_div_stem("ad-header-0-_R_9slinpflik6lb_-container"), + "ad-header-0" + ); + let stem = + normalize_div_stem("ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0"); + assert_eq!(stem, "ad-in_content"); + assert!( + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0".starts_with(&stem), + "stem must prefix-match any re-rendered hex variant" + ); + } + + #[test] + fn hex_hash_truncation_requires_a_segment_boundary() { + // Hex UUID bounded by `-` → truncated to the stem. + assert_eq!( + normalize_div_stem("ad-x-de669245b2ea4b05826dc96f07a36272-y"), + "ad-x" + ); + // A token that merely starts with 16 hex chars (no boundary) is left intact. + assert_eq!( + normalize_div_stem("ad-de669245b2ea4b05z"), + "ad-de669245b2ea4b05z" + ); + } + + #[test] + fn long_numeric_segments_are_stable_ids_not_hex_hashes() { + assert_eq!( + normalize_div_stem("ad-slot-1234567890123456-tail"), + "ad-slot-1234567890123456-tail" + ); + } + + #[test] + fn comma_separated_sra_dids_are_ignored() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cnews%2Catf&dids=ad-a%2Cad-b&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a comma-joined SRA did list is not one element" + ); + } + + #[test] + fn one_element_under_two_render_tokens_is_not_a_collision() { + // Both ids describe in-content placement 0; only the hash between the + // two copies of the placement name differs, which is what one element + // re-rendered looks like. Refusing here would refuse the very shape + // normalization exists to absorb. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-0", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "two renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-in_content"); + assert!( + discovered.warnings.is_empty(), + "a re-render is not an ambiguity to report, got {:?}", + discovered.warnings + ); + assert!(discovered.ambiguous_stems.is_empty()); + } + + #[test] + fn sibling_placements_sharing_one_stem_are_refused() { + // Same shape as above, but the trailing placement index differs: these + // are two live elements, and one prefix cannot resolve to both. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "neither a broad prefix nor per-render exact IDs are safe" + ); + assert_ambiguous_collision_warning(&discovered, "ad-in_content"); + assert!( + discovered.ambiguous_stems.contains("ad-in_content"), + "the verdict must travel with the evidence, got {:?}", + discovered.ambiguous_stems + ); + } + + #[test] + fn react_server_and_client_render_tokens_are_one_slot() { + // A hydrating publisher reports the SSR id and the client id for the + // same element. Both must collapse rather than refuse each other. + let registry = vec![ + registry_slot("/123456789/site/news", "ad-header-0-_R_3f_", &[(728, 90)]), + registry_slot("/123456789/site/news", "ad-header-0-_r_0_", &[(728, 90)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "SSR and client renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-header-0"); + assert!(discovered.warnings.is_empty()); + } + + #[test] + fn repeated_raw_div_after_a_normalization_collision_is_deduplicated() { + let first = "ad-x-aaaaaaaaaaaaaaaa-0"; + let second = "ad-x-bbbbbbbbbbbbbbbb-1"; + let third = "ad-x-cccccccccccccccc-2"; + let registry = vec![ + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", third, &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "no repeat or later collision member may resurrect the group" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_normalization_collision_is_refused() { + let discovered = from_requests(&[ + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-aaaaaaaaaaaaaaaa-0&prev_iu_szs=300x250", + ), + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + ), + ]); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn single_volatile_family_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/123456789/site_in-article_desktop_1", + "vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1", + &[(300, 250)], + )], + &[], + false, + ); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "one observation of a per-render family must not be written literally" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert!( + discovered + .refused_div_ids + .contains("vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1"), + "registry refusal should retain its normalized div as observed evidence" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn single_volatile_family_request_slot_is_refused() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite_in-article_desktop_1&dids=vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1&prev_iu_szs=300x250", + )]); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); + assert!( + discovered + .refused_div_ids + .contains("vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1"), + "request refusal should retain its normalized div as observed evidence" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn shorter_high_entropy_singleton_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/123456789/publisher.example_overlay_mobile", + SHORT_VOLATILE_DIV, + &[(300, 250)], + )], + &[], + false, + ); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "a singleton per-render ID must not be written literally" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn shorter_high_entropy_singleton_request_slot_is_refused() { + let discovered = from_requests(&[request(&format!( + "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cpublisher.example_overlay_mobile\ + &dids={SHORT_VOLATILE_DIV}&prev_iu_szs=300x250" + ))]); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "request fallback must not write a singleton per-render ID" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn volatile_prefix_covers_every_placement_after_the_token() { + // The token's position is what makes the id unusable, so the placement + // that follows it is irrelevant: every one of these leaves `vendor-tag` + // as the only stable prefix, and that prefix reaches all of them. + for volatile in [ + "vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_20260820AbCdEfGh_slot_inarticle_1", + "vendor-tag_20260820deadbeef_slot_inarticle_1", + "vendor-tag_20260820XKMPQRST_slot_inarticle_1", + "vendor-tag_20260820ABCD1234_slot_inarticle_1", + "vendor-tag_20260820A1B2C3D4_slot_inarticle_1", + // Vowel-free single-case runs are hashes in either case. + "vendor-tag_20260820zzqxwvkm_slot_inarticle_1", + "vendor-tag_20260820qwrtypsd_slot_inarticle_1", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1-container", + "vendor-tag_1724112345678AbCdEfGh_slot_sidebar_1", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_stable", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1_extra", + ] { + assert_eq!( + volatile_prefix_before_placement(volatile).as_deref(), + Some("vendor-tag"), + "`{volatile}` should be refused as a volatile family" + ); + } + } + + #[test] + fn volatile_prefix_does_not_claim_stable_div_ids() { + for stable in [ + // No per-render token at all. + "vendor-tag_stable_slot_inarticle_1", + // A bare digit run is how stable placement indices are written. + "vendor-tag_12345678_slot_inarticle_1", + "ad-slot-1234567890123456-tail", + // Shorter counter/suffix combinations do not carry enough entropy. + "vendor-tag_1234567AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfG_slot_inarticle_1", + // An eight-digit calendar date plus a stable suffix is not a + // timestamp-like per-render token. + "promo-20260820a-sidebar", + "promo-20260820Football-sidebar", + "promo-20260820football-sidebar", + "promo-20260820TopStories-sidebar", + "promo-20260820TopUsNewsAd-sidebar", + "promo-20260820MyAdUnitXy-sidebar", + "promo-20260820Top10Stories-sidebar", + "ad-19700101Thumbnail-rail", + "ad-00000001AAAAAAAA-rail", + // ALL-CAPS is a common publisher convention for placement labels, + // including the standard IAB format names. + "promo-20260820BILLBOARD-sidebar", + "promo-20260820HEADLINE-sidebar", + "promo-20260820LEADERBOARD-sidebar", + "promo-20260820SKYSCRAPER-sidebar", + "promo-20260820INARTICLE-sidebar", + "promo-20260820RECTANGLE-sidebar", + // The token is trailing, so the prefix before it still identifies + // this element and normalization/collision handling own the case. + "vendor-tag_slot_inarticle_1724112345678AbCdEfGh", + "vendor-tag-header", + ] { + assert_eq!( + volatile_prefix_before_placement(stable), + None, + "`{stable}` should stay eligible" + ); + } + } + + #[test] + fn ambiguous_registry_stem_still_suppresses_request_fallback() { + let registry = vec![ + registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + ), + registry_slot( + "/123456789/site/home", + "ad-x-bbbbbbbbbbbbbbbb-1", + &[(300, 250)], + ), + ]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-cccccccccccccccc-2&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "request fallback must not resurrect an ambiguous registry stem" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_rerender_does_not_rewrite_a_stable_registry_slot() { + let registry = vec![registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + )]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert_eq!(discovered.slots.len(), 1, "registry evidence should win"); + assert_eq!( + discovered.slots[0].div_id, "ad-x", + "request fallback must not destabilize a registry-derived prefix" + ); + } + + fn assert_ambiguous_collision_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!(discovered.warnings.len(), 1); + let warning = &discovered.warnings[0]; + assert!(warning.contains(prefix), "warning should name the prefix"); + assert!( + warning.contains("one active element"), + "warning should explain why the broad prefix is unsafe" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why raw IDs are unsafe" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable" + ); + } + + fn assert_volatile_prefix_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!( + discovered.warnings.len(), + 1, + "should report the family once, got {:?}", + discovered.warnings + ); + let warning = &discovered.warnings[0]; + assert!( + warning.contains(prefix), + "warning should name the family prefix, got {warning}" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why the exact ids are unsafe, got {warning}" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable, got {warning}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs new file mode 100644 index 000000000..828346930 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -0,0 +1,4299 @@ +mod analyzer; +pub(crate) mod browser_collector; +pub(crate) mod collector; +mod crawl_plan; +mod evidence; +mod gpt_slots; +mod page_patterns; +mod slot_toml; +mod unit_template; +mod validate; + +use std::collections::BTreeSet; +use std::fmt::Write as _; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use rand::RngCore as _; +use serde::Serialize; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, validate_page_pattern, +}; +use url::Url; + +use crate::commands::audit::ad_templates::{origin_changed, without_fragment}; +use crate::commands::audit::collector::GenerateBrowserOpts; +use crate::commands::audit::generate::collector::AuditCollector; +use crate::commands::audit::generate::slot_toml::{ + render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, toml_string, +}; +use crate::commands::config::init::EXAMPLE_CONFIG; +use crate::error::{CliResult, cli_error, report_error}; + +use analyzer::{analyze_collected_page, extract_gtm_container_id}; + +pub(crate) use browser_collector::DeviceProfile; +pub(crate) use crawl_plan::CrawlBudget; + +/// Writes `contents` to `path` atomically: a same-directory temp file is +/// written and fsynced, then renamed over the target, then the directory entry +/// is fsynced. +/// +/// A plain `fs::write` truncates the destination before writing, so a full disk +/// or an interrupted run would leave an operator's `trusted-server.toml` empty +/// or half-written. `rename` within a directory is atomic, so a reader sees +/// either the old file or the complete new one. +/// +/// The target's existing permissions are carried onto the replacement, since +/// the temp file is created 0600 and the config may intentionally be broader. +/// +/// # Errors +/// +/// Returns the underlying I/O error when the temp file cannot be created, +/// written, synced, or renamed over `path`. +fn write_file_atomically(path: &Path, contents: &str) -> std::io::Result<()> { + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + + let mut temp = tempfile::Builder::new() + .prefix(".ts-audit-") + .tempfile_in(directory)?; + temp.write_all(contents.as_bytes())?; + temp.as_file().sync_all()?; + if let Ok(metadata) = fs::metadata(path) { + temp.as_file().set_permissions(metadata.permissions())?; + } + temp.persist(path).map_err(|error| error.error)?; + + // Best-effort durability for the rename itself. Opening a directory handle + // is not portable (Windows rejects it), and the content is already safely + // on disk either way, so a failure here is not worth failing the command. + let _ = fs::File::open(directory).and_then(|handle| handle.sync_all()); + Ok(()) +} + +/// Arguments for `ts audit generate ` — bootstraps draft Trusted Server +/// config and JavaScript asset audit files from a live page (issue #800). +#[derive(Debug, clap::Args)] +pub(crate) struct GenerateArgs { + /// Public HTTP(S) URL to audit. + pub(crate) url: String, + /// JavaScript asset audit output path. + #[arg(long)] + pub(crate) js_assets: Option, + /// Draft Trusted Server config output path. + #[arg(long)] + pub(crate) config: Option, + /// Do not write the JavaScript asset audit file. + #[arg(long)] + pub(crate) no_js_assets: bool, + /// Do not write the draft Trusted Server config file. + #[arg(long)] + pub(crate) no_config: bool, + /// Overwrite existing output files. + #[arg(long)] + pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = crate::commands::audit::parse_cookie)] + pub(crate) cookies: Vec<(String, String)>, + /// Browser and consent options shared with `ts audit ad-templates generate`. + #[command(flatten)] + pub(crate) browser: GenerateBrowserOpts, +} + +const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; +const DEFAULT_CONFIG_PATH: &str = "trusted-server.toml"; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum AssetParty { + FirstParty, + ThirdParty, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct AuditedAsset { + pub(crate) kind: String, + pub(crate) url: String, + pub(crate) host: String, + pub(crate) party: AssetParty, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) integration: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct DetectedIntegration { + pub(crate) id: String, + pub(crate) evidence: String, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct AuditArtifact { + pub(crate) audited_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) page_title: Option, + pub(crate) js_asset_count: usize, + pub(crate) third_party_asset_count: usize, + pub(crate) detected_integrations: Vec, + pub(crate) assets: Vec, + pub(crate) warnings: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct AuditOutputs { + pub(crate) artifact: AuditArtifact, + pub(crate) js_assets_toml: String, + pub(crate) draft_config_toml: String, + pub(crate) ad_slot_count: usize, + pub(crate) js_asset_proxy_candidate_count: usize, +} + +#[derive(Debug, Clone)] +struct DraftConfig { + toml: String, + js_asset_proxy_candidate_count: usize, +} + +#[derive(Debug, Clone)] +struct JsAssetProxySection { + toml: String, + candidate_count: usize, +} + +#[derive(Debug, Default)] +struct JsAssetProxySkipCounts { + first_party: usize, + malformed_url: usize, + non_https: usize, + duplicate_url: usize, + non_script: usize, +} + +#[derive(Debug)] +struct JsAssetProxyCandidate<'a> { + origin_url: String, + integration: Option<&'a str>, +} + +trait OpaqueAssetPathGenerator { + fn next_path(&mut self) -> String; +} + +#[derive(Debug, Default)] +struct RandomOpaqueAssetPathGenerator; + +impl OpaqueAssetPathGenerator for RandomOpaqueAssetPathGenerator { + fn next_path(&mut self) -> String { + let mut bytes = [0_u8; 12]; + rand::rngs::OsRng.fill_bytes(&mut bytes); + format!("/assets/{}.js", lowercase_hex(&bytes)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AuditOutputPlan { + js_assets_path: Option, + config_path: Option, +} + +pub(crate) fn run_generate( + args: &GenerateArgs, + collector: &dyn AuditCollector, + out: &mut dyn Write, +) -> CliResult<()> { + let target_url = parse_audit_url(&args.url)?; + let plan = resolve_output_plan(args)?; + let collected = collector.collect_page(&target_url, &args.cookies)?; + let outputs = build_audit_outputs(&collected)?; + let wrote_config = plan.config_path.is_some(); + let written = write_audit_outputs(&outputs, &plan)?; + write_success_summary(&outputs, &written, wrote_config, out) +} + +fn parse_audit_url(value: &str) -> CliResult { + let url = Url::parse(value) + .map_err(|error| report_error(format!("invalid audit URL `{value}`: {error}")))?; + if !matches!(url.scheme(), "http" | "https") { + return cli_error(format!( + "`ts audit` only supports http/https URLs, got `{}`", + url.scheme() + )); + } + Ok(url) +} + +fn resolve_output_plan(args: &GenerateArgs) -> CliResult { + if args.no_js_assets && args.no_config { + return cli_error("nothing to do: both --no-js-assets and --no-config were set"); + } + + let js_assets_path = if args.no_js_assets { + None + } else { + Some(resolve_output_path( + args.js_assets.as_deref(), + DEFAULT_JS_ASSETS_PATH, + )?) + }; + let config_path = if args.no_config { + None + } else { + Some(resolve_output_path( + args.config.as_deref(), + DEFAULT_CONFIG_PATH, + )?) + }; + + if js_assets_path.is_some() && js_assets_path == config_path { + return cli_error("audit output paths must be distinct"); + } + + for path in [&js_assets_path, &config_path].into_iter().flatten() { + if path.exists() && !args.force { + return cli_error(format!( + "refusing to overwrite existing file `{}`; re-run with --force", + path.display() + )); + } + } + + Ok(AuditOutputPlan { + js_assets_path, + config_path, + }) +} + +fn resolve_output_path(path: Option<&Path>, default: &str) -> CliResult { + let candidate = path.unwrap_or_else(|| Path::new(default)); + if candidate.is_absolute() { + Ok(candidate.to_path_buf()) + } else { + Ok(std::env::current_dir() + .map_err(|error| report_error(format!("failed to read current directory: {error}")))? + .join(candidate)) + } +} + +fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult { + let artifact = analyze_collected_page(collected)?; + let final_url = collected + .final_url() + .map_err(|error| report_error(format!("invalid final URL: {error}")))?; + let js_assets_toml = toml::to_string_pretty(&artifact) + .map_err(|error| report_error(format!("failed to serialize audit artifact: {error}")))?; + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let slots = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + let ad_slot_count = slots.slots.len(); + let mut path_generator = RandomOpaqueAssetPathGenerator; + let draft_config = + build_draft_config_with_generator(&final_url, &artifact, &slots, &mut path_generator)?; + + Ok(AuditOutputs { + artifact, + js_assets_toml, + draft_config_toml: draft_config.toml, + ad_slot_count, + js_asset_proxy_candidate_count: draft_config.js_asset_proxy_candidate_count, + }) +} + +fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliResult> { + let selected_paths = [&plan.js_assets_path, &plan.config_path] + .into_iter() + .flatten() + .collect::>(); + for path in &selected_paths { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent).map_err(|error| { + report_error(format!( + "failed to create parent directory {}: {error}", + parent.display() + )) + })?; + } + } + + let mut written_paths = Vec::new(); + if let Some(path) = &plan.js_assets_path { + write_file_atomically(path, &outputs.js_assets_toml).map_err(|error| { + report_error(format!( + "failed to write JS asset audit {}: {error}", + path.display() + )) + })?; + written_paths.push(path.display().to_string()); + } + if let Some(path) = &plan.config_path { + write_file_atomically(path, &outputs.draft_config_toml).map_err(|error| { + report_error(format!( + "failed to write draft config {}: {error}", + path.display() + )) + })?; + written_paths.push(path.display().to_string()); + } + + Ok(written_paths) +} + +fn write_success_summary( + outputs: &AuditOutputs, + written: &[String], + wrote_config: bool, + out: &mut dyn Write, +) -> CliResult<()> { + let integrations = outputs + .artifact + .detected_integrations + .iter() + .map(|integration| integration.id.as_str()) + .collect::>(); + let draft_note = if wrote_config { + "\nDraft config: review before validation and push" + } else { + "" + }; + let asset_proxy_note = if wrote_config && outputs.js_asset_proxy_candidate_count > 0 { + format!( + "{} disabled entries written to draft config", + outputs.js_asset_proxy_candidate_count + ) + } else if wrote_config { + "none".to_string() + } else { + "not written (--no-config)".to_string() + }; + writeln!( + out, + "Audited {}\nTitle: {}\nJS assets: {}\nThird-party assets: {}\nAd slots: {}\nDetected integrations: {}\nJS asset proxy candidates: {}\nWrote: {}{}", + outputs.artifact.audited_url, + outputs + .artifact + .page_title + .as_deref() + .unwrap_or(""), + outputs.artifact.js_asset_count, + outputs.artifact.third_party_asset_count, + outputs.ad_slot_count, + if integrations.is_empty() { + "none".to_string() + } else { + integrations.join(", ") + }, + asset_proxy_note, + if written.is_empty() { + "none".to_string() + } else { + written.join(", ") + }, + draft_note + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +#[cfg(test)] +fn build_draft_config( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, +) -> CliResult { + let mut path_generator = RandomOpaqueAssetPathGenerator; + build_draft_config_with_generator(target_url, artifact, slots, &mut path_generator) + .map(|draft| draft.toml) +} + +fn build_draft_config_with_generator( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, + path_generator: &mut dyn OpaqueAssetPathGenerator, +) -> CliResult { + let host = target_url + .host_str() + .ok_or_else(|| report_error("audited URL is missing a host"))?; + let origin = target_url.origin().ascii_serialization(); + let mut draft = EXAMPLE_CONFIG.to_string(); + + draft = replace_key_in_section( + &draft, + "publisher", + "domain", + &format!("domain = \"{host}\""), + )?; + draft = replace_key_in_section( + &draft, + "publisher", + "cookie_domain", + &format!("cookie_domain = \".{host}\""), + )?; + draft = replace_key_in_section( + &draft, + "publisher", + "origin_url", + &format!("origin_url = \"{origin}\""), + )?; + + let detected = artifact + .detected_integrations + .iter() + .map(|integration| integration.id.as_str()) + .collect::>(); + + if detected.contains("gpt") { + draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; + } + if detected.contains("didomi") { + draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; + } + if detected.contains("datadome") { + draft = + replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; + } + + let asset_proxy_section = build_js_asset_proxy_section(artifact, path_generator)?; + draft = replace_js_asset_proxy_section(&draft, &asset_proxy_section.toml)?; + + let mut manual_review = Vec::new(); + if detected.contains("google_tag_manager") { + if let Some(gtm_id) = extract_gtm_container_id(artifact) { + draft = replace_key_in_section( + &draft, + "integrations.google_tag_manager", + "enabled", + "enabled = true", + )?; + draft = replace_key_in_section( + &draft, + "integrations.google_tag_manager", + "container_id", + &format!("container_id = \"{gtm_id}\""), + )?; + } else { + manual_review.push("google_tag_manager"); + } + } + + for integration in detected { + if !matches!( + integration, + "gpt" | "didomi" | "datadome" | "google_tag_manager" + ) { + manual_review.push(integration); + } + } + + if !manual_review.is_empty() { + if !draft.ends_with('\n') { + draft.push('\n'); + } + draft.push_str("\n# Audit findings requiring manual review\n"); + for integration in manual_review { + draft.push_str(&format!( + "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" + )); + } + } + + if !slots.slots.is_empty() { + if let Some(network_id) = &slots.gam_network_id { + draft = replace_key_in_section( + &draft, + "creative_opportunities", + "gam_network_id", + &format!("gam_network_id = {}", toml_string(network_id)), + )?; + } + draft.push_str(&render_discovered_slots(target_url, slots)); + } + + Ok(DraftConfig { + toml: draft, + js_asset_proxy_candidate_count: asset_proxy_section.candidate_count, + }) +} + +fn build_js_asset_proxy_section( + artifact: &AuditArtifact, + path_generator: &mut dyn OpaqueAssetPathGenerator, +) -> CliResult { + let (candidates, skipped) = select_js_asset_proxy_candidates(artifact); + let mut used_paths = BTreeSet::new(); + let mut toml = String::new(); + + toml.push_str("[integrations.js_asset_proxy]\n"); + toml.push_str("enabled = false\n"); + toml.push_str("# Uncomment to override upstream cache headers for every asset below.\n"); + toml.push_str("# This replaces upstream directives, including private and no-store.\n"); + toml.push_str("# Use only when each asset's bytes are identical for every visitor.\n"); + toml.push_str("# cache_ttl_seconds = 3600\n"); + toml.push_str( + "# Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets\n", + ); + toml.push_str( + "# that vary by browser User-Agent or use integrity hashes for UA-specific bytes.\n\n", + ); + toml.push_str("# Generated by `ts audit`; review before enabling.\n"); + toml.push_str( + "# Audit note: some discovered scripts may be runtime-injected and may not appear\n", + ); + toml.push_str( + "# in origin HTML. JS Asset Proxy rewrites only matching script src URLs present in\n", + ); + toml.push_str("# HTML processed by Trusted Server.\n"); + + if candidates.is_empty() { + toml.push_str( + "# No eligible third-party HTTPS script assets were detected by `ts audit`.\n", + ); + } + + for candidate in &candidates { + let generated_path = generate_unique_asset_path(path_generator, &mut used_paths)?; + toml.push('\n'); + toml.push_str("# Generated by `ts audit`; review before enabling.\n"); + if let Some(integration) = candidate.integration { + let integration = sanitized_comment_value(integration); + toml.push_str(&format!("# Detected integration: {integration}\n")); + toml.push_str(&format!( + "# Native integration may be preferable: [integrations.{integration}]\n" + )); + } + toml.push_str("[[integrations.js_asset_proxy.assets]]\n"); + toml.push_str(&format!("path = {}\n", toml_quoted_string(&generated_path))); + toml.push_str(&format!( + "origin_url = {}\n", + toml_quoted_string(&candidate.origin_url) + )); + if Url::parse(&candidate.origin_url).is_ok_and(|url| url.query().is_some()) { + toml.push_str( + "# This URL includes a query string and must remain stable for proxy matching.\n", + ); + } + toml.push_str("proxy = \"disabled\"\n"); + } + + append_js_asset_proxy_skip_comments(&mut toml, &skipped); + toml.push('\n'); + + Ok(JsAssetProxySection { + toml, + candidate_count: candidates.len(), + }) +} + +fn select_js_asset_proxy_candidates( + artifact: &AuditArtifact, +) -> (Vec>, JsAssetProxySkipCounts) { + let mut candidates = Vec::new(); + let mut skipped = JsAssetProxySkipCounts::default(); + let mut seen_origin_urls = BTreeSet::new(); + + for asset in &artifact.assets { + if asset.kind != "script" { + skipped.non_script += 1; + continue; + } + if asset.party != AssetParty::ThirdParty { + skipped.first_party += 1; + continue; + } + + let Ok(url) = Url::parse(&asset.url) else { + skipped.malformed_url += 1; + continue; + }; + if url.host_str().is_none() { + skipped.malformed_url += 1; + continue; + } + if url.scheme() != "https" { + skipped.non_https += 1; + continue; + } + + let origin_url = url.to_string(); + if !seen_origin_urls.insert(origin_url.clone()) { + skipped.duplicate_url += 1; + continue; + } + + candidates.push(JsAssetProxyCandidate { + origin_url, + integration: asset.integration.as_deref(), + }); + } + + (candidates, skipped) +} + +fn generate_unique_asset_path( + path_generator: &mut dyn OpaqueAssetPathGenerator, + used_paths: &mut BTreeSet, +) -> CliResult { + for _ in 0..128 { + let path = path_generator.next_path(); + if !is_valid_generated_asset_path(&path) { + return cli_error(format!( + "generated JS asset proxy path `{path}` is invalid; expected /assets/.js" + )); + } + if used_paths.insert(path.clone()) { + return Ok(path); + } + } + + cli_error("failed to generate a unique JS asset proxy path after 128 attempts") +} + +fn is_valid_generated_asset_path(path: &str) -> bool { + let Some(opaque_id) = path + .strip_prefix("/assets/") + .and_then(|value| value.strip_suffix(".js")) + else { + return false; + }; + + !opaque_id.is_empty() + && opaque_id + .chars() + .all(|ch| ch.is_ascii_hexdigit() && !ch.is_ascii_uppercase()) +} + +fn replace_js_asset_proxy_section(document: &str, replacement: &str) -> CliResult { + let lines = document.lines().collect::>(); + let start = lines + .iter() + .position(|line| line.trim() == "[integrations.js_asset_proxy]") + .ok_or_else(|| { + report_error( + "failed to update starter config because section `[integrations.js_asset_proxy]` was not found", + ) + })?; + let mut end = start + 1; + + while end < lines.len() { + let trimmed = lines[end].trim(); + if trimmed.starts_with('[') + && trimmed.ends_with(']') + && trimmed != "[[integrations.js_asset_proxy.assets]]" + { + break; + } + end += 1; + } + + // Blank lines and comments directly above the next section header document + // that section, not this one, so leave them in the draft. + while end > start + 1 { + let trimmed = lines[end - 1].trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + end -= 1; + } else { + break; + } + } + + let mut output_lines = Vec::new(); + output_lines.extend_from_slice(&lines[..start]); + output_lines.extend(replacement.trim_end_matches('\n').lines()); + if end < lines.len() && !lines[end].trim().is_empty() { + output_lines.push(""); + } + output_lines.extend_from_slice(&lines[end..]); + + let mut output = output_lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + Ok(output) +} + +fn append_js_asset_proxy_skip_comments(toml: &mut String, skipped: &JsAssetProxySkipCounts) { + if skipped.first_party == 0 + && skipped.malformed_url == 0 + && skipped.non_https == 0 + && skipped.duplicate_url == 0 + && skipped.non_script == 0 + { + return; + } + + toml.push('\n'); + toml.push_str("# Skipped JS Asset Proxy audit candidates:\n"); + append_skip_count(toml, skipped.first_party, "first-party script"); + append_skip_count(toml, skipped.malformed_url, "malformed script URL"); + append_skip_count(toml, skipped.non_https, "non-HTTPS third-party script"); + append_skip_count(toml, skipped.duplicate_url, "duplicate script URL"); + append_skip_count(toml, skipped.non_script, "non-script asset"); +} + +fn append_skip_count(toml: &mut String, count: usize, label: &str) { + if count == 0 { + return; + } + + let plural = if count == 1 { "" } else { "s" }; + toml.push_str(&format!("# - {count} {label}{plural}\n")); +} + +fn sanitized_comment_value(value: &str) -> String { + value + .chars() + .map(|ch| if ch.is_control() { ' ' } else { ch }) + .collect() +} + +fn toml_quoted_string(value: &str) -> String { + let mut quoted = String::from("\""); + for ch in value.chars() { + match ch { + '\\' => quoted.push_str("\\\\"), + '"' => quoted.push_str("\\\""), + '\n' => quoted.push_str("\\n"), + '\r' => quoted.push_str("\\r"), + '\t' => quoted.push_str("\\t"), + ch if ch.is_control() => { + write!(&mut quoted, "\\u{:04X}", ch as u32).expect("should write to string"); + } + ch => quoted.push(ch), + } + } + quoted.push('"'); + quoted +} + +fn lowercase_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(bytes.len() * 2); + for byte in bytes { + encoded.push(HEX[(byte >> 4) as usize] as char); + encoded.push(HEX[(byte & 0x0f) as usize] as char); + } + encoded +} + +/// Renders discovered GPT slots as appended `[[creative_opportunities.slot]]` +/// tables. Page patterns default to the audited path and are flagged for review. +fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) -> String { + let path = target_url.path(); + let page_pattern = if path.is_empty() { "/" } else { path }; + + let mut out = String::from( + "\n# Slots discovered from live GPT ad requests during the audit.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in &slots.slots { + let formats = slot + .formats + .iter() + .map(|(width, height)| format!("{{ width = {width}, height = {height} }}")) + .collect::>() + .join(", "); + out.push_str(&format!( + "\n[[creative_opportunities.slot]]\n\ + id = {id}\n\ + div_id = {div_id}\n\ + gam_unit_path = {gam_unit_path}\n\ + page_patterns = [{page_pattern}]\n\ + formats = [{formats}]\n", + id = toml_string(&slot.id), + div_id = toml_string(&slot.div_id), + gam_unit_path = toml_string(&slot.gam_unit_path), + page_pattern = toml_string(page_pattern), + )); + if slot.has_prebid { + out.push_str("[creative_opportunities.slot.providers.prebid]\nbidders = {}\n"); + } + } + out +} + +/// Everything one `ts audit ad-templates generate` invocation needs. +pub(crate) struct UpdateSlotsRequest<'a> { + /// Page URL to start from; also bounds the crawl to its origin. + pub(crate) url: &'a str, + /// Operator config to rewrite in place. + pub(crate) config_path: &'a Path, + /// The config's current `[creative_opportunities]`, when it has one. + pub(crate) existing_creative: Option<&'a CreativeOpportunitiesConfig>, + /// Explicit `--page-pattern` values. When non-empty these apply to every + /// slot and pattern inference is skipped entirely. + pub(crate) page_patterns: &'a [String], + /// Replace existing slots rather than merging into them. + pub(crate) replace: bool, + /// Cookies to carry into the crawl. + pub(crate) cookies: &'a [(String, String)], + /// Print the candidate instead of writing it. + pub(crate) dry_run: bool, + /// Whether the crawl used the deterministic scroll pass. + pub(crate) scroll: bool, + /// Crawl bounds. + pub(crate) budget: crawl_plan::CrawlBudget, +} + +/// Share of crawled pages that may yield no slots before the run is refused. +/// +/// A bot-protection challenge serves an interstitial that loads fine and +/// contains no ad stack, so it looks like a page with no slots. Writing a config +/// from a crawl that was mostly challenges would silently narrow the operator's +/// slot set; refusing is the safer failure. +const MAX_EMPTY_PAGE_SHARE: f64 = 0.25; + +/// Runs `ts audit ad-templates generate`: crawl the site's sections, reconcile +/// what each slot looked like across them, infer a `{section}` ad-unit template +/// where the evidence proves one, and rewrite the config's slot array in place. +/// +/// # Errors +/// +/// Returns an error when the config cannot be read, the root page cannot be +/// collected, no slots are discovered, too many pages came back empty, the +/// pages disagree about the GAM network id, or the resulting config would not +/// load. +pub(crate) fn run_update_slots( + request: &UpdateSlotsRequest<'_>, + collectors: &[(&str, &dyn AuditCollector)], + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()> { + let Some((first_label, first_collector)) = collectors.first() else { + return cli_error("no device profile was selected to audit with"); + }; + let target_url = parse_audit_url(request.url)?; + let existing = fs::read_to_string(request.config_path).map_err(|error| { + report_error(format!( + "failed to read config {}: {error}", + request.config_path.display() + )) + })?; + + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + let mut root_url = target_url.clone(); + let mut planned = None; + let mut fold_error = None; + + { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: first_label, + }; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + first_collector.collect_site( + &target_url, + request.cookies, + &mut report_progress, + &mut |_, root| { + root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); + if origin_changed(&target_url, &root_url) { + // Origins only: the origin is what the refusal is about, and + // a full URL would echo any `user:password@` the operator + // passed into stderr. + return cli_error(format!( + "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", + target_url.origin().ascii_serialization(), + root_url.origin().ascii_serialization() + )); + } + let plan = crawl_plan::plan_crawl( + &root_url, + &root.links, + &root.sitemap_locs, + request.budget, + ); + let targets = plan.targets(); + planned = Some(plan); + Ok(targets) + }, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + // The requested origin is the trust boundary for every + // page, not just the root: a section page that redirects + // away would otherwise contribute foreign slots, formats + // and ad-unit paths to the generated config. + if origin_changed(&target_url, &final_url) { + notes.push(format!( + "skipped `{}` on {first_label}: it left the audited origin for {}", + url.path(), + final_url.origin().ascii_serialization() + )); + return Ok(collector::ControlFlow::Continue); + } + if let Err(error) = + fold_collected( + &mut table, + &final_url, + &page, + first_label, + &mut notes, + ) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + // Path only, like the progress lines: a planned target + // still carries the origin and any userinfo. + notes.push(format!( + "skipped `{}` on {first_label}: {error}", + url.path() + )); + } + } + Ok(collector::ControlFlow::Continue) + }, + )?; + } + if let Some(error) = fold_error { + return Err(error); + } + let plan = planned.ok_or_else(|| { + report_error(format!( + "the {first_label} browser session did not produce a root page" + )) + })?; + notes.extend(plan.notes.iter().cloned()); + // Fragments never reach the server, so only a difference the origin acted on + // counts as a redirect worth reporting. + if without_fragment(&root_url) != without_fragment(&target_url) { + notes.push(format!( + "followed a root redirect from `{}{}` to `{}{}`; slots and page patterns are derived from the final URL", + target_url.origin().ascii_serialization(), + target_url.path(), + root_url.origin().ascii_serialization(), + root_url.path() + )); + } + + // Every profile walks the same pages into the same table. When two profiles + // disagree about a slot's ad-unit path, that shows up as two observations of + // one page, which inference already refuses to represent. + for (label, collector) in collectors.iter().skip(1) { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: label, + }; + let successful_pages = crawl_sections( + *collector, + &root_url, + &plan, + request.cookies, + &mut table, + &mut notes, + &mut progress_writer, + )?; + if successful_pages == 0 { + return cli_error(format!( + "the selected {label} device profile did not collect any required page; refusing to generate from incomplete profile coverage" + )); + } + } + if collectors.len() > 1 { + notes.push(format!( + "audited {} device profile(s): {}", + collectors.len(), + collectors + .iter() + .map(|(label, _)| *label) + .collect::>() + .join(", ") + )); + } + + // Emit what the crawl learned before any refusal below can return early. + // The guards exist precisely for runs that went wrong, so that is when the + // per-page reasons matter most. + emit_notes(err, &mut notes)?; + + if table.is_empty() { + return cli_error(format!( + "no ad-template slots were discovered on any of the {} crawled page(s); \ + see the notes above for what each page reported", + table.pages().len() + )); + } + guard_challenge_rate(&table)?; + + let discovered_network_id = table.network_id()?; + let network_id = resolve_network_id( + request.existing_creative, + discovered_network_id.as_deref(), + request.replace, + ); + + // Templating needs a network id to bind `{network_id}` against; without one + // every path stays literal. + let inference = network_id + .as_deref() + .map(|id| unit_template::infer_unit_templates(&table, id)); + if let Some(outcome) = &inference { + notes.extend(outcome.diagnostics.iter().cloned()); + } + let policy = inference + .as_ref() + .and_then(|outcome| outcome.policy.clone()); + validate_merge_policy(request.existing_creative, policy.as_ref(), request.replace)?; + + // Slots that are one placement wearing a per-render div id cannot be + // written: the ids never match at runtime. Report them so the operator can + // add the placement once with a prefix they know is stable. + let fragmented = table.fragmented_slots(); + for group in &fragmented { + let suggestion = group.suggested_prefix.as_deref().map_or_else( + || "no stable prefix was shared".to_string(), + |prefix| format!("they share the prefix `{prefix}`"), + ); + notes.push(format!( + "skipped {} slot(s) that look like one placement under a per-render div id on \ + `{}` ({}); {suggestion}. Add it once by hand with a div_id prefix that is \ + stable across renders", + group.div_ids.len(), + group.unit_path, + group.div_ids.join(", "), + )); + } + + let slots = build_render_slots( + &table, + inference.as_ref(), + policy.as_ref(), + request, + plan.section_segment, + &fragmented, + &mut notes, + )?; + let observed_div_ids = table + .observed_div_ids() + .map(str::to_string) + .collect::>(); + let observed_literals = table + .observed_literals() + .map(str::to_string) + .collect::>(); + let (merged, merge_diagnostics) = slot_toml::merge_render_slots_with_observed_diagnostics( + request.existing_creative, + slots, + &observed_div_ids, + &observed_literals, + request.replace, + ); + notes.extend(merge_diagnostics.notes); + if !merge_diagnostics.unobserved_existing_slot_ids.is_empty() { + let slot_ids = merge_diagnostics.unobserved_existing_slot_ids.join(", "); + let follow_up = if request.scroll { + "Re-run with broader page/profile coverage; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." + } else { + "Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." + }; + notes.push(format!( + "preserved {} configured slot(s) not observed during this crawl: {slot_ids}. {follow_up}", + merge_diagnostics.unobserved_existing_slot_ids.len(), + )); + } + if merged.is_empty() { + emit_notes(err, &mut notes)?; + return cli_error( + "refusing to write zero generated slots after the crawl discovered slot evidence; review the refused-slot notes and keep the existing configuration", + ); + } + let rendered_slots = render_slots(&merged); + let updated = splice_creative_slots( + &existing, + &slot_toml::CreativeSectionKeys { + network_id: network_id.as_deref(), + section_root: policy.as_ref().map(|policy| policy.section_root.as_str()), + section_segment: policy.as_ref().map(|policy| policy.section_segment), + }, + &rendered_slots, + )?; + + // Everything above is derived from a live, page-controlled ad stack, so the + // candidate has to clear the runtime's own load path before it can replace + // the operator's file. This runs on the dry-run path too — otherwise "the + // preview looked fine" would not be evidence that the config loads. + notes.extend(validate::check_candidate(&updated, &existing)?); + + emit_notes(err, &mut notes)?; + if policy.is_some() { + writeln!( + err, + "note: this config now uses a {{section}} ad-unit template. Deploy a \ + template-aware binary BEFORE pushing it, and do not roll that binary \ + back while this config is live — an older binary rejects the whole \ + config and serves an error on every route." + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + + if request.dry_run { + let old_managed = managed_creative_projection(&existing)?; + let new_managed = managed_creative_projection(&updated)?; + if old_managed == new_managed { + // Stdout is the diff surface, so an English sentence there would + // break a redirected `--dry-run`; an empty diff is the stdout answer. + writeln!(err, "No managed creative-opportunity changes.").map_err(|error| { + report_error(format!("failed to write preview output: {error}")) + })?; + return Ok(()); + } + let diff = similar::TextDiff::from_lines(&old_managed, &new_managed); + writeln!( + out, + "{}", + diff.unified_diff().context_radius(0).header( + "configured creative opportunities", + "generated creative opportunities" + ) + ) + .map_err(|error| report_error(format!("failed to write preview diff: {error}")))?; + return Ok(()); + } + let current = fs::read_to_string(request.config_path).map_err(|error| { + report_error(format!( + "failed to re-read config {} before writing: {error}", + request.config_path.display() + )) + })?; + if current != existing { + return cli_error(format!( + "refusing to overwrite {} because it changed during the browser audit; re-run against the current file", + request.config_path.display() + )); + } + // A writer could still land between this check and the rename below. That + // window is microseconds against a browser crawl's minutes, and the rename + // is atomic, so the loser of the race loses a whole write rather than half + // of one. Closing it properly would need file locking the operator's editor + // does not take part in. + write_file_atomically(request.config_path, &updated).map_err(|error| { + report_error(format!( + "failed to write config {}: {error}", + request.config_path.display() + )) + })?; + writeln!( + out, + "Wrote {} slot(s) to {} ({} slot(s) seen across {} page(s))", + merged.len(), + request.config_path.display(), + table.slot_count(), + table.pages().len(), + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +/// Renders only fields managed by ad-template generation, excluding secrets and +/// unrelated operator configuration from dry-run output. +fn managed_creative_projection(document: &str) -> CliResult { + let value = toml::from_str::(document).map_err(|error| { + report_error(format!("failed to parse config for dry-run diff: {error}")) + })?; + let creative = value + .get("creative_opportunities") + .and_then(toml::Value::as_table); + let mut managed = toml::map::Map::new(); + if let Some(creative) = creative { + for key in ["gam_network_id", "section_root", "section_segment", "slot"] { + if let Some(value) = creative.get(key) { + managed.insert(key.to_string(), value.clone()); + } + } + } + let mut root = toml::map::Map::new(); + root.insert( + "creative_opportunities".to_string(), + toml::Value::Table(managed), + ); + toml::to_string_pretty(&toml::Value::Table(root)) + .map_err(|error| report_error(format!("failed to render dry-run projection: {error}"))) +} + +/// A page carrying fewer scripts than this is not a real publisher page. +/// +/// A production page runs dozens: the ad stack, analytics, consent, and the +/// site's own bundles. A bot-protection interstitial runs its own challenge +/// script and little else. +const INTERSTITIAL_SCRIPT_CEILING: usize = 3; + +/// Whether a page that loaded successfully is nonetheless not the real page. +/// +/// Bot protection commonly answers with **200** and a challenge document rather +/// than a 4xx, so status-code checks pass and the page simply appears to have no +/// ad stack. Left unexplained, that is indistinguishable from a publisher who +/// genuinely runs no ads on that page — and the operator's next move is entirely +/// different in each case. +fn looks_like_an_interstitial(artifact: &AuditArtifact) -> Option { + if artifact.js_asset_count > INTERSTITIAL_SCRIPT_CEILING + || !artifact.detected_integrations.is_empty() + { + return None; + } + Some(format!( + "the page returned successfully but carried only {} script(s) and no recognised \ + integrations, which is the shape of a bot-protection challenge rather than the \ + real page. Supply a current --cookie for the origin", + artifact.js_asset_count + )) +} + +/// Writes and clears the pending notes, so each is reported exactly once. +fn emit_notes(out: &mut dyn Write, notes: &mut Vec) -> CliResult<()> { + for note in notes.drain(..) { + writeln!( + out, + "note: {}", + crate::ad_templates::output::escape_terminal_text(¬e) + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + Ok(()) +} + +/// Writes one immediately visible, profile-aware crawl progress line. +fn write_collection_progress( + out: &mut dyn Write, + profile_label: &str, + progress: collector::CollectionProgress<'_>, +) -> CliResult<()> { + let line = match progress { + collector::CollectionProgress::Launching => { + format!("Auditing {profile_label}: launching browser") + } + collector::CollectionProgress::Loading { + current, + total, + url, + } => { + let path = if url.path().is_empty() { + "/" + } else { + url.path() + }; + let path = crate::ad_templates::output::escape_terminal_text(path); + let total = total.map_or_else(|| "?".to_string(), |total| total.to_string()); + format!("Auditing {profile_label} [{current}/{total}]: {path}") + } + collector::CollectionProgress::Planning => { + format!("Auditing {profile_label}: planning site crawl") + } + collector::CollectionProgress::Finalizing => { + format!("Auditing {profile_label}: finalizing browser session") + } + }; + writeln!(out, "{line}") + .map_err(|error| report_error(format!("failed to write audit progress: {error}")))?; + out.flush() + .map_err(|error| report_error(format!("failed to flush audit progress: {error}"))) +} + +struct CollectionProgressWriter<'a> { + out: &'a mut dyn Write, + profile_label: &'a str, +} + +impl CollectionProgressWriter<'_> { + fn write(&mut self, progress: collector::CollectionProgress<'_>) -> CliResult<()> { + write_collection_progress(self.out, self.profile_label, progress) + } +} + +/// Discovers a collected page's slots and folds them into `table`. +/// +/// Per-page collector warnings are appended to `notes`. They carry the reason a +/// page came back without slots — a non-2xx main document, a navigation that +/// never settled — which is the difference between "this publisher has no ad +/// stack here" and "bot protection served a challenge". Dropping them leaves +/// the operator with a refusal and no way to act on it. +fn fold_collected( + table: &mut evidence::EvidenceTable, + url: &Url, + collected: &collector::CollectedPage, + profile_label: &str, + notes: &mut Vec, +) -> CliResult<()> { + // `analyze_collected_page` already carries the collector's warnings forward, + // so this is the complete set, not a second copy. + let artifact = analyze_collected_page(collected)?; + for warning in &artifact.warnings { + // The consent stub is a property of the run, not of this page. Scoping it + // to a path and repeating it per page and profile buries the per-page + // diagnostics an operator is reading these notes for. + let note = if warning == collector::CONSENT_STUB_WARNING { + warning.clone() + } else { + format!("`{}` on {profile_label}: {warning}", url.path()) + }; + if !notes.contains(¬e) { + notes.push(note); + } + } + if let Some(reason) = looks_like_an_interstitial(&artifact) { + notes.push(format!("`{}` on {profile_label}: {reason}", url.path())); + } + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let discovered = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + for warning in &discovered.warnings { + if !notes.contains(warning) { + notes.push(warning.clone()); + } + } + table.fold_page(url.path(), &discovered); + Ok(()) +} + +/// Walks the planned section pages, folding each into `table`. +/// +/// A page that fails to collect is recorded as a note rather than aborting: on a +/// multi-section crawl one blocked or slow page should not discard the sections +/// that did work. The empty-page guard afterwards catches the case where enough +/// of them failed that the result is untrustworthy. +fn crawl_sections( + collector: &dyn AuditCollector, + root_url: &Url, + plan: &crawl_plan::CrawlPlan, + cookies: &[(String, String)], + table: &mut evidence::EvidenceTable, + notes: &mut Vec, + progress_writer: &mut CollectionProgressWriter<'_>, +) -> CliResult { + let additional_targets = plan.targets(); + if additional_targets.is_empty() { + notes.push( + "no additional site sections were discovered, so only the requested page was \ + audited; pass explicit --page-pattern values or more URLs to widen coverage" + .to_string(), + ); + } + // The root is deliberately part of every profile's shared batch: browser + // clearance/session state established there then carries into section pages. + let mut targets = Vec::with_capacity(additional_targets.len() + 1); + targets.push(root_url.clone()); + targets.extend(additional_targets); + + let mut fold_error = None; + let mut successful_pages = 0_usize; + { + let profile_label = progress_writer.profile_label; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + collector.collect_pages( + &targets, + cookies, + &mut report_progress, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + // Same boundary as the first profile, and it covers this + // profile's root page too: a cross-origin redirect is not + // a page this run may learn inventory from, so it must + // not count towards profile coverage either. + if origin_changed(root_url, &final_url) { + notes.push(format!( + "skipped `{}` on {profile_label}: it left the audited origin for {}", + url.path(), + final_url.origin().ascii_serialization() + )); + return Ok(collector::ControlFlow::Continue); + } + successful_pages += 1; + if let Err(error) = + fold_collected(table, &final_url, &page, profile_label, notes) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + notes.push(format!( + "skipped `{}` on {profile_label}: {error}", + url.path() + )); + } + } + Ok(collector::ControlFlow::Continue) + }, + )?; + } + match fold_error { + Some(error) => Err(error), + None => Ok(successful_pages), + } +} + +/// Refuses a crawl where too many pages produced no slots. +fn guard_challenge_rate(table: &evidence::EvidenceTable) -> CliResult<()> { + let total = table.pages().len(); + let empty = table.empty_pages().len(); + if total == 0 || (empty as f64) <= (total as f64) * MAX_EMPTY_PAGE_SHARE { + return Ok(()); + } + let blocked: Vec<&str> = table.empty_pages().iter().map(String::as_str).collect(); + cli_error(format!( + "{empty} of {total} crawled page(s) produced no ad slots ({}), which usually means \ + bot protection served a challenge instead of the real page. Refusing to write a \ + config from partial evidence; re-run with a valid --cookie for the origin", + blocked.join(", ") + )) +} + +/// Refuses a merge that would reinterpret templated slots the config already has. +/// +/// # Errors +/// +/// Returns an error when preserved `{section}` slots were written against a +/// different section policy than this run inferred, since the merge would leave +/// them pointing at ad units nobody configured. +fn validate_merge_policy( + existing: Option<&CreativeOpportunitiesConfig>, + inferred: Option<&unit_template::SectionPolicy>, + replace: bool, +) -> CliResult<()> { + if replace { + return Ok(()); + } + let Some(existing) = existing else { + return Ok(()); + }; + let preserves_template = existing.slot.iter().any(|slot| { + slot.gam_unit_path + .as_deref() + .is_some_and(|path| path.contains("{section}")) + }); + let Some(inferred) = inferred.filter(|_| preserves_template) else { + return Ok(()); + }; + if let Some(configured_segment) = existing.section_segment + && configured_segment != inferred.section_segment + { + return cli_error(format!( + "refusing to change the section_segment used by preserved templated slots during merge: configured section_segment={configured_segment}; inferred section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_segment + )); + } + // A `{section}` slot with no `section_root` cannot load at all — + // `validate_runtime` requires one — so there is no root value to preserve. + // Adopting the inferred root makes such a config loadable, provided the + // independently configured section segment above still agrees. + let Some(configured_root) = existing + .section_root + .as_deref() + .filter(|root| !root.is_empty()) + else { + return Ok(()); + }; + let configured_segment = existing.section_segment.unwrap_or(0); + if configured_root != inferred.section_root || configured_segment != inferred.section_segment { + return cli_error(format!( + "refusing to change the section policy used by preserved templated slots during merge: configured section_root={configured_root:?}, section_segment={configured_segment}; inferred section_root={:?}, section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_root, inferred.section_segment + )); + } + Ok(()) +} + +/// Turns the evidence table into slots ready to render. +fn build_render_slots( + table: &evidence::EvidenceTable, + inference: Option<&unit_template::InferenceOutcome>, + policy: Option<&unit_template::SectionPolicy>, + request: &UpdateSlotsRequest<'_>, + fallback_section_segment: usize, + fragmented: &[evidence::FragmentGroup], + notes: &mut Vec, +) -> CliResult> { + let skip: std::collections::BTreeSet<&str> = fragmented + .iter() + .flat_map(|group| group.div_ids.iter().map(String::as_str)) + .collect(); + // Explicit `--page-pattern` values are an operator override: they apply to + // every slot and disable inference from observed paths entirely. + let explicit = !request.page_patterns.is_empty(); + if explicit { + validate_page_patterns(request.page_patterns)?; + // Not filtered against `skip`: a borrowed root implies the slot's + // ad-unit path varied across pages, and `fragmented_slots` only groups + // slots pinned to exactly one unit path, so the two sets are disjoint. + if let Some(outcome) = inference + && !outcome.borrowed_section_root.is_empty() + { + let affected = outcome + .borrowed_section_root + .iter() + .map(|stem| format!("`{stem}`")) + .collect::>() + .join(", "); + return cli_error(format!( + "cannot apply --page-pattern to slot(s) with div id(s) {affected} because their \ + {{section}} templates borrow section_root; remove --page-pattern so patterns \ + can be derived from the paths where each slot was observed" + )); + } + } + let section_segment = policy.map_or(fallback_section_segment, |policy| policy.section_segment); + + let mut slots = Vec::with_capacity(table.slot_count()); + for slot in table.slots() { + if skip.contains(slot.div_id.as_str()) { + continue; + } + let patterns = if explicit { + request.page_patterns.to_vec() + } else { + let derived = page_patterns::patterns_for_paths(slot.paths(), section_segment); + validate_page_patterns(&derived)?; + derived + }; + let unit_path = match inference.and_then(|outcome| outcome.decision(&slot.div_id)) { + Some(unit_template::SlotDecision::Template(template)) => Some(template.clone()), + Some(unit_template::SlotDecision::Literal(path)) => Some(path.clone()), + Some(unit_template::SlotDecision::Refuse { reasons }) => { + notes.push(format!( + "skipped refused slot `{}` (`{}`): {}", + slot.id, + slot.div_id, + reasons.join("; ") + )); + continue; + } + None => None, + }; + slots.push(slot_toml::RenderSlot::from_evidence( + &slot.id, + &slot.div_id, + unit_path, + slot.formats.iter().copied(), + patterns, + slot.has_prebid, + )); + } + Ok(slots) +} +/// Rejects any page pattern the runtime's glob compiler would not accept. +/// +/// Uses [`validate_page_pattern`] so the accepted set is exactly what +/// `CreativeOpportunitySlot::compile_patterns` accepts at startup, including the +/// `**`→`*` normalisation. All patterns are reported at once so an operator +/// passing several `--page-pattern` values fixes them in one pass. +/// +/// # Errors +/// +/// Returns a user-facing error listing every pattern that does not compile. +fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { + let invalid: Vec = patterns + .iter() + .filter_map(|pattern| validate_page_pattern(pattern).err()) + .collect(); + if invalid.is_empty() { + return Ok(()); + } + cli_error(format!( + "refusing to write invalid page pattern(s): {}", + invalid.join("; ") + )) +} + +#[cfg(test)] +mod tests { + use std::cell::{Cell, RefCell}; + use std::io; + use std::rc::Rc; + + use tempfile::TempDir; + + use super::*; + use crate::app_config::AppConfigArgs; + use crate::commands::audit::generate::collector::{ + CollectedPage, CollectedRequest, CollectedScriptTag, + }; + use crate::commands::config::init::EXAMPLE_CONFIG; + + struct FakeCollector { + collected: CollectedPage, + calls: Cell, + } + + struct MutatingCollector { + collected: CollectedPage, + config_path: std::path::PathBuf, + replacement: String, + } + + impl AuditCollector for MutatingCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + fs::write(&self.config_path, &self.replacement) + .map_err(|error| report_error(format!("failed to mutate test config: {error}")))?; + Ok(self.collected.clone()) + } + } + + impl FakeCollector { + fn new(collected: CollectedPage) -> Self { + Self { + collected, + calls: Cell::new(0), + } + } + } + + impl AuditCollector for FakeCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.calls.set(self.calls.get() + 1); + Ok(self.collected.clone()) + } + } + + /// A collector serving a distinct page per URL, recording the crawl order. + struct SiteCollector { + pages: std::collections::HashMap, + visited: std::cell::RefCell>, + } + + struct FailingCollector; + + #[derive(Clone, Default)] + struct SharedProgressState { + bytes: Rc>>, + flushes: Rc>, + } + + struct SharedProgressWriter { + state: SharedProgressState, + } + + impl Write for SharedProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.state.bytes.borrow_mut().extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.state.flushes.set(self.state.flushes.get() + 1); + Ok(()) + } + } + + struct ObservingProgressCollector { + collected: CollectedPage, + state: SharedProgressState, + saw_flushed_progress: Cell, + } + + impl AuditCollector for ObservingProgressCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + Ok(self.collected.clone()) + } + + fn collect_site( + &self, + root: &Url, + _cookies: &[(String, String)], + on_progress: collector::ProgressSink<'_>, + planner: collector::RootPlanner<'_>, + on_page: collector::PageSink<'_>, + ) -> CliResult<()> { + on_progress(collector::CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + self.saw_flushed_progress + .set(!self.state.bytes.borrow().is_empty() && self.state.flushes.get() > 0); + on_progress(collector::CollectionProgress::Planning)?; + let _ = planner(root, &self.collected)?; + let _ = on_page(root, Ok(self.collected.clone()))?; + Ok(()) + } + } + + #[derive(Default)] + struct ProgressWriter { + bytes: Vec, + flushes: usize, + fail_write: bool, + fail_flush: bool, + } + + impl Write for ProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + if self.fail_write { + return Err(io::Error::other("simulated progress write failure")); + } + self.bytes.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + if self.fail_flush { + return Err(io::Error::other("simulated progress flush failure")); + } + Ok(()) + } + } + + #[test] + fn progress_lines_are_profile_aware_and_flush_immediately() { + let url = + Url::parse("https://user:pass@publisher.example/news\u{1b}[31m?token=secret#fragment") + .expect("should parse progress URL"); + let mut writer = ProgressWriter::default(); + + for progress in [ + collector::CollectionProgress::Launching, + collector::CollectionProgress::Loading { + current: 1, + total: None, + url: &url, + }, + collector::CollectionProgress::Planning, + collector::CollectionProgress::Loading { + current: 2, + total: Some(17), + url: &url, + }, + collector::CollectionProgress::Finalizing, + ] { + write_collection_progress(&mut writer, "desktop", progress) + .expect("should write progress"); + } + + let rendered = String::from_utf8(writer.bytes).expect("should render UTF-8 progress"); + assert_eq!( + rendered, + "Auditing desktop: launching browser\n\ + Auditing desktop [1/?]: /news%1B[31m\n\ + Auditing desktop: planning site crawl\n\ + Auditing desktop [2/17]: /news%1B[31m\n\ + Auditing desktop: finalizing browser session\n" + ); + assert_eq!(writer.flushes, 5, "should flush every progress line"); + assert!(!rendered.contains("user"), "should omit URL userinfo"); + assert!(!rendered.contains("secret"), "should omit URL query values"); + assert!(!rendered.contains("fragment"), "should omit URL fragments"); + assert!( + !rendered.contains('\u{1b}'), + "should not emit terminal escapes" + ); + } + + #[test] + fn progress_write_and_flush_failures_are_reported() { + let mut write_failure = ProgressWriter { + fail_write: true, + ..ProgressWriter::default() + }; + let write_error = write_collection_progress( + &mut write_failure, + "desktop", + collector::CollectionProgress::Launching, + ) + .expect_err("should report progress write failure"); + assert!(format!("{write_error:?}").contains("failed to write audit progress")); + + let mut flush_failure = ProgressWriter { + fail_flush: true, + ..ProgressWriter::default() + }; + let flush_error = write_collection_progress( + &mut flush_failure, + "desktop", + collector::CollectionProgress::Finalizing, + ) + .expect_err("should report progress flush failure"); + assert!(format!("{flush_error:?}").contains("failed to flush audit progress")); + } + + #[test] + fn update_slots_flushes_progress_before_collection_returns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let state = SharedProgressState::default(); + let collector = ObservingProgressCollector { + collected: collected_page_with_header_slot(), + state: state.clone(), + saw_flushed_progress: Cell::new(false), + }; + let mut progress_writer = SharedProgressWriter { state }; + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut progress_writer, + ) + .expect("should generate slots"); + + assert!( + collector.saw_flushed_progress.get(), + "collector should observe flushed progress before returning" + ); + assert!( + !String::from_utf8(out) + .expect("should write UTF-8 output") + .contains("Auditing "), + "stdout should not contain progress" + ); + } + + impl AuditCollector for FailingCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + cli_error(format!("simulated navigation failure for {target_url}")) + } + } + + impl SiteCollector { + fn new(pages: Vec<(&str, CollectedPage)>) -> Self { + Self { + pages: pages + .into_iter() + .map(|(url, page)| (url.to_string(), page)) + .collect(), + visited: std::cell::RefCell::new(Vec::new()), + } + } + } + + impl AuditCollector for SiteCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.visited.borrow_mut().push(target_url.to_string()); + self.pages + .get(target_url.as_str()) + .cloned() + .ok_or_else(|| report_error(format!("no fake page for {target_url}"))) + } + } + + /// Builds a page carrying one GPT slot plus same-origin nav links. + fn site_page(url: &str, unit_path: &str, nav_paths: &[&str]) -> CollectedPage { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: unit_path.to_string(), + div_id: "ad-header-0".to_string(), + sizes: vec![(728, 90)], + }]; + page.links = nav_paths + .iter() + .map(|path| collector::CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + }) + .collect(); + page + } + + fn collected_page() -> CollectedPage { + CollectedPage { + requested_url: "https://publisher.example/page".to_string(), + final_url: "https://publisher.example/page".to_string(), + page_title: Some("Example Publisher".to_string()), + html: r#"Example Publisher"#.to_string(), + script_tags: vec![ + CollectedScriptTag { + src: Some("https://www.googletagmanager.com/gtm.js?id=GTM-ABC123".to_string()), + inline_text: None, + }, + CollectedScriptTag { + src: Some("https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string()), + inline_text: None, + }, + ], + network_requests: vec![CollectedRequest { + url: "https://cdn.publisher.example/app.js".to_string(), + resource_type: Some("script".to_string()), + }], + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + } + } + + /// A collected page carrying one discoverable GPT slot, for `run_update_slots`. + fn collected_page_with_header_slot() -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + collected + } + + fn collected_page_with_ambiguous_slots(url: &str) -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = url.to_string(); + collected.final_url = url.to_string(); + collected.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-aaaaaaaaaaaaaaaa-0".to_string(), + sizes: vec![(300, 250)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-bbbbbbbbbbbbbbbb-1".to_string(), + sizes: vec![(300, 250)], + }, + ]; + collected + } + + fn audited_asset(url: &str, party: AssetParty, integration: Option<&str>) -> AuditedAsset { + AuditedAsset { + kind: "script".to_string(), + url: url.to_string(), + host: Url::parse(url) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_string)) + .unwrap_or_default(), + party, + integration: integration.map(str::to_string), + } + } + + fn audit_args(url: &str) -> GenerateArgs { + GenerateArgs { + url: url.to_string(), + js_assets: None, + config: None, + no_js_assets: false, + no_config: false, + force: false, + cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), + } + } + + #[test] + fn parse_audit_url_accepts_http_and_https() { + assert!(parse_audit_url("http://publisher.example").is_ok()); + assert!(parse_audit_url("https://publisher.example").is_ok()); + } + + #[test] + fn parse_audit_url_rejects_non_http_schemes() { + for url in [ + "file:///etc/passwd", + "data:text/html,hello", + "chrome://version", + ] { + let error = parse_audit_url(url).expect_err("should reject non-http URL"); + assert!( + format!("{error:?}").contains("only supports http/https"), + "should explain scheme restriction" + ); + } + } + + #[test] + fn repeated_ambiguous_collision_note_is_emitted_once() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &collected_page_with_ambiguous_slots(url), + "desktop", + &mut notes, + ) + .expect("should fold ambiguous page evidence"); + } + + assert_eq!( + notes.len(), + 1, + "the same site-wide collision guidance should not repeat per page" + ); + } + + #[test] + fn merge_refuses_to_change_policy_used_by_preserved_templates() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_root = \"home\"\nsection_segment = 0\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + let error = validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect_err("merge must preserve the existing template policy"); + + assert!(format!("{error:?}").contains("--replace")); + validate_merge_policy(Some(&existing), Some(&inferred), true) + .expect("replace is an explicit policy migration"); + } + + #[test] + fn the_consent_stub_note_is_reported_once_and_unscoped() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.warnings + .push(collector::CONSENT_STUB_WARNING.to_string()); + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &page, + "desktop", + &mut notes, + ) + .expect("should fold page evidence"); + } + + assert_eq!( + notes, + [collector::CONSENT_STUB_WARNING.to_string()], + "a run-wide fact should appear once, without a page path" + ); + } + + #[test] + fn page_warnings_remain_distinct_across_profiles() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + let mut page = collected_page(); + page.requested_url = "https://publisher.example/news".to_string(); + page.final_url = page.requested_url.clone(); + page.warnings.push("navigation did not settle".to_string()); + let url = Url::parse(&page.final_url).expect("should parse fixture URL"); + + fold_collected(&mut table, &url, &page, "desktop", &mut notes) + .expect("should fold desktop evidence"); + fold_collected(&mut table, &url, &page, "mobile", &mut notes) + .expect("should fold mobile evidence"); + + assert_eq!( + notes.len(), + 2, + "profile-specific warnings must not collapse" + ); + assert!(notes.iter().any(|note| note.contains("on desktop"))); + assert!(notes.iter().any(|note| note.contains("on mobile"))); + } + + #[test] + fn merge_adopts_the_inferred_policy_when_none_is_configured() { + // A hand-written `{section}` slot with no `section_root` describes a + // config the runtime refuses to load, so the first merge should repair it + // rather than demand `--replace` (which would discard the hand-tuned + // slots it is preserving). + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect("should have no policy to preserve when section_root is unset"); + } + + #[test] + fn merge_preserves_an_explicit_segment_when_section_root_is_unset() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_segment = 1\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let mismatched = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }; + + let error = validate_merge_policy(Some(&existing), Some(&mismatched), false) + .expect_err("should preserve an explicitly configured segment"); + + assert!(format!("{error:?}").contains("section_segment=1")); + + let matching = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + validate_merge_policy(Some(&existing), Some(&matching), false) + .expect("should adopt a root without changing the configured segment"); + } + + #[test] + fn resolve_output_plan_rejects_no_outputs() { + let mut args = audit_args("https://publisher.example"); + args.no_js_assets = true; + args.no_config = true; + + let error = resolve_output_plan(&args).expect_err("should reject empty output set"); + + assert!( + format!("{error:?}").contains("nothing to do"), + "should explain no-output error" + ); + } + + #[test] + fn resolve_output_plan_rejects_existing_files_without_force() { + let temp = TempDir::new().expect("should create temp dir"); + let path = temp.path().join("js-assets.toml"); + fs::write(&path, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example"); + args.js_assets = Some(path); + args.no_config = true; + + let error = resolve_output_plan(&args).expect_err("should reject overwrite"); + + assert!( + format!("{error:?}").contains("refusing to overwrite"), + "should explain overwrite refusal" + ); + } + + #[test] + fn resolve_output_plan_allows_existing_files_with_force() { + let temp = TempDir::new().expect("should create temp dir"); + let path = temp.path().join("js-assets.toml"); + fs::write(&path, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example"); + args.js_assets = Some(path.clone()); + args.no_config = true; + args.force = true; + + let plan = resolve_output_plan(&args).expect("should allow forced overwrite"); + + assert_eq!(plan.js_assets_path.as_deref(), Some(path.as_path())); + } + + #[test] + fn run_generate_writes_selected_outputs_and_summary() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("audit/js-assets.toml"); + let config = temp.path().join("audit/trusted-server.toml"); + let args = GenerateArgs { + url: "https://publisher.example/page".to_string(), + js_assets: Some(js_assets.clone()), + config: Some(config.clone()), + no_js_assets: false, + no_config: false, + force: false, + cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), + }; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + assert_eq!(collector.calls.get(), 1, "should collect page once"); + assert!(js_assets.exists(), "should write JS assets"); + assert!(config.exists(), "should write draft config"); + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("Audited https://publisher.example/page")); + assert!(summary.contains("Detected integrations: google_tag_manager, gpt")); + assert!(summary.contains("Draft config: review before validation and push")); + } + + #[test] + fn run_generate_respects_no_config() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets.clone()); + args.no_config = true; + let collector = FakeCollector::new(collected_page()); + + run_generate(&args, &collector, &mut Vec::new()).expect("should run audit"); + + assert!(js_assets.exists(), "should write assets"); + assert!( + !temp.path().join("trusted-server.toml").exists(), + "should not write config" + ); + } + + #[test] + fn run_generate_respects_no_js_assets() { + let temp = TempDir::new().expect("should create temp dir"); + let config = temp.path().join("trusted-server.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.config = Some(config.clone()); + args.no_js_assets = true; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + assert!(config.exists(), "should write config"); + assert!( + !temp.path().join("js-assets.toml").exists(), + "should not write JS assets" + ); + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("Draft config: review before validation and push")); + } + + #[test] + fn run_generate_writes_collector_warnings_to_asset_artifact() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets.clone()); + args.no_config = true; + let mut collected = collected_page(); + collected.warnings.push( + "browser audit timed out while waiting for the page to settle; results may be partial" + .to_string(), + ); + let collector = FakeCollector::new(collected); + + run_generate(&args, &collector, &mut Vec::new()).expect("should run audit"); + + let artifact = fs::read_to_string(js_assets).expect("should read artifact"); + assert!( + artifact.contains("results may be partial"), + "should persist collector warning" + ); + } + + #[test] + fn run_generate_conflict_prevents_collection() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + fs::write(&js_assets, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets); + args.no_config = true; + let collector = FakeCollector::new(collected_page()); + + let error = run_generate(&args, &collector, &mut Vec::new()) + .expect_err("should reject existing output"); + + assert_eq!(collector.calls.get(), 0, "should not collect page"); + assert!( + format!("{error:?}").contains("refusing to overwrite"), + "should report overwrite conflict" + ); + } + + struct FixedPathGenerator { + paths: std::collections::VecDeque, + } + + impl FixedPathGenerator { + fn new(paths: &[&str]) -> Self { + Self { + paths: paths.iter().map(|path| (*path).to_string()).collect(), + } + } + } + + impl OpaqueAssetPathGenerator for FixedPathGenerator { + fn next_path(&mut self) -> String { + self.paths + .pop_front() + .expect("should have a fixed generated asset path") + } + } + + #[test] + fn build_draft_config_writes_disabled_js_asset_proxy_candidates() { + let url = Url::parse("https://publisher.example.com/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: Some("Example".to_string()), + js_asset_count: 2, + third_party_asset_count: 2, + detected_integrations: vec![DetectedIntegration { + id: "gpt".to_string(), + evidence: "https://gpt.example.com/gpt.js".to_string(), + }], + assets: vec![ + audited_asset( + "https://cdn.vendor.example.com/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://gpt.example.com/gpt.js", + AssetParty::ThirdParty, + Some("gpt"), + ), + ], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&[ + "/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js", + "/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js", + ]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + assert_eq!( + draft.js_asset_proxy_candidate_count, 2, + "should report generated disabled entries" + ); + assert!( + draft + .toml + .contains("[integrations.js_asset_proxy]\nenabled = false") + ); + assert!(draft.toml.contains("/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js")); + assert!(draft.toml.contains("/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js")); + assert!( + draft + .toml + .contains("origin_url = \"https://cdn.vendor.example.com/sdk.js\"") + ); + assert!(draft.toml.contains("proxy = \"disabled\"")); + assert!(draft.toml.contains("Detected integration: gpt")); + assert!( + draft + .toml + .contains("Native integration may be preferable: [integrations.gpt]") + ); + assert!( + !draft.toml.contains("example-vendor-loader"), + "should remove starter-template placeholder asset" + ); + assert!( + draft.toml.contains( + "# Proxy behavior and first-party asset routing. Kept active with defaults.\n[proxy]" + ), + "should preserve documentation for the section following the replaced block" + ); + let parsed = + toml::from_str::(&draft.toml).expect("draft should parse as TOML"); + assert!( + parsed["integrations"]["js_asset_proxy"] + .get("cache_ttl_seconds") + .is_none(), + "generated config should inherit upstream cache headers by default" + ); + } + + #[test] + fn generated_asset_proxy_paths_are_opaque() { + let url = Url::parse("https://publisher.example.com/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 1, + detected_integrations: Vec::new(), + assets: vec![audited_asset( + "https://cdn.vendor.example.com/vendor-loader.js", + AssetParty::ThirdParty, + None, + )], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&["/assets/0123456789abcdef01234567.js"]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + let path_line = draft + .toml + .lines() + .find(|line| line.starts_with("path = ") && line.contains("0123456789abcdef")) + .expect("should include generated path"); + + assert!(path_line.contains("/assets/0123456789abcdef01234567.js")); + assert!( + !path_line.contains("vendor") + && !path_line.contains("cdn") + && !path_line.contains("loader"), + "generated path should not include vendor, domain, or filename semantics" + ); + } + + #[test] + fn asset_proxy_generation_deduplicates_and_summarizes_skips() { + let url = Url::parse("https://publisher.example.com/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 4, + third_party_asset_count: 3, + detected_integrations: Vec::new(), + assets: vec![ + audited_asset( + "https://cdn.vendor.example.com/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://cdn.vendor.example.com/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://publisher.example.com/app.js", + AssetParty::FirstParty, + None, + ), + audited_asset( + "http://cdn.vendor.example.com/insecure.js", + AssetParty::ThirdParty, + None, + ), + ], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&["/assets/111111111111111111111111.js"]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + assert_eq!(draft.js_asset_proxy_candidate_count, 1); + assert_eq!( + draft + .toml + .matches("[[integrations.js_asset_proxy.assets]]") + .count(), + 1, + "should only emit one candidate entry" + ); + assert!(draft.toml.contains("# - 1 first-party script")); + assert!(draft.toml.contains("# - 1 non-HTTPS third-party script")); + assert!(draft.toml.contains("# - 1 duplicate script URL")); + } + + #[test] + fn asset_proxy_generation_warns_about_query_string_candidates() { + let url = Url::parse("https://publisher.example.com/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 2, + third_party_asset_count: 2, + detected_integrations: Vec::new(), + assets: vec![ + audited_asset( + "https://cdn.vendor.example.com/sdk.js?v=one", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://cdn.vendor.example.com/sdk.js?v=two", + AssetParty::ThirdParty, + None, + ), + ], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&[ + "/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js", + "/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js", + ]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + assert_eq!(draft.js_asset_proxy_candidate_count, 2); + assert_eq!( + draft + .toml + .matches("This URL includes a query string and must remain stable") + .count(), + 2, + "each query-string candidate should explain exact-match behavior" + ); + } + + #[test] + fn asset_proxy_generation_with_no_candidates_removes_placeholder_asset() { + let url = Url::parse("https://publisher.example.com/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 0, + detected_integrations: Vec::new(), + assets: vec![audited_asset( + "https://publisher.example.com/app.js", + AssetParty::FirstParty, + None, + )], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&[]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + assert_eq!(draft.js_asset_proxy_candidate_count, 0); + assert!( + draft + .toml + .contains("No eligible third-party HTTPS script assets") + ); + assert!( + !draft + .toml + .contains("[[integrations.js_asset_proxy.assets]]"), + "should not emit asset array entries without candidates" + ); + assert!( + !draft.toml.contains("example-vendor-loader"), + "should remove starter-template placeholder asset" + ); + toml::from_str::(&draft.toml).expect("draft should parse as TOML"); + } + + #[test] + fn run_generate_summary_reports_written_asset_proxy_candidates() { + let temp = TempDir::new().expect("should create temp dir"); + let config = temp.path().join("trusted-server.toml"); + let mut args = audit_args("https://publisher.example.com/page"); + args.config = Some(config); + args.no_js_assets = true; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("JS asset proxy candidates:")); + assert!(summary.contains("disabled entries written to draft config")); + } + + #[test] + fn build_draft_config_uses_final_url_and_detected_integrations() { + let url = Url::parse("https://www.publisher.example:8443/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: Some("Example".to_string()), + js_asset_count: 2, + third_party_asset_count: 2, + detected_integrations: vec![ + DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "GTM-ABC123".to_string(), + }, + DetectedIntegration { + id: "gpt".to_string(), + evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), + }, + DetectedIntegration { + id: "prebid".to_string(), + evidence: "inline script matched `prebid`".to_string(), + }, + ], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + assert!(draft.contains("domain = \"www.publisher.example\"")); + assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); + assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); + assert!(draft.contains("[integrations.gpt]\nenabled = true")); + assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); + assert!(draft.contains("container_id = \"GTM-ABC123\"")); + assert!(draft.contains("Detected prebid")); + toml::from_str::(&draft).expect("draft should parse as TOML"); + } + + #[test] + fn build_draft_config_does_not_enable_gtm_without_container_id() { + let url = Url::parse("https://publisher.example/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 1, + detected_integrations: vec![DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "https://www.googletagmanager.com/gtm.js".to_string(), + }], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); + assert!(draft.contains("Detected google_tag_manager")); + } + + #[test] + fn build_audit_outputs_reconstructs_creative_opportunity_slots() { + let collected = CollectedPage { + requested_url: "https://example.com/".to_string(), + final_url: "https://example.com/".to_string(), + page_title: Some("Example Publisher".to_string()), + html: "".to_string(), + script_tags: Vec::new(), + network_requests: vec![CollectedRequest { + url: "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C620x366\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid" + .to_string(), + resource_type: Some("fetch".to_string()), + }], + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }; + + let outputs = build_audit_outputs(&collected).expect("should build outputs"); + assert_eq!(outputs.ad_slot_count, 1, "should discover one slot"); + + // The drafted config must be valid TOML with the reconstructed slot. + let value = toml::from_str::(&outputs.draft_config_toml) + .expect("should parse draft config"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("123456789")); + let slot = &creative["slot"][0]; + assert_eq!(slot["id"].as_str(), Some("leaderboard-1")); + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/123456789/desktop/homepage/leaderboard1") + ); + assert_eq!( + slot["formats"][0]["width"].as_integer(), + Some(970), + "should keep the 970x250 pixel size" + ); + assert!( + slot["providers"]["prebid"].is_table(), + "prev_scp test=prebid should emit a prebid provider" + ); + } + + #[test] + fn render_discovered_slots_escapes_page_controlled_strings() { + // Slot fields scraped from the live page must be escaped so a quote + // cannot inject TOML into the drafted config. + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/head\"er".to_string(), + div_id: "div-gpt-ad-head\"er".to_string(), + sizes: vec![(728, 90)], + }]; + let slots = gpt_slots::discover_gpt_slots(®istry, &[], false); + let url = Url::parse("https://publisher.example/").expect("should parse URL"); + + let rendered = render_discovered_slots(&url, &slots); + + let value = toml::from_str::(&rendered) + .expect("should render valid TOML despite embedded quotes"); + let slot = &value["creative_opportunities"]["slot"][0]; + assert_eq!( + slot["div_id"].as_str(), + Some("div-gpt-ad-head\"er"), + "should keep the quote as data, not TOML syntax" + ); + } + + #[test] + fn update_slots_defaults_pattern_to_final_url_after_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + // The requested URL redirects; slots are scraped from the final page. + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/news/story".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse valid TOML"); + let patterns: Vec<&str> = value["creative_opportunities"]["slot"][0]["page_patterns"] + .as_array() + .expect("should have page_patterns array") + .iter() + .map(|entry| entry.as_str().expect("should have pattern string")) + .collect(); + // Patterns come from the post-redirect path: had the requested `/` been + // used, this would be `["/"]`. They now cover the whole section rather + // than only the one article that happened to be scraped. + assert_eq!( + patterns, + ["/news", "/news/*"], + "should derive section patterns from the post-redirect path" + ); + } + + #[test] + fn update_slots_reports_preserved_unobserved_slots_contextually() { + for (scroll, expected_follow_up, unexpected_follow_up) in [ + (false, "or --scroll", "page/profile coverage"), + (true, "page/profile coverage", "or --scroll"), + ] { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config() + .replace("gam_network_id = \"123456789\"", "gam_network_id = \"222\""); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"sidebar\"\n\ + div_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut notes, + ) + .expect("should preserve unobserved slot"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains( + "preserved 1 configured slot(s) not observed during this crawl: sidebar" + ), + "should name the preserved slot, got {notes:?}" + ); + assert!( + notes.contains(expected_follow_up), + "should suggest the follow-up matching the scroll setting, got {notes:?}" + ); + assert!( + !notes.contains(unexpected_follow_up), + "should omit the follow-up that does not apply, got {notes:?}" + ); + assert!( + notes.contains("discards every hand-written field"), + "should explain the full cost of --replace, got {notes:?}" + ); + assert!(out.is_empty(), "unchanged dry-run stdout should stay empty"); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "dry-run should preserve the original config" + ); + } + } + + #[test] + fn observed_but_refused_slot_is_not_reported_as_unobserved() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config(); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"stable\"\n\ + div_id = \"ad-stable\"\n\ + gam_unit_path = \"/123456789/site/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-refused\"\n\ + gam_unit_path = \"/123456789/desktop/homepage\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + + let page = |profile: &str| { + let mut page = collected_page(); + page.requested_url = "https://publisher.example/".to_string(); + page.final_url = page.requested_url.clone(); + page.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }, + collector::CollectedGptSlot { + gam_unit_path: format!("/123456789/{profile}/homepage"), + div_id: "ad-refused".to_string(), + sizes: vec![(300, 250)], + }, + ]; + page + }; + let desktop = FakeCollector::new(page("desktop")); + let mobile = FakeCollector::new(page("mobile")); + let mut out = Vec::new(); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut notes, + ) + .expect("the accepted slot should let generation complete"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped refused slot `ad-refused` (`ad-refused`)"), + "should retain the refusal diagnostic, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: ad-refused"), + "a crawl-observed refused slot must not be labeled unobserved, got {notes:?}" + ); + } + + #[test] + fn ambiguous_configured_stem_is_not_reported_as_unobserved() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = + loadable_config().replace("gam_network_id = \"123456789\"", "gam_network_id = \"222\""); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"in-content\"\n\ + div_id = \"ad-x\"\n\ + gam_unit_path = \"/222/homepage/in-content\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let mut page = collected_page_with_ambiguous_slots("https://publisher.example/"); + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/222/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }); + let collector = FakeCollector::new(page); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: true, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("should preserve the configured ambiguous placement"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped ambiguous div-id prefix `ad-x`"), + "should retain the ambiguity diagnostic, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: in-content"), + "an ambiguity-refused placement must not be labeled unobserved, got {notes:?}" + ); + } + + #[test] + fn volatile_refusals_from_registry_and_requests_keep_prefix_observed() { + for source in ["registry", "request"] { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config(); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"stable\"\n\ + div_id = \"ad-stable\"\n\ + gam_unit_path = \"/123456789/site/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"volatile-family\"\n\ + div_id = \"vendor-tag\"\n\ + gam_unit_path = \"/123456789/site/overlay\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let mut page = collected_page(); + page.requested_url = "https://publisher.example/".to_string(); + page.final_url = page.requested_url.clone(); + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }); + let volatile_div = "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1"; + if source == "registry" { + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/overlay".to_string(), + div_id: volatile_div.to_string(), + sizes: vec![(300, 250)], + }); + } else { + page.network_requests.push(CollectedRequest { + url: format!( + "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Csite%2Coverlay&dids={volatile_div}\ + &prev_iu_szs=300x250" + ), + resource_type: Some("fetch".to_string()), + }); + } + let collector = FakeCollector::new(page); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: true, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("the stable slot should let generation complete"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped volatile div-id family `vendor-tag`"), + "should retain the {source} volatile refusal, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: volatile-family"), + "a live configured prefix refused from {source} evidence must stay observed, got {notes:?}" + ); + } + } + + #[test] + fn static_locale_root_slot_uses_the_planned_section_depth_for_patterns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/en/news"]; + let mut root_page = site_page("https://publisher.example/en", "/123456789/site/root", &nav); + root_page.gpt_slots[0].div_id = "ad-root-only".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/en", root_page), + ( + "https://publisher.example/en/news", + site_page( + "https://publisher.example/en/news", + "/123456789/site/static", + &nav, + ), + ), + ]); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/en", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect("should write static locale-root slot"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("should have slots"); + let target = slots + .iter() + .find(|slot| slot["div_id"].as_str() == Some("ad-header-0")) + .expect("should have the section slot"); + let patterns = target["page_patterns"] + .as_array() + .expect("should have patterns") + .iter() + .map(|pattern| pattern.as_str().expect("should be string")) + .collect::>(); + assert_eq!(patterns, ["/en/news", "/en/news/*"]); + } + + #[test] + fn update_slots_rejects_a_cross_origin_root_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "https://foreign.example/news".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("cross-origin redirect must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "foreign evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_skips_a_section_page_that_redirects_off_origin() { + // Only the root navigation was origin-checked before planning. A section + // page that redirects away must not contribute its slots, unit paths or + // page patterns to the generated config either. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/news"]; + let mut root_page = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + root_page.gpt_slots[0].div_id = "ad-root".to_string(); + let mut redirected = site_page( + "https://publisher.example/news", + "/999888777/foreign/news", + &nav, + ); + redirected.final_url = "https://foreign.example/news".to_string(); + redirected.gpt_slots[0].div_id = "ad-foreign".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/", root_page), + ("https://publisher.example/news", redirected), + ]); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + scroll: false, + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut err, + ) + .expect("should generate from the same-origin evidence alone"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + assert!( + written.contains("ad-root"), + "same-origin evidence should still be written, got:\n{written}" + ); + assert!( + !written.contains("ad-foreign") && !written.contains("999888777"), + "the redirect destination must not reach the config, got:\n{written}" + ); + let progress = String::from_utf8_lossy(&err); + assert!( + progress.contains( + "skipped `/news` on desktop: it left the audited origin for https://foreign.example" + ), + "the skipped section page should be reported, got:\n{progress}" + ); + } + + #[test] + fn update_slots_skips_a_later_profile_root_that_redirects_off_origin() { + // The later profiles re-walk the plan without a fresh root origin check. + // A mobile root that redirects away carries a foreign ad unit for the + // same div the desktop profile saw; folding it would both write foreign + // inventory and fake a device disagreement on the real slot. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/news"]; + let section_page = |unit_path: &str| { + let mut page = site_page("https://publisher.example/news", unit_path, &nav); + page.gpt_slots[0].div_id = "ad-news".to_string(); + page + }; + let mut desktop_root = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + desktop_root.gpt_slots[0].div_id = "ad-root".to_string(); + let mut mobile_root = site_page( + "https://publisher.example/", + "/999888777/foreign/homepage", + &nav, + ); + mobile_root.gpt_slots[0].div_id = "ad-root".to_string(); + mobile_root.final_url = "https://foreign.example/".to_string(); + let desktop = SiteCollector::new(vec![ + ("https://publisher.example/", desktop_root), + ( + "https://publisher.example/news", + section_page("/123456789/site/news"), + ), + ]); + let mobile = SiteCollector::new(vec![ + ("https://publisher.example/", mobile_root), + ( + "https://publisher.example/news", + section_page("/123456789/site/news"), + ), + ]); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + scroll: false, + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut std::io::sink(), + &mut err, + ) + .expect("the same-origin pages of both profiles agree"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + assert!( + written.contains("/123456789/site/homepage"), + "the same-origin root unit path should be written, got:\n{written}" + ); + assert!( + !written.contains("999888777"), + "the redirect destination must not reach the config, got:\n{written}" + ); + let progress = String::from_utf8_lossy(&err); + assert!( + progress.contains( + "skipped `/` on mobile: it left the audited origin for https://foreign.example" + ), + "the skipped profile root should be reported, got:\n{progress}" + ); + } + + #[test] + fn update_slots_accepts_a_same_host_https_upgrade() { + // The ordinary canonical redirect: an operator types the bare http URL + // and the site upgrades it. The host is unchanged, so the cookie and + // audit trust boundary is unchanged, and generation must not stall on it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.requested_url = "http://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "http://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("a same-host HTTPS upgrade should not be treated as cross-origin"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "evidence from the upgraded root should be written" + ); + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains( + "followed a root redirect from `http://publisher.example/` to \ + `https://publisher.example/`" + ), + "an accepted redirect should say the run switched URLs, got {notes:?}" + ); + } + + #[test] + fn update_slots_rejects_an_https_downgrade_root_redirect() { + // The mirror image of the accepted upgrade: same host, but dropping TLS + // leaves the requested trust boundary and must still be refused. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "http://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("an HTTPS downgrade must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "downgraded evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_requires_evidence_from_every_selected_profile() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + let desktop = FakeCollector::new(collected_page_with_header_slot()); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &FailingCollector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a selected profile with no usable page must refuse generation"); + + assert!(format!("{error:?}").contains("mobile")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "incomplete profile coverage must not rewrite the config" + ); + } + + #[test] + fn update_slots_rejects_invalid_page_pattern_without_touching_config() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["[".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("should reject an invalid glob"); + + assert!( + format!("{error:?}").contains("page pattern '['"), + "error should name the offending pattern, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a rejected pattern must leave the operator config untouched" + ); + } + + #[test] + fn explicit_page_patterns_refuse_a_template_that_borrows_section_root() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let root = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + let mut news = site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ); + news.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/news".to_string(), + div_id: "ad-sidebar".to_string(), + sizes: vec![(300, 250)], + }); + let mut deals = site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ); + deals.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/deals".to_string(), + div_id: "ad-sidebar".to_string(), + sizes: vec![(300, 250)], + }); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/", root), + ("https://publisher.example/news", news), + ("https://publisher.example/deals", deals), + ]); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/".to_string(), "/*".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("explicit patterns cannot preserve borrowed-root safety"); + + let message = format!("{error:?}"); + assert!(message.contains("--page-pattern"), "got {message}"); + assert!(message.contains("ad-sidebar"), "got {message}"); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a refused override must leave the config unchanged" + ); + } + + #[test] + fn update_slots_accepts_double_star_pattern_like_the_runtime() { + // `/20**` does not compile directly but the runtime normalises it to + // `/20*`; validation must accept exactly what the runtime accepts. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/20**".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should accept a runtime-normalisable pattern"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), + Some("/20**") + ); + } + + #[test] + fn update_slots_write_replaces_the_config_without_leaving_temp_files() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let entries: Vec = fs::read_dir(temp.path()) + .expect("should read temp dir") + .map(|entry| { + entry + .expect("should read entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!( + entries, + ["trusted-server.toml"], + "the atomic write should leave no stray temp file behind" + ); + let written = fs::read_to_string(&config_path).expect("should read config"); + toml::from_str::(&written).expect("rewritten config is valid TOML"); + } + + /// A config with fictional resolved secrets and non-placeholder publisher + /// values, so both source validation and runtime loading can be exercised. + fn loadable_config() -> String { + EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ) + .replace("\"example.com\"", "\"publisher.example.com\"") + .replace("\".example.com\"", "\".publisher.example.com\"") + .replace( + "https://origin.example.com", + "https://origin.publisher.example.com", + ) + } + + #[test] + fn a_crawl_writes_a_section_template_and_per_section_patterns() { + // The end-to-end payoff: crawl sections, reconcile the slot across them, + // infer `{section}`, and write a config the runtime loads. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ), + ), + ( + "https://publisher.example/deals", + site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut err, + ) + .expect("should crawl and update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "the unvisited-section fallback should come from the root page" + ); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + let slot = &creative["slot"][0]; + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/{network_id}/site/{section}"), + "the varying segment should become a template" + ); + let patterns: Vec<&str> = slot["page_patterns"] + .as_array() + .expect("patterns array") + .iter() + .map(|entry| entry.as_str().expect("pattern")) + .collect(); + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "each witnessed section should contribute both halves of its pair" + ); + + // The whole point of the gate: what was written must actually load. + trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + + let report = String::from_utf8(err).expect("should produce UTF-8 output"); + assert!( + report.contains("Deploy a template-aware binary BEFORE pushing"), + "a templated config must warn about the rollback contract, got:\n{report}" + ); + } + + #[test] + fn disagreeing_device_profiles_refuse_to_write_a_unit_path() { + // Two profiles serving different ad units for the same page is exactly + // the failure a single-profile crawl cannot see. Writing either path + // would be correct for one device and silently wrong for the other. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news"]; + let desktop = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/desktop/news", + &nav, + ), + ), + ]); + let mobile = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/mobile/news", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + let mut err = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut err, + ) + .expect_err("an all-refused crawl must not write an empty slot array"); + + assert!(format!("{error:?}").contains("zero generated slots")); + let progress = String::from_utf8_lossy(&err); + for expected in [ + "Auditing desktop [1/?]: /", + "Auditing desktop: planning site crawl", + "Auditing desktop [2/2]: /news", + "Auditing mobile [1/2]: /", + "Auditing mobile [2/2]: /news", + ] { + assert!( + progress.contains(expected), + "should report `{expected}` while crawling, got:\n{progress}" + ); + } + assert!( + !String::from_utf8_lossy(&out).contains("Auditing "), + "progress must remain on stderr" + ); + assert!( + progress.contains("skipped refused slot"), + "the refusal reason should be reported" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused crawl must preserve the operator config" + ); + } + + #[test] + fn a_root_only_site_is_still_collected_on_every_device_profile() { + // A site whose root offers no crawl targets is audited on the root page + // alone. If the later profiles never load it, a device split there is + // invisible and the first profile's literal path gets written as if + // every device agreed with it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let desktop = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &[], + ), + )]); + let mobile = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &[], + ), + )]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("an all-refused crawl must not write an empty slot array"); + + assert_eq!( + mobile.visited.borrow().as_slice(), + ["https://publisher.example/"], + "the mobile profile must load the root even when there is nothing else to crawl" + ); + assert!(format!("{error:?}").contains("zero generated slots")); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a root-only refusal must preserve the operator config" + ); + } + + #[test] + fn a_crawl_refuses_when_most_pages_are_challenged() { + // Bot protection serves an interstitial that loads fine and has no ad + // stack, so it looks like a page with no slots. Writing from that would + // silently narrow the operator's slot set. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let mut blocked_news = site_page("https://publisher.example/news", "/123456789/x", &nav); + blocked_news.gpt_slots.clear(); + let mut blocked_deals = site_page("https://publisher.example/deals", "/123456789/x", &nav); + blocked_deals.gpt_slots.clear(); + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ("https://publisher.example/news", blocked_news), + ("https://publisher.example/deals", blocked_deals), + ]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("a mostly-challenged crawl should refuse"); + + assert!( + format!("{error:?}").contains("bot protection"), + "the error should name the likely cause, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused run must leave the config untouched" + ); + } + + #[test] + fn max_pages_one_restores_single_page_behavior() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + )]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget { + max_sections: 8, + max_pages: 1, + }, + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update from the single page"); + + assert_eq!( + collector.visited.borrow().len(), + 1, + "max_pages = 1 must not crawl beyond the requested page" + ); + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert!( + value["creative_opportunities"] + .get("section_root") + .is_none(), + "one page cannot witness a section, so no rollback-fatal key may be written" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["gam_unit_path"].as_str(), + Some("/123456789/site/homepage"), + "a single page keeps the literal path" + ); + } + + #[test] + fn generated_config_loads_through_the_runtime_settings_path() { + // The end-to-end contract: whatever `generate` writes must survive the + // same load path the adapter runs at startup. An unloadable config is a + // full-site outage once pushed, not a degraded ad stack. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let baseline = loadable_config(); + trusted_server_core::settings::Settings::from_toml(&baseline) + .expect("test baseline must itself be loadable or the gate is not exercised"); + fs::write(&config_path, &baseline).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let settings = trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + let creative = settings + .creative_opportunities + .expect("generated config should carry creative opportunities"); + assert_eq!( + creative.slot.len(), + 1, + "the discovered slot should be present after a real load" + ); + assert_eq!( + creative.slot[0].div_id.as_deref(), + Some("div-gpt-ad-header") + ); + } + + #[test] + fn update_slots_dry_run_does_not_persist_environment_overlay_config() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let config = EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ); + let config = format!( + "{config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"file-only\"\n\ + div_id = \"div-gpt-ad-file\"\n\ + gam_unit_path = \"/123456789/homepage/file\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 728, height = 90 }}]\n" + ); + fs::write(&config_path, &config).expect("should write config"); + let args = AppConfigArgs { + app_config: Some(config_path.clone()), + manifest: manifest_path, + no_env: false, + }; + + temp_env::with_var( + "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__GAM_NETWORK_ID", + Some("987654321"), + || { + let effective = crate::app_config::load_settings(&args) + .expect("should load effective settings"); + assert_eq!( + effective + .settings + .creative_opportunities + .as_ref() + .expect("should have creative config") + .gam_network_id, + "987654321", + "test environment should override the network id" + ); + let loaded = crate::app_config::load_file_settings(&args) + .expect("should load file-only settings"); + let mut collected = collected_page(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/123456789/homepage/file".to_string(), + div_id: "div-gpt-ad-file".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &loaded.app_config_path, + existing_creative: loaded.settings.creative_opportunities.as_ref(), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should render dry-run update"); + + let output = String::from_utf8(out).expect("output should be UTF-8"); + assert!(output.starts_with("--- configured creative opportunities\n")); + assert!(output.contains("+++ generated creative opportunities\n")); + assert!( + !output.contains("test-admin-password-32-bytes-minimum"), + "dry run must not expose unrelated secrets" + ); + assert!( + !output.contains("987654321"), + "dry run must not persist environment-only config" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should re-read config"), + config, + "dry run must not modify the config file" + ); + }, + ); + } + + #[test] + fn update_slots_refuses_to_overwrite_a_config_changed_during_collection() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + let replacement = format!("{original}\n# edited while the browser was running\n"); + fs::write(&config_path, &original).expect("should write config"); + let collector = MutatingCollector { + collected: collected_page_with_header_slot(), + config_path: config_path.clone(), + replacement: replacement.clone(), + }; + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a stale update should be refused"); + + assert!(format!("{error:?}").contains("changed during the browser audit")); + assert_eq!( + fs::read_to_string(&config_path).expect("should re-read config"), + replacement, + "the concurrent edit must not be overwritten" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs new file mode 100644 index 000000000..740acc86f --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -0,0 +1,150 @@ +//! Derives `page_patterns` globs from the paths a slot was actually observed on. +//! +//! A slot seen on `/news/story-abc` should serve every article in that section, +//! not just that one URL — but nothing here extrapolates beyond a *witnessed* +//! section. Each observed path contributes the section prefix it belongs to and +//! nothing else, so a crawl that never visited `/reviews` never claims it. +//! +//! Each section yields a pair, because one glob cannot cover both halves: +//! `*` crosses `/` in this glob dialect, so `/news/*` matches `/news/a/b` but +//! **not** the bare `/news` landing page. Emitting only the star form silently +//! drops the landing page from the slot. + +use std::collections::BTreeSet; + +/// The root pattern, matching only the site root. +const ROOT_PATTERN: &str = "/"; + +/// Expands observed page paths into the glob set a slot should carry. +/// +/// `section_segment` is the index the section is taken from, matching the +/// config key of the same name: a path is reduced to its first +/// `section_segment + 1` segments, which is the prefix every page of that +/// section shares. A shorter observed landing path is emitted literally; only +/// the actual site root contributes `/`. +/// +/// Results are deduplicated and ordered with `/` first, then alphabetically, so +/// re-running against unchanged evidence produces an unchanged file. +pub(super) fn patterns_for_paths<'a>( + paths: impl IntoIterator, + section_segment: usize, +) -> Vec { + let mut patterns: BTreeSet = BTreeSet::new(); + let mut has_root = false; + + for path in paths { + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.len() <= section_segment { + if segments.is_empty() { + has_root = true; + } else { + patterns.insert(glob::Pattern::escape(path)); + } + continue; + } + let prefix = glob::Pattern::escape(&format!("/{}", segments[..=section_segment].join("/"))); + // The landing page and everything beneath it. + patterns.insert(prefix.clone()); + patterns.insert(format!("{prefix}/*")); + } + + let mut out = Vec::with_capacity(patterns.len() + usize::from(has_root)); + if has_root { + out.push(ROOT_PATTERN.to_string()); + } + out.extend(patterns); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_section_article_yields_both_halves_of_the_pair() { + // `/news/*` alone would not match the bare `/news` landing page, because + // `*` crosses `/` but does not match the empty remainder. + let patterns = patterns_for_paths(["/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"]); + } + + #[test] + fn the_root_path_contributes_the_root_pattern_first() { + let patterns = patterns_for_paths(["/deals/x", "/", "/news/y"], 0); + + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "root first, then sections alphabetically" + ); + } + + #[test] + fn a_landing_page_and_its_article_collapse_to_one_pair() { + let patterns = patterns_for_paths(["/news", "/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"], "no duplicate entries"); + } + + #[test] + fn a_locale_prefixed_site_keeps_the_locale_in_the_prefix() { + // section_segment = 1 means the section is the second segment, so the + // shared prefix every page of that section carries includes the locale. + let patterns = patterns_for_paths(["/en/news/story", "/en/deals/x", "/en"], 1); + + assert_eq!( + patterns, + ["/en", "/en/deals", "/en/deals/*", "/en/news", "/en/news/*"] + ); + } + + #[test] + fn literal_glob_metacharacters_are_escaped_and_match_the_source() { + let source = "/news[local]/story"; + let patterns = patterns_for_paths([source], 0); + + assert_eq!(patterns, ["/news[[]local[]]", "/news[[]local[]]/*"]); + assert!(patterns.iter().any(|pattern| { + glob::Pattern::new(pattern) + .expect("should compile emitted glob") + .matches(source) + })); + } + + #[test] + fn unwitnessed_sections_are_never_invented() { + let patterns = patterns_for_paths(["/news/story"], 0); + + assert_eq!( + patterns, + ["/news", "/news/*"], + "only the crawled section may appear" + ); + } + + #[test] + fn output_is_stable_regardless_of_input_order() { + let one = patterns_for_paths(["/news/a", "/deals/b", "/"], 0); + let two = patterns_for_paths(["/", "/deals/b", "/news/a"], 0); + + assert_eq!(one, two, "re-running should not reorder the written file"); + } + + #[test] + fn every_emitted_pattern_compiles_as_a_runtime_glob() { + let patterns = patterns_for_paths(["/", "/news/story", "/site-news/x"], 0); + + for pattern in &patterns { + trusted_server_core::creative_opportunities::validate_page_pattern(pattern) + .unwrap_or_else(|error| { + panic!("emitted pattern `{pattern}` must compile: {error}") + }); + } + } + + #[test] + fn no_paths_yield_no_patterns() { + assert!(patterns_for_paths([], 0).is_empty()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs new file mode 100644 index 000000000..c5d1627d8 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -0,0 +1,2469 @@ +//! TOML-side slot config: the [`RenderSlot`] model, run merging, rendering, +//! and in-place `[creative_opportunities]` splicing for `ts audit ad-templates +//! generate`. + +use std::collections::{BTreeMap, BTreeSet}; + +use toml_edit::{DocumentMut, Item, Table}; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, +}; + +#[cfg(test)] +use crate::commands::audit::generate::gpt_slots; +use crate::error::{CliResult, cli_error, report_error}; + +/// A slot ready to render — the union of discovered and existing fields, without +/// the core type's `pub(crate)` compiled-pattern cache. +#[derive(Debug, Clone)] +pub(super) struct RenderSlot { + id: String, + div_id: Option, + gam_unit_path: Option, + page_patterns: Vec, + /// `(width, height, non-banner media type)`. + formats: Vec<(u32, u32, Option<&'static str>)>, + floor_price: Option, + targeting: BTreeMap, + aps_slot_id: Option, + /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). + prebid_bidders: Option>, +} + +impl RenderSlot { + /// The stable exact identity fallback used when no configured div prefix + /// matches a discovered slot. + fn key(&self) -> String { + self.div_id + .as_deref() + .unwrap_or(&self.id) + .trim_end_matches('-') + .to_string() + } + + /// Whether this configured slot carries fields that discovery cannot infer. + fn has_tuned_fields(&self) -> bool { + self.floor_price.is_some() + || !self.targeting.is_empty() + || self.aps_slot_id.is_some() + || self.prebid_bidders.is_some() + } + + /// Builds a slot from one page's discovery. + /// + /// Superseded in production by [`RenderSlot::from_evidence`], which reads + /// cross-page evidence; retained as test scaffolding for the merge cases. + #[cfg(test)] + fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { + Self { + id: slot.id.clone(), + div_id: Some(slot.div_id.clone()), + gam_unit_path: Some(slot.gam_unit_path.clone()), + page_patterns: patterns.to_vec(), + formats: slot + .formats + .iter() + .map(|&(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: slot.has_prebid.then(BTreeMap::new), + } + } + + /// Builds a slot from cross-page evidence and the inferred unit path. + /// + /// Refused inference decisions are filtered before this constructor. A + /// `None` path therefore means inference was unavailable and deliberately + /// leaves the runtime's configured default-path behavior in effect. + pub(super) fn from_evidence( + id: &str, + div_id: &str, + gam_unit_path: Option, + formats: impl IntoIterator, + page_patterns: Vec, + has_prebid: bool, + ) -> Self { + Self { + id: id.to_string(), + div_id: Some(div_id.to_string()), + gam_unit_path, + page_patterns, + formats: formats + .into_iter() + .map(|(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: has_prebid.then(BTreeMap::new), + } + } + + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { + Self { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + gam_unit_path: slot.gam_unit_path.clone(), + page_patterns: slot.page_patterns.clone(), + formats: slot + .formats + .iter() + .map(|format| { + ( + format.width, + format.height, + media_type_label(&format.media_type), + ) + }) + .collect(), + floor_price: slot.floor_price, + targeting: slot + .targeting + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { + prebid + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect() + }), + } + } +} + +/// The non-default (non-banner) media-type label to emit, or `None` for banner. +fn media_type_label(media_type: &MediaType) -> Option<&'static str> { + match media_type { + MediaType::Banner => None, + MediaType::Video => Some("video"), + MediaType::Native => Some("native"), + } +} + +/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. +/// +/// - `--replace` (or no existing slots): the result is exactly the discovered set. +/// - Otherwise existing slots are preserved (covering other pages / hand-tuned +/// fields); a slot re-seen this run has its page patterns and formats unioned; +/// slots seen only this run are appended. +/// - Format identity includes media type, so equal dimensions observed for two +/// media types remain two intentional entries. +#[cfg(test)] +pub(super) fn merge_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered: &gpt_slots::DiscoveredSlots, + run_patterns: &[String], + replace: bool, +) -> Vec { + let discovered_slots: Vec = discovered + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) + .collect(); + merge_render_slots(existing, discovered_slots, replace) +} + +/// Merges already-built slots into the existing set. +/// +/// Same reconciliation as the single-page test helper, but the caller supplies the slots — +/// the crawl path builds them from cross-page evidence rather than from one +/// page's discoveries. A slot re-seen this run keeps its configured fields and +/// gains this run's patterns; a genuinely new slot is appended with a +/// non-colliding id. +#[cfg(test)] +pub(super) fn merge_render_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> Vec { + merge_render_slots_with_diagnostics(existing, discovered_slots, replace).0 +} + +/// Diagnostics produced while merging discovered and configured slots. +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct MergeDiagnostics { + /// Operator-facing reconciliation notes. + pub(super) notes: Vec, + /// Configured slots preserved without matching any normalized evidence div. + pub(super) unobserved_existing_slot_ids: Vec, +} + +/// Merges slots and reports prefix collisions and unobserved preserved slots. +#[cfg(test)] +pub(super) fn merge_render_slots_with_diagnostics( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> (Vec, MergeDiagnostics) { + let observed_div_ids = discovered_slots + .iter() + .filter_map(|slot| slot.div_id.clone()) + .collect::>(); + merge_render_slots_with_observed_diagnostics( + existing, + discovered_slots, + &observed_div_ids, + &observed_div_ids, + replace, + ) +} + +/// Merges renderable slots using normalized evidence div IDs for observation. +/// +/// `observed_div_ids` must be the full normalized evidence set, including divs +/// refused by template inference, skipped as fragments, or refused as +/// ambiguous. `observed_literal_div_ids` contains only concrete live elements; +/// it controls whether a configured div ID remains eligible as a runtime prefix. +/// Passing only the rendered subset for observation can falsely report a live +/// configured slot as unobserved, while treating refused stems as literals can +/// incorrectly disqualify a configured prefix from merge routing. +pub(super) fn merge_render_slots_with_observed_diagnostics( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + observed_div_ids: &[String], + observed_literal_div_ids: &[String], + replace: bool, +) -> (Vec, MergeDiagnostics) { + let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); + if replace || existing_slots.is_empty() { + return (discovered_slots, MergeDiagnostics::default()); + } + + let observed_literals = observed_literal_div_ids + .iter() + .map(String::as_str) + .collect::>(); + + let mut merged: Vec = existing_slots + .iter() + .map(RenderSlot::from_existing) + .collect(); + let existing_count = merged.len(); + let mut prefix_claims: BTreeMap> = BTreeMap::new(); + let mut split_warnings = BTreeSet::new(); + let mut observed_existing = observed_div_ids + .iter() + .flat_map(|div_id| matching_observed_div_indexes(&merged, div_id, &observed_literals)) + .collect::>(); + for mut slot in discovered_slots { + // Prefix reconciliation is a property of the operator's config, so only + // the slots that were already configured may claim a discovered div. + // Slots this run appended match by exact identity instead, otherwise + // discovery order decides whether `ad-top` swallows a later + // `ad-top-sidebar` and discards its unit path and provider state. + let matched = matching_slot_index(&merged[..existing_count], &slot, &observed_literals) + .or_else(|| { + let key = slot.key(); + merged[existing_count..] + .iter() + .position(|added| added.key() == key) + .map(|offset| offset + existing_count) + }); + if let Some(index) = matched { + if index < existing_count { + observed_existing.insert(index); + } + if index < existing_count + && let (Some(prefix), Some(discovered_div)) = + (merged[index].div_id.as_deref(), slot.div_id.as_deref()) + && discovered_div.starts_with(prefix) + { + prefix_claims + .entry(index) + .or_default() + .insert(discovered_div.to_string()); + } + let present = &mut merged[index]; + for pattern in &slot.page_patterns { + if !present.page_patterns.contains(pattern) { + present.page_patterns.push(pattern.clone()); + } + } + for format in &slot.formats { + if !present.formats.contains(format) { + present.formats.push(*format); + } + } + } else { + if let Some(discovered_div) = slot.div_id.as_deref() + && let Some(parent) = merged[..existing_count] + .iter() + .filter(|configured| { + configured.div_id.as_deref().is_some_and(|prefix| { + !prefix.is_empty() + && observed_literals.contains(prefix) + && discovered_div != prefix + && discovered_div.starts_with(prefix) + }) + }) + .max_by_key(|configured| configured.div_id.as_deref().map_or(0, str::len)) + && parent.has_tuned_fields() + { + split_warnings.insert(format!( + "discovered div `{discovered_div}` was split from configured div_id prefix \ + `{}`; the new slot does not inherit that configured slot's floor price, \ + targeting, or provider settings", + parent.div_id.as_deref().unwrap_or_default(), + )); + } + slot.id = unique_slot_id(&slot.id, &merged); + merged.push(slot); + } + } + let notes = split_warnings + .into_iter() + .chain( + prefix_claims + .into_iter() + .filter(|(_, divs)| divs.len() > 1) + .map(|(index, divs)| { + let slot = &merged[index]; + let sample = divs.iter().take(5).cloned().collect::>().join(", "); + let remainder = divs.len().saturating_sub(5); + let suffix = if remainder == 0 { + String::new() + } else { + format!(", and {remainder} more") + }; + format!( + "configured slot `{}` with div_id prefix `{}` matched {} discovered divs \ + ({sample}{suffix}); runtime can resolve this configured slot to at most one \ + active element, so review whether they are distinct placements", + slot.id, + slot.div_id.as_deref().unwrap_or_default(), + divs.len(), + ) + }), + ) + .collect(); + let unobserved_existing_slot_ids = existing_slots + .iter() + .enumerate() + .filter(|(index, _)| !observed_existing.contains(index)) + .map(|(_, slot)| slot.id.clone()) + .collect(); + ( + merged, + MergeDiagnostics { + notes, + unobserved_existing_slot_ids, + }, + ) +} + +fn unique_slot_id(candidate: &str, existing: &[RenderSlot]) -> String { + if existing.iter().all(|slot| slot.id != candidate) { + return candidate.to_string(); + } + + let mut suffix = 2_usize; + loop { + let unique = format!("{candidate}-{suffix}"); + if existing.iter().all(|slot| slot.id != unique) { + return unique; + } + suffix += 1; + } +} + +/// Finds the configured slot matching a discovered normalized slot. +/// +/// Stable-key equality wins first. Otherwise, configured `div_id` values are +/// eligible runtime prefixes unless that value was itself observed as a +/// distinct literal. Equal-length prefix ties retain configuration order. +fn matching_slot_index( + existing: &[RenderSlot], + discovered: &RenderSlot, + observed_literals: &BTreeSet<&str>, +) -> Option { + let key = discovered.key(); + if let Some(index) = existing.iter().position(|slot| slot.key() == key) { + return Some(index); + } + + discovered + .div_id + .as_deref() + .and_then(|div_id| matching_div_id_index(existing, div_id, observed_literals)) +} + +fn matching_div_id_index( + existing: &[RenderSlot], + discovered_div: &str, + observed_literals: &BTreeSet<&str>, +) -> Option { + let mut best = None; + let mut best_length = 0; + for (index, slot) in existing.iter().enumerate() { + let Some(prefix) = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty()) else { + continue; + }; + if observed_literals.contains(prefix) { + continue; + } + if discovered_div.starts_with(prefix) && prefix.len() > best_length { + best = Some(index); + best_length = prefix.len(); + } + } + best +} + +/// Finds every configured slot that can resolve to one normalized evidence div. +/// +/// Merge routing remains exact-then-longest-prefix through +/// [`matching_slot_index`], but observation is deliberately multi-match: an +/// exact configured slot and every eligible broad prefix are all live when the +/// element exists. +fn matching_observed_div_indexes<'a>( + existing: &'a [RenderSlot], + discovered_div: &'a str, + observed_literals: &'a BTreeSet<&'a str>, +) -> impl Iterator + 'a { + let discovered_key = discovered_div.trim_end_matches('-'); + existing + .iter() + .enumerate() + .filter_map(move |(index, slot)| { + if slot.key() == discovered_key { + return Some(index); + } + let prefix = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty())?; + (!observed_literals.contains(prefix) && discovered_div.starts_with(prefix)) + .then_some(index) + }) +} + +/// Header comment emitted above the structurally replaced managed slot array. +const MANAGED_SLOTS_COMMENT: &str = "# Slots managed by `ts audit ad-templates generate`."; +/// Second line of the managed-slot header comment. +const MANAGED_SLOTS_REVIEW_COMMENT: &str = + "# Review page_patterns and formats before validating/pushing."; + +/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. +pub(super) fn render_slots(slots: &[RenderSlot]) -> String { + let mut out = format!("\n{MANAGED_SLOTS_COMMENT}\n{MANAGED_SLOTS_REVIEW_COMMENT}\n"); + for slot in slots { + out.push_str("\n[[creative_opportunities.slot]]\n"); + out.push_str(&format!("id = {}\n", toml_string(&slot.id))); + if let Some(div_id) = &slot.div_id { + out.push_str(&format!("div_id = {}\n", toml_string(div_id))); + } + if let Some(path) = &slot.gam_unit_path { + out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); + } + out.push_str("page_patterns = [\n"); + for pattern in &slot.page_patterns { + out.push_str(&format!(" {},\n", toml_string(pattern))); + } + out.push_str("]\n"); + out.push_str("formats = [\n"); + for (width, height, media_type) in &slot.formats { + let rendered = match media_type { + Some(kind) => { + format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") + } + None => format!("{{ width = {width}, height = {height} }}"), + }; + out.push_str(&format!(" {rendered},\n")); + } + out.push_str("]\n"); + if let Some(floor) = slot.floor_price { + // `f64` Display prints `NaN`, which is not valid TOML (`nan` is); + // normalize non-finite values so the spliced config stays parseable. + if floor.is_finite() { + out.push_str(&format!("floor_price = {floor}\n")); + } else if floor.is_nan() { + out.push_str("floor_price = nan\n"); + } else if floor.is_sign_positive() { + out.push_str("floor_price = inf\n"); + } else { + out.push_str("floor_price = -inf\n"); + } + } + if !slot.targeting.is_empty() { + let pairs = slot + .targeting + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) + .collect::>() + .join(", "); + out.push_str(&format!("targeting = {{ {pairs} }}\n")); + } + if let Some(slot_id) = &slot.aps_slot_id { + out.push_str("[creative_opportunities.slot.providers.aps]\n"); + out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); + } + if let Some(bidders) = &slot.prebid_bidders { + out.push_str("[creative_opportunities.slot.providers.prebid]\n"); + let rendered = bidders + .iter() + .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) + .collect::>() + .join(", "); + if rendered.is_empty() { + out.push_str("bidders = {}\n"); + } else { + out.push_str(&format!("bidders = {{ {rendered} }}\n")); + } + } + } + out +} + +/// Quotes and escapes a string as a TOML basic string, including control chars. +pub(super) fn toml_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + // TOML basic strings reject U+0000..U+001F and DEL (U+007F). + control if (control as u32) < 0x20 || control == '\u{7f}' => { + out.push_str(&format!("\\u{:04X}", control as u32)); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. +fn toml_key(key: &str) -> String { + let is_bare = !key.is_empty() + && key + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if is_bare { + key.to_string() + } else { + toml_string(key) + } +} + +/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). +fn toml_inline_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Null => "{}".to_string(), + serde_json::Value::Bool(bool) => bool.to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::String(string) => toml_string(string), + serde_json::Value::Array(items) => { + let rendered = items + .iter() + .map(toml_inline_value) + .collect::>() + .join(", "); + format!("[{rendered}]") + } + serde_json::Value::Object(map) => { + let rendered = map + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) + .collect::>() + .join(", "); + format!("{{ {rendered} }}") + } + } +} + +/// The config-level values a splice writes alongside the slot array. +#[derive(Debug, Clone, Default)] +pub(super) struct CreativeSectionKeys<'a> { + /// GAM network id, when one was resolved. + pub(super) network_id: Option<&'a str>, + /// `section_root`, written only when a slot uses a `{section}` template. + pub(super) section_root: Option<&'a str>, + /// `section_segment`, written only alongside `section_root`. + pub(super) section_segment: Option, +} + +fn max_table_position(table: &Table) -> Option { + table.iter().fold(table.position(), |maximum, (_, item)| { + let child_maximum = match item { + Item::Table(child) => max_table_position(child), + Item::ArrayOfTables(array) => array.iter().filter_map(max_table_position).max(), + Item::None | Item::Value(_) => None, + }; + maximum.max(child_maximum) + }) +} + +fn set_table_position_recursive(table: &mut Table, position: isize) { + table.set_position(position); + for (_, item) in table.iter_mut() { + match item { + Item::Table(child) => set_table_position_recursive(child, position), + Item::ArrayOfTables(array) => { + for child in array.iter_mut() { + set_table_position_recursive(child, position); + } + } + Item::None | Item::Value(_) => {} + } + } +} + +/// Structurally replaces the generator-managed creative-opportunities fields. +/// +/// All unrelated TOML items and their decorations remain in the parsed +/// document. Missing inferred scalar values preserve their existing values; a +/// fresh section is created only when a network id is available. +pub(super) fn splice_creative_slots( + existing: &str, + keys: &CreativeSectionKeys<'_>, + rendered_slots: &str, +) -> CliResult { + let mut document = existing.parse::().map_err(|error| { + report_error(format!( + "failed to parse target config before updating slots: {error}" + )) + })?; + let had_section = document.get("creative_opportunities").is_some(); + let existing_section_position = document + .get("creative_opportunities") + .and_then(Item::as_table) + .and_then(Table::position); + let section_position = existing_section_position + .unwrap_or_else(|| max_table_position(document.as_table()).unwrap_or(0) + 1); + if !had_section && keys.network_id.is_none() { + return cli_error( + "refusing to create a `[creative_opportunities]` section without a \ + GAM network id: none could be determined from the audited page, and \ + the key is required. Add `[creative_opportunities]` with a \ + `gam_network_id` to the config and re-run", + ); + } + + let generated = format!( + "[creative_opportunities]\n{}\n", + rendered_slots.trim_matches('\n') + ); + let mut generated = generated + .parse::() + .map_err(|error| report_error(format!("failed to parse generated slot tables: {error}")))?; + let mut generated_slots = generated["creative_opportunities"] + .as_table_mut() + .and_then(|table| table.remove("slot")) + .unwrap_or_else(|| Item::ArrayOfTables(toml_edit::ArrayOfTables::new())); + if let Item::ArrayOfTables(array) = &mut generated_slots { + for table in array.iter_mut() { + set_table_position_recursive(table, section_position); + } + } + + if !had_section { + document["creative_opportunities"] = Item::Table(toml_edit::Table::new()); + } + let creative = document["creative_opportunities"] + .as_table_mut() + .ok_or_else(|| { + report_error( + "target config's `creative_opportunities` value is not an editable table; \ + rewrite it as a `[creative_opportunities]` table and re-run", + ) + })?; + // `toml_edit` stably sorts tables by document position. Imported tables + // retain positions from their source document, so anchor the whole subtree + // here to keep the parent, slots, and provider tables together. + creative.set_position(section_position); + if let Some(network_id) = keys.network_id { + creative["gam_network_id"] = toml_edit::value(network_id); + } + if let Some(section_root) = keys.section_root { + creative["section_root"] = toml_edit::value(section_root); + if let Some(section_segment) = keys.section_segment { + creative["section_segment"] = toml_edit::value(section_segment as i64); + } + } + creative.insert("slot", generated_slots); + + let mut result = document.to_string(); + if uses_crlf(existing) { + result = convert_document_lf_to_crlf(&result); + } + ensure_only_managed_fields_changed(existing, &result)?; + Ok(result) +} + +/// Verifies that the structural update changed only generator-managed fields. +fn ensure_only_managed_fields_changed(before: &str, after: &str) -> CliResult<()> { + fn unmanaged(document: &str) -> CliResult { + let mut value = toml::from_str::(document) + .map_err(|error| report_error(format!("failed to validate updated config: {error}")))?; + if let Some(root) = value.as_table_mut() { + let remove_empty = if let Some(creative) = root + .get_mut("creative_opportunities") + .and_then(toml::Value::as_table_mut) + { + for key in ["slot", "gam_network_id", "section_root", "section_segment"] { + creative.remove(key); + } + creative.is_empty() + } else { + false + }; + if remove_empty { + root.remove("creative_opportunities"); + } + } + Ok(value) + } + + if unmanaged(before)? != unmanaged(after)? { + return cli_error( + "refusing to update config because fields outside the managed \ + creative-opportunities keys would change", + ); + } + Ok(()) +} + +/// Byte offsets of the `\n` bytes that terminate a document line. +/// +/// Only newlines outside comments and string values delimit lines, so the scan +/// skips a `#` comment to end of line, skips single-line basic and literal +/// strings, and tracks multiline `"""` / `'''` bodies. Without the comment and +/// single-line-string cases a stray triple quote desynchronizes the scan and the +/// document's line endings are flipped or left mixed — a rewrite +/// [`ensure_only_managed_fields_changed`] cannot catch, because it compares +/// parsed values. +fn document_newlines(document: &str) -> Vec { + let bytes = document.as_bytes(); + let mut newlines = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + match bytes[index] { + b'#' => { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'\n' => { + newlines.push(index); + index += 1; + } + quote @ (b'"' | b'\'') => { + if bytes[index..].starts_with(&[quote, quote, quote]) { + index += 3; + while index < bytes.len() && !bytes[index..].starts_with(&[quote, quote, quote]) + { + index += 1; + } + index = index.saturating_add(3).min(bytes.len()); + } else { + index += 1; + while index < bytes.len() && bytes[index] != quote && bytes[index] != b'\n' { + index += if quote == b'"' && bytes[index] == b'\\' { + 2 + } else { + 1 + }; + } + if index < bytes.len() && bytes[index] == quote { + index += 1; + } + } + } + _ => index += 1, + } + } + newlines +} + +/// Whether `document` uses CRLF line endings (so edits preserve them). +fn uses_crlf(document: &str) -> bool { + let bytes = document.as_bytes(); + document_newlines(document) + .first() + .is_some_and(|&index| index > 0 && bytes[index - 1] == b'\r') +} + +/// Converts document line terminators while leaving string content intact. +fn convert_document_lf_to_crlf(document: &str) -> String { + let bytes = document.as_bytes(); + let mut output = String::with_capacity(document.len()); + let mut previous = 0_usize; + for index in document_newlines(document) { + output.push_str(&document[previous..index]); + if index == 0 || bytes[index - 1] != b'\r' { + output.push('\r'); + } + output.push('\n'); + previous = index + 1; + } + output.push_str(&document[previous..]); + output +} + +/// Strips a trailing inline `# comment` from a candidate table-header line. +/// +/// Only valid on header candidates: header lines cannot contain `#` before the +/// closing bracket unless it is inside a quoted key, which the configs this +/// updater manages never use. +fn strip_inline_comment(line: &str) -> &str { + match line.find('#') { + Some(position) => line[..position].trim_end(), + None => line, + } +} + +pub(super) fn replace_key_in_section( + document: &str, + section: &str, + key: &str, + replacement_line: &str, +) -> CliResult { + let section_header = format!("[{section}]"); + let mut in_section = false; + let mut replaced = false; + let mut saw_section = false; + let mut lines = Vec::new(); + + for line in document.lines() { + let trimmed = line.trim(); + let header_candidate = strip_inline_comment(trimmed); + if header_candidate.starts_with('[') && header_candidate.ends_with(']') { + in_section = header_candidate == section_header; + saw_section |= in_section; + } + + if in_section && !replaced && is_key_line(trimmed, key) { + lines.push(replacement_line.to_string()); + replaced = true; + } else { + lines.push(line.to_string()); + } + } + + if !saw_section { + return cli_error(format!( + "failed to update starter config because section `{section_header}` was not found" + )); + } + if !replaced { + return cli_error(format!( + "failed to update starter config because key `{key}` was not found in `{section_header}`" + )); + } + + let mut output = lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + if uses_crlf(document) { + // `lines()` stripped the `\r`s; restore the document's CRLF endings. + output = output.replace("\r\n", "\n").replace('\n', "\r\n"); + } + Ok(output) +} + +fn is_key_line(trimmed_line: &str, key: &str) -> bool { + trimmed_line + .strip_prefix(key) + .and_then(|remaining| remaining.trim_start().strip_prefix('=')) + .is_some() +} + +/// Chooses the `gam_network_id` to write. +/// +/// The existing id is kept only when a real merge preserves existing slots. +/// On `--replace`, or when the config had no slots (e.g. a placeholder +/// `[creative_opportunities]` section), the discovered id wins — mirroring +/// the slot merge, which returns discovered-only in those cases. +pub(super) fn resolve_network_id( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_network_id: Option<&str>, + replace: bool, +) -> Option { + let existing_network_id = existing.map(|config| config.gam_network_id.clone()); + let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); + if preserving_existing { + existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) + } else { + discovered_network_id + .map(str::to_string) + .or(existing_network_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector; + + fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + gpt_slots::discover_gpt_slots(®istry, &[], false) + } + + /// Rendered slot text for the discovered header slot, patterns = `/`. + fn header_rendered() -> String { + let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); + render_slots(&merged) + } + + fn two_provider_slots_rendered() -> &'static str { + r#" +# Slots managed by `ts audit ad-templates generate`. +# Review page_patterns and formats before validating/pushing. + +[[creative_opportunities.slot]] +id = "header" +div_id = "header" +gam_unit_path = "/222/{section}/header" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }] +[creative_opportunities.slot.providers.prebid] +bidders = {} + +[[creative_opportunities.slot]] +id = "sidebar" +div_id = "sidebar" +gam_unit_path = "/222/{section}/sidebar" +page_patterns = ["/"] +formats = [{ width = 300, height = 250 }] +[creative_opportunities.slot.providers.aps] +slot_id = "sidebar" +"# + } + + fn table_headers(document: &str) -> Vec<&str> { + document + .lines() + .map(str::trim) + .filter(|line| line.starts_with('[')) + .collect() + } + + /// Section keys carrying only a network id, the common test case. + fn network_keys(network_id: &str) -> CreativeSectionKeys<'_> { + CreativeSectionKeys { + network_id: Some(network_id), + ..CreativeSectionKeys::default() + } + } + + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { + toml::from_str::(toml_str).expect("valid creative config") + } + + #[test] + fn splice_replaces_slots_and_preserves_other_sections() { + let existing = "[publisher]\ndomain = \"x\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ + [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ + gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + out.contains("gam_network_id = \"222\""), + "network id updated" + ); + assert!(!out.contains("id = \"old\""), "old slot removed"); + assert!( + out.contains("gam_unit_path = \"/222/homepage/header\""), + "new slot written" + ); + assert!( + out.contains("[publisher]") && out.contains("domain = \"x\""), + "publisher section preserved" + ); + assert!( + out.contains("[auction]") && out.contains("enabled = true"), + "trailing auction section preserved" + ); + toml::from_str::(&out).expect("spliced config is valid TOML"); + } + + #[test] + fn splice_updates_a_quoted_section_header_structurally() { + let existing = "[\"creative_opportunities\"]\ngam_network_id = \"111\"\n"; + + let updated = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should update quoted table structurally"); + + assert_eq!(updated.matches("creative_opportunities").count(), 2); + assert!(updated.contains("gam_network_id = \"222\"")); + toml::from_str::(&updated).expect("should remain valid TOML"); + } + + #[test] + fn splice_preserves_multiline_values_comments_and_noncontiguous_tables() { + let existing = "title = \"publisher\" # keep this comment\n\ + description = \"\"\"a line that looks like [creative_opportunities]\n\ + and another [[creative_opportunities.slot]] line\"\"\"\n\ + dimensions = [\n 300,\n 250,\n]\n\n\ + [creative_opportunities] # managed section\n\ + gam_network_id = \"111\" # old network\n\n\ + [[creative_opportunities.slot]]\nid = \"old-a\"\ndiv_id = \"old-a\"\n\ + gam_unit_path = \"/111/a\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true # keep auction comment\n\n\ + [[creative_opportunities.slot]]\nid = \"old-b\"\ndiv_id = \"old-b\"\n\ + gam_unit_path = \"/111/b\"\npage_patterns = [\"/b\"]\n\ + formats = [{ width = 320, height = 50 }]\n"; + + let updated = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should update structurally"); + + assert!(updated.contains("looks like [creative_opportunities]")); + assert!(updated.contains("dimensions = [\n 300,\n 250,\n]")); + assert!(updated.contains("enabled = true # keep auction comment")); + assert!(!updated.contains("id = \"old-a\"")); + assert!(!updated.contains("id = \"old-b\"")); + let value = toml::from_str::(&updated).expect("should remain valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"] + .as_array() + .map(Vec::len), + Some(1) + ); + } + + #[test] + fn splice_keeps_generated_slots_and_providers_contiguous() { + let existing = "[publisher]\ndomain = \"example.com\"\n\n\ + [tester_cookie]\nenabled = true\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [debug]\nauction_html_comment = true\n"; + + let updated = splice_creative_slots( + existing, + &network_keys("222"), + two_provider_slots_rendered(), + ) + .expect("should splice slots"); + + assert_eq!( + table_headers(&updated), + vec![ + "[publisher]", + "[tester_cookie]", + "[creative_opportunities]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.prebid]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.aps]", + "[debug]", + ] + ); + } + + #[test] + fn splice_groups_a_new_creative_section_with_its_slots() { + let existing = "[publisher]\ndomain = \"example.com\"\n\n\ + [debug]\nauction_html_comment = true\n\n\ + [auction]\nenabled = true\n"; + + let updated = splice_creative_slots( + existing, + &network_keys("222"), + two_provider_slots_rendered(), + ) + .expect("should create creative section and splice slots"); + + assert_eq!( + table_headers(&updated), + vec![ + "[publisher]", + "[debug]", + "[auction]", + "[creative_opportunities]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.prebid]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.aps]", + ] + ); + } + + #[test] + fn splice_rejects_top_level_inline_creative_opportunities_table() { + let existing = "creative_opportunities = { gam_network_id = \"111\" }\n"; + + let error = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect_err("should refuse a top-level inline table"); + + assert!( + format!("{error:?}").contains("rewrite it as"), + "error should tell the operator to rewrite the section, got {error:?}" + ); + } + + /// Section keys for a templated run: network id plus the section policy. + fn template_keys<'a>( + network_id: &'a str, + root: &'a str, + segment: usize, + ) -> CreativeSectionKeys<'a> { + CreativeSectionKeys { + network_id: Some(network_id), + section_root: Some(root), + section_segment: Some(segment), + } + } + + #[test] + fn splice_inserts_section_policy_keys_a_config_does_not_have_yet() { + // The whole point of `upsert`: every config predating templating lacks + // these keys, so a replace-only writer could never add them. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "inserting must not disturb later sections" + ); + } + + #[test] + fn splice_replaces_section_policy_keys_that_are_already_present() { + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\ + section_root = \"old\"\nsection_segment = 2\n"; + + let out = splice_creative_slots( + existing, + &template_keys("111", "homepage", 1), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(1)); + assert_eq!( + out.matches("section_root").count(), + 1, + "the key must be replaced, not duplicated" + ); + } + + #[test] + fn splice_omits_section_policy_when_no_slot_needs_it() { + // `section_root`/`section_segment` are `deny_unknown_fields` additions: + // writing them into a config that does not need them would make it + // unloadable by an older binary for no benefit. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.contains("section_root") && !out.contains("section_segment"), + "an untemplated run must not add rollback-fatal keys, got:\n{out}" + ); + } + + #[test] + fn splice_writes_section_policy_into_a_freshly_created_section() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + } + + #[test] + fn splice_refuses_fresh_section_without_a_network_id() { + // Reachable whenever the scraped unit path has no all-digit leading + // segment (MCM/child-network paths). Writing the section anyway produces + // a config missing a required field, which fails load and takes every + // route to the startup error router once pushed. + let existing = "[publisher]\ndomain = \"x\"\n"; + + let error = splice_creative_slots( + existing, + &CreativeSectionKeys::default(), + &header_rendered(), + ) + .expect_err("should refuse to create a section with no network id"); + + assert!( + format!("{error:?}").contains("without a GAM network id"), + "error should name the missing network id, got {error:?}" + ); + } + + #[test] + fn splice_appends_section_when_config_has_none() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("appended config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222") + ); + } + + #[test] + fn splice_preserves_section_scalars_and_provider_subtables() { + // Mirrors the templated operator shape: section policy scalars in the + // head block and a per-slot prebid provider subtable. + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + auction_timeout_ms = 2000\n\ + section_root = \"homepage\"\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-header-0\"\n\ + div_id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {}\n\n\ + [auction]\nenabled = true\n"; + let existing_config = existing_config( + &existing + .replace("[creative_opportunities]\n", "") + .replace("[[creative_opportunities.slot]]", "[[slot]]") + .replace("[creative_opportunities.slot.", "[slot.") + .replace("\n[auction]\nenabled = true\n", ""), + ); + let discovered = discovered_header_slot(); + let merged = merge_slots( + Some(&existing_config), + &discovered, + &["/news/*".to_string()], + false, + ); + + let out = splice_creative_slots(existing, &network_keys("111"), &render_slots(&merged)) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "section policy scalars must survive the splice" + ); + assert_eq!(creative["auction_timeout_ms"].as_integer(), Some(2000)); + assert_eq!( + creative["slot"][0]["gam_unit_path"].as_str(), + Some("/{network_id}/example/{section}"), + "an existing templated unit path must not be rewritten to a literal" + ); + assert!( + creative["slot"][0]["providers"]["prebid"]["bidders"].is_table(), + "the prebid provider subtable must be re-emitted" + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "trailing sections must be preserved" + ); + } + + #[test] + fn splice_preserves_crlf_line_endings() { + let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ + [auction]\r\nenabled = true\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "every line ending should stay CRLF" + ); + let value = toml::from_str::(&out).expect("spliced CRLF config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated in CRLF config" + ); + } + + #[test] + fn splice_does_not_infer_document_endings_from_multiline_string_content() { + let existing = "[publisher]\nother = \"\"\"a\r\nb\"\"\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice LF document"); + + assert!( + out.contains("[publisher]\nother"), + "an embedded CRLF must not convert document line endings" + ); + assert!( + out.contains("a\r\nb"), + "an unrelated multiline string value must remain byte-identical" + ); + } + + #[test] + fn a_triple_quote_in_a_comment_does_not_desynchronize_the_line_scan() { + // A `"""` inside a comment is not a multiline string. Treating it as one + // makes the rest of the document read as string content, so a CRLF file + // is detected as LF and gets rewritten wholesale. + let existing = "# see \"\"\" docs\r\n[creative_opportunities]\r\n\ + gam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a comment, got {out:?}" + ); + } + + #[test] + fn a_triple_quote_in_a_single_line_string_does_not_desynchronize_the_line_scan() { + let existing = "[publisher]\r\nlabel = 'a \"\"\" b'\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a value, got {out:?}" + ); + } + + #[test] + fn splice_does_not_rewrite_bare_lf_inside_crlf_multiline_string() { + let existing = "[publisher]\r\nother = \"\"\"a\nb\"\"\"\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + out.contains("a\nb"), + "a bare LF inside an unrelated multiline value must remain unchanged" + ); + } + + #[test] + fn render_slots_writes_non_finite_floor_price_as_valid_toml() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string()], + formats: vec![(728, 90, None)], + floor_price: Some(f64::NAN), + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("floor_price = nan"), + "NaN should render as TOML `nan`, not Rust `NaN`" + ); + toml::from_str::(&rendered).expect("rendered slots are valid TOML"); + } + + #[test] + fn render_slots_formats_long_arrays_across_indented_lines() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string(), "/news".to_string(), "/news/*".to_string()], + formats: vec![(728, 90, None), (970, 250, None), (300, 250, None)], + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("page_patterns = [\n \"/\",\n \"/news\",\n \"/news/*\",\n]\n"), + "page patterns should be readable one-per-line" + ); + assert!( + rendered.contains( + "formats = [\n { width = 728, height = 90 },\n \ + { width = 970, height = 250 },\n \ + { width = 300, height = 250 },\n]\n" + ), + "formats should be readable one-per-line" + ); + toml::from_str::(&rendered).expect("formatted slots are valid TOML"); + } + + #[test] + fn splice_creates_section_when_absent() { + // Config with no [creative_opportunities] at all — generate should append it. + let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "appended section carries the discovered network id" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert!( + value["publisher"]["domain"].as_str() == Some("x") + && value["auction"]["enabled"].as_bool() == Some(true), + "existing sections preserved when appending" + ); + } + + #[test] + fn resplice_does_not_accumulate_managed_comment() { + // A re-run splices into a config that already carries the managed + // header comment; it must keep exactly one copy, not append another. + let first = splice_creative_slots( + "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n", + &network_keys("222"), + &header_rendered(), + ) + .expect("first splice"); + let second = splice_creative_slots(&first, &network_keys("222"), &header_rendered()) + .expect("second splice"); + let third = splice_creative_slots(&second, &network_keys("222"), &header_rendered()) + .expect("third splice"); + + assert_eq!( + third + .lines() + .filter(|line| line.trim() == MANAGED_SLOTS_COMMENT) + .count(), + 1, + "managed header comment must not accumulate across re-splices" + ); + toml::from_str::(&third).expect("re-spliced config stays valid TOML"); + } + + #[test] + fn splice_recognizes_inline_commented_section_header() { + // `[creative_opportunities] # comment` is valid TOML; the splice must + // update it in place instead of appending a duplicate section. + let existing = "[creative_opportunities] # ad templates\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert_eq!( + out.lines() + .filter(|line| { strip_inline_comment(line.trim()) == "[creative_opportunities]" }) + .count(), + 1, + "commented header must not be duplicated" + ); + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated under a commented header" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "commented trailing section preserved" + ); + } + + #[test] + fn splice_inserts_when_no_existing_slots() { + let existing = + "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header"), + "inserted slot id strips the div-gpt-ad- prefix" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "div_id keeps the stable stem" + ); + assert!( + value["auction"]["enabled"].as_bool() == Some(true), + "auction section preserved after inserted slots" + ); + } + + #[test] + fn splice_replaces_inline_slot_array() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = [{ id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should replace inline slot array"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "unrelated tables should be preserved" + ); + } + + #[test] + fn splice_replaces_inline_slot_map() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = { \"0\" = { id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] } }\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should replace inline slot map"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + } + + #[test] + fn merge_second_run_unions_page_patterns() { + // Existing slot on "/"; re-discovered this run with "/news/*". + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/news/*".to_string()], + false, + ); + + assert_eq!(merged.len(), 1, "same slot is not duplicated"); + assert_eq!( + merged[0].page_patterns, + vec!["/".to_string(), "/news/*".to_string()], + "this run's pattern is unioned into the existing slot" + ); + } + + #[test] + fn merge_second_run_unions_formats() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90), (970, 250)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots(Some(&existing), &discovered, &["/".to_string()], false); + + assert_eq!( + merged[0].formats, + [(728, 90, None), (970, 250, None)], + "a later audit must retain newly observed formats" + ); + } + + #[test] + fn merge_uses_longest_existing_div_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/broad/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"atf\"\ndiv_id = \"ad-atf-\"\n\ + gam_unit_path = \"/222/atf\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 2, + "prefix match should not append a duplicate" + ); + let broad = merged + .iter() + .find(|slot| slot.id == "broad") + .expect("should keep broad slot"); + assert_eq!( + broad.page_patterns, + ["/broad/*"], + "shorter prefix should not claim the discovered div" + ); + let atf = merged + .iter() + .find(|slot| slot.id == "atf") + .expect("should keep specific slot"); + assert_eq!( + atf.page_patterns, + ["/", "/news/*"], + "longest matching prefix should receive this run's pattern" + ); + } + + #[test] + fn observed_literal_does_not_claim_numeric_siblings() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10", "ad-sidebar-11"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-11")); + assert!(diagnostics.notes.is_empty()); + assert!(diagnostics.unobserved_existing_slot_ids.is_empty()); + } + + #[test] + fn split_sibling_warns_when_tuned_parent_fields_are_not_inherited() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.5\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + let sibling = merged + .iter() + .find(|slot| slot.id == "ad-sidebar-10") + .expect("should append the distinct sibling"); + assert_eq!( + sibling.floor_price, None, + "a distinct placement must not inherit the configured parent's floor" + ); + assert_eq!(diagnostics.notes.len(), 1, "should emit one split warning"); + assert!( + diagnostics.notes[0].contains("discovered div `ad-sidebar-10`"), + "should name the split sibling, got {:?}", + diagnostics.notes + ); + assert!( + diagnostics.notes[0].contains("configured div_id prefix `ad-sidebar-1`"), + "should name the disqualified parent prefix, got {:?}", + diagnostics.notes + ); + assert!( + diagnostics.notes[0].contains("does not inherit"), + "should explain the tuned-field consequence, got {:?}", + diagnostics.notes + ); + } + + #[test] + fn refused_stem_observes_prefix_without_disqualifying_prefix_routing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-x\"\ndiv_id = \"ad-x\"\n\ + gam_unit_path = \"/222/ad-x\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.5\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "ad-x-stable", + "ad-x-stable", + Some("/222/ad-x".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + )]; + + let (merged, diagnostics) = merge_render_slots_with_observed_diagnostics( + Some(&existing), + discovered, + &["ad-x".to_string(), "ad-x-stable".to_string()], + &["ad-x-stable".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 1, + "the configured prefix should absorb its sibling" + ); + assert_eq!(merged[0].page_patterns, ["/", "/news/*"]); + assert!( + diagnostics.notes.is_empty(), + "a refused stem is not a literal split boundary" + ); + assert!( + diagnostics.unobserved_existing_slot_ids.is_empty(), + "the refused stem should still prove the configured prefix was observed" + ); + } + + #[test] + fn split_sibling_warns_only_for_longest_parent_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.0\n\n\ + [[slot]]\nid = \"side\"\ndiv_id = \"ad-side\"\n\ + gam_unit_path = \"/222/side\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 2.0\n", + ); + let discovered = ["ad", "ad-side", "ad-sidebar"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/new".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (_, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + diagnostics.notes.len(), + 1, + "should name only the routing ancestor" + ); + assert!( + !diagnostics + .notes + .iter() + .any(|note| note.contains("prefix `ad`")), + "should not claim inheritance from the broader ancestor" + ); + assert!( + diagnostics + .notes + .iter() + .any(|note| note.contains("prefix `ad-side`")), + "the narrower tuned ancestor should be named" + ); + } + + #[test] + fn split_sibling_does_not_warn_about_a_tuned_broader_ancestor() { + let existing = existing_config( + r#" + gam_network_id = "222" + [[slot]] + id = "broad" + div_id = "ad" + gam_unit_path = "/222/broad" + page_patterns = ["/"] + formats = [{ width = 300, height = 250 }] + floor_price = 1.0 + [[slot]] + id = "side" + div_id = "ad-side" + gam_unit_path = "/222/side" + page_patterns = ["/"] + formats = [{ width = 300, height = 250 }] + "#, + ); + let discovered = ["ad", "ad-side", "ad-sidebar"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/new".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + let (_, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + assert!( + diagnostics.notes.is_empty(), + "should not claim a broader floor price would have been inherited" + ); + } + + #[test] + fn refused_but_written_literal_keeps_a_sibling_distinct_during_merge() { + let existing = existing_config( + r#" + gam_network_id = "222" + [[slot]] + id = "ad-x" + div_id = "ad-x" + gam_unit_path = "/222/original" + page_patterns = ["/"] + formats = [{ width = 300, height = 250 }] + floor_price = 5.0 + "#, + ); + let accepted = gpt_slots::DiscoveredSlots { + had_slot_evidence: true, + slots: ["ad-x", "ad-x-extra"] + .into_iter() + .map(|div_id| gpt_slots::DiscoveredSlot { + id: div_id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: format!("/222/{div_id}"), + formats: vec![(300, 250)], + has_prebid: false, + }) + .collect(), + ..Default::default() + }; + let refused = gpt_slots::DiscoveredSlots { + had_slot_evidence: true, + refused_div_ids: BTreeSet::from(["ad-x".to_string()]), + ..Default::default() + }; + for reverse in [false, true] { + let mut table = super::super::evidence::EvidenceTable::default(); + if reverse { + table.fold_page("/news", &refused); + } + table.fold_page("/", &accepted); + if !reverse { + table.fold_page("/news", &refused); + } + let discovered = accepted + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, &["/".to_string()])) + .collect(); + let (merged, _) = merge_render_slots_with_observed_diagnostics( + Some(&existing), + discovered, + &table + .observed_div_ids() + .map(str::to_string) + .collect::>(), + &table + .observed_literals() + .map(str::to_string) + .collect::>(), + false, + ); + assert_eq!( + merged.len(), + 2, + "should keep the sibling regardless of refusal order" + ); + let sibling = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-x-extra")) + .expect("should keep the sibling"); + assert_eq!( + sibling.floor_price, None, + "should not inherit the configured floor price" + ); + assert_eq!( + sibling.gam_unit_path.as_deref(), + Some("/222/ad-x-extra"), + "should preserve the observed unit path" + ); + } + } + + #[test] + fn newly_appended_literal_does_not_claim_numeric_sibling() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"legacy\"\ndiv_id = \"legacy-slot\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-1")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(diagnostics.notes.is_empty()); + } + + #[test] + fn normalized_stem_is_the_literal_merge_boundary() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-header-0\"\ndiv_id = \"ad-header-0\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "ad-header-0-_R_3f_".to_string(), + sizes: vec![(728, 90)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "ad-header-01".to_string(), + sizes: vec![(728, 90)], + }, + ]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots(Some(&existing), &discovered, &["/".to_string()], false); + + assert_eq!(merged.len(), 2); + assert!(merged.iter().any(|slot| slot.id == "ad-header-0")); + assert!(merged.iter().any(|slot| slot.id == "ad-header-01")); + } + + #[test] + fn merge_reports_when_a_broad_prefix_claims_multiple_discovered_divs() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = vec![ + RenderSlot::from_evidence( + "header", + "ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "footer", + "ad-footer", + Some("/222/footer".to_string()), + [(300, 250)], + vec!["/".to_string()], + false, + ), + ]; + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + merged.len(), + 1, + "the configured prefix still controls merging" + ); + assert_eq!(diagnostics.notes.len(), 1); + assert!(diagnostics.notes[0].contains("matched 2 discovered divs")); + assert!(diagnostics.notes[0].contains("ad-footer")); + assert!( + diagnostics.notes[0] + .contains("runtime can resolve this configured slot to at most one"), + "diagnostic should explain the runtime consequence" + ); + assert!(diagnostics.notes[0].contains("ad-header")); + } + + #[test] + fn a_slot_appended_this_run_never_absorbs_a_later_discovery() { + // Prefix reconciliation belongs to the operator's config. If a slot + // appended during this run could act as a prefix, `ad-top` would swallow + // `ad-top-sidebar` whenever discovery happened to see it first, dropping + // the absorbed slot's unit path and provider state, and no broad-prefix + // diagnostic would report it. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"sidebar-ad\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 600 }]\n", + ); + let candidates = [ + RenderSlot::from_evidence( + "ad-top", + "ad-top", + Some("/222/top".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "ad-top-sidebar", + "ad-top-sidebar", + Some("/222/top-sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + true, + ), + ]; + + for order in [[0_usize, 1], [1, 0]] { + let discovered: Vec = order + .iter() + .map(|index| candidates[*index].clone()) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert!( + diagnostics.notes.is_empty(), + "no configured prefix claimed a discovered div in order {order:?}, got {diagnostics:?}" + ); + assert_eq!( + merged.len(), + 3, + "both discovered slots must survive in order {order:?}" + ); + let sidebar_ad = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-top-sidebar")) + .unwrap_or_else(|| { + panic!("the longer div must stay its own slot in order {order:?}") + }); + assert_eq!( + sidebar_ad.gam_unit_path.as_deref(), + Some("/222/top-sidebar"), + "the absorbed slot's unit path must survive in order {order:?}" + ); + assert_eq!( + sidebar_ad.page_patterns, + ["/news/*"], + "patterns must not be pooled in order {order:?}" + ); + assert!( + sidebar_ad.prebid_bidders.is_some(), + "provider state must survive in order {order:?}" + ); + let top = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-top")) + .unwrap_or_else(|| panic!("the shorter div must stay in order {order:?}")); + assert_eq!( + top.page_patterns, + ["/"], + "the longer slot's pattern must not leak into the shorter one in order {order:?}" + ); + } + } + + #[test] + fn merge_renames_new_slot_id_that_collides_with_existing_config() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header-main\"\ndiv_id = \"legacy-header\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "div-gpt-ad-header.main".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + let ids = merged + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + + #[test] + fn merge_keeps_existing_only_slots() { + // Existing has header + sidebar; this run re-sees only header. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); + let sidebar = merged + .iter() + .find(|slot| slot.id == "sidebar") + .expect("sidebar"); + assert_eq!( + sidebar.floor_price, + Some(0.5), + "hand-tuned fields preserved" + ); + } + + #[test] + fn merge_reports_preserved_unobserved_slots_in_config_order() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"footer\"\ndiv_id = \"ad-footer\"\n\ + gam_unit_path = \"/222/footer\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "header", + "div-gpt-ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + )]; + + let (_, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + diagnostics.unobserved_existing_slot_ids, + ["sidebar", "footer"], + "unobserved slots should retain configuration order" + ); + + let all_discovered = vec![ + RenderSlot::from_evidence( + "header", + "div-gpt-ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "sidebar", + "ad-sidebar", + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ), + RenderSlot::from_evidence( + "footer", + "ad-footer", + Some("/222/footer".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + ]; + let (_, fully_observed) = + merge_render_slots_with_diagnostics(Some(&existing), all_discovered.clone(), false); + let (_, replaced) = + merge_render_slots_with_diagnostics(Some(&existing), all_discovered.clone(), true); + let (_, no_existing) = merge_render_slots_with_diagnostics(None, all_discovered, false); + + assert!( + fully_observed.unobserved_existing_slot_ids.is_empty(), + "fully observed slots should not be reported as stale" + ); + assert!( + replaced.unobserved_existing_slot_ids.is_empty(), + "--replace should not report discarded existing slots as stale" + ); + assert!( + no_existing.unobserved_existing_slot_ids.is_empty(), + "a config without existing slots should not report stale slots" + ); + } + + #[test] + fn observed_div_marks_exact_slot_and_live_broad_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "header", + "ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/news/*".to_string()], + false, + )]; + + let (_, diagnostics) = merge_render_slots_with_observed_diagnostics( + Some(&existing), + discovered, + &["ad-header".to_string()], + &["ad-header".to_string()], + false, + ); + + assert!( + diagnostics.unobserved_existing_slot_ids.is_empty(), + "the exact slot and every live configured prefix should be observed, got {diagnostics:?}" + ); + } + + #[test] + fn merge_replace_wipes_existing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + true, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); + } + + #[test] + fn resolve_network_id_prefers_discovered_unless_preserving_existing() { + let with_slots = existing_config( + "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ + gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let empty = existing_config("gam_network_id = \"111\"\n"); + + // Real merge → keep existing. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), + Some("111") + ); + // Placeholder section with no slots → discovered wins. + assert_eq!( + resolve_network_id(Some(&empty), Some("222"), false).as_deref(), + Some("222") + ); + // --replace → discovered wins. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), + Some("222") + ); + // No existing config → discovered. + assert_eq!( + resolve_network_id(None, Some("222"), false).as_deref(), + Some("222") + ); + } + + #[test] + fn toml_key_quotes_only_non_bare_keys() { + assert_eq!(toml_key("zone"), "zone"); + assert_eq!(toml_key("ad-loc"), "ad-loc"); + assert_eq!(toml_key("a.b"), "\"a.b\""); + assert_eq!(toml_key("with space"), "\"with space\""); + assert_eq!(toml_key(""), "\"\""); + } + + #[test] + fn toml_string_escapes_quotes_backslashes_and_controls() { + assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); + assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); + } + + #[test] + fn toml_string_escapes_del_control_char() { + assert_eq!(toml_string("a\u{7f}b"), "\"a\\u007Fb\""); + let doc = format!("value = {}", toml_string("a\u{7f}b")); + let value = toml::from_str::(&doc).expect("DEL escapes to valid TOML"); + assert_eq!( + value["value"].as_str(), + Some("a\u{7f}b"), + "escaped DEL round-trips as data" + ); + } + + #[test] + fn replace_key_handles_inline_commented_headers() { + let document = "[creative_opportunities] # managed\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let updated = replace_key_in_section( + document, + "creative_opportunities", + "gam_network_id", + "gam_network_id = \"222\"", + ) + .expect("should find the commented section header"); + + assert!( + updated.contains("gam_network_id = \"222\""), + "key replaced under a commented header" + ); + assert!( + updated.contains("enabled = true"), + "later commented section left untouched" + ); + } + + #[test] + fn render_quotes_exotic_targeting_keys_to_valid_toml() { + let existing = existing_config( + "gam_network_id = \"1\"\n\n\ + [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ + targeting = { \"a.b\" = \"x\" }\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + let doc = format!( + "[creative_opportunities]\ngam_network_id = \"1\"\n{}", + render_slots(&merged) + ); + + toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs new file mode 100644 index 000000000..06b7c955d --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -0,0 +1,1028 @@ +//! Infers a `{network_id}`/`{section}` ad-unit template from observed evidence. +//! +//! The generator otherwise writes the literal path each page happened to +//! request, which pins a slot to the one section it was scraped from. A template +//! generalizes across sections — but a *wrong* template makes the publisher bid +//! against inventory that does not exist, which is worse than a narrow literal. +//! So this module is built to refuse rather than guess. +//! +//! The inference applies three evidence rules: +//! +//! 1. **Positional binding.** `{network_id}` is bound to unit segment 0 and only +//! if that segment is the resolved network id. Substring replacement would +//! corrupt `/123/sports123/home` into `/{network_id}/sports{network_id}/home`. +//! 2. **Exactly one varying segment.** Zero means nothing was proven and the +//! path stays literal; two means the unit varies along a dimension the +//! request path cannot supply (device, geo, experiment), so it is refused. +//! 3. **Cross-page variation.** Two pages must show *different* derived sections +//! and different unit segments. A single-page crawl is +//! indistinguishable from a static path — literal, `{network_id}`-only and +//! `{section}` all reproduce one observation equally well, and round-trip +//! verification cannot tell them apart. Only variation can. +//! +//! Every accepted template is then replayed through the runtime's own +//! [`render_gam_unit_path`](CreativeOpportunitySlot::render_gam_unit_path) and +//! [`derive_section`] against every observation. A template that does not +//! reproduce what the live page actually requested is downgraded, not written. + +use std::collections::{BTreeMap, BTreeSet}; + +use trusted_server_core::creative_opportunities::{CreativeOpportunitySlot, derive_section}; + +use super::evidence::{EvidenceTable, SlotEvidence}; +use super::slot_toml::toml_string; + +/// Candidate `section_segment` values considered, `0..=MAX_SECTION_SEGMENT`. +/// +/// A locale-prefixed site (`/en/news/story`) needs 1. Beyond 2 the "section" is +/// no longer a taxonomy the operator would recognise, and every extra candidate +/// is another chance for two indices to both fit and force a refusal. +const MAX_SECTION_SEGMENT: usize = 2; + +/// The config-level section policy an inferred template depends on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SectionPolicy { + /// Value substituted for `{section}` on paths with no section segment. + pub(super) section_root: String, + /// Index of the path segment `{section}` is taken from. + pub(super) section_segment: usize, +} + +/// What to write for one slot's `gam_unit_path`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum SlotDecision { + /// Write this templated path; it reproduced every observation. + Template(String), + /// Write this literal path; nothing generalizable was proven. + Literal(String), + /// Write no path at all — the observations cannot be represented. + Refuse { + /// Operator-facing explanations, one per reason. + reasons: Vec, + }, +} + +/// The outcome of inference across the whole evidence table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct InferenceOutcome { + /// Section policy to write, present only when some slot templated. + pub(super) policy: Option, + /// Per-slot decision, keyed by div stem, in evidence order. + pub(super) decisions: Vec<(String, SlotDecision)>, + /// Operator-facing notes about why inference went the way it did. + pub(super) diagnostics: Vec, + /// Div stems whose templates rely on a root witnessed by another slot. + pub(super) borrowed_section_root: Vec, +} + +impl InferenceOutcome { + /// The decision for a slot, by div stem. + pub(super) fn decision(&self, div_id: &str) -> Option<&SlotDecision> { + self.decisions + .iter() + .find(|(key, _)| key == div_id) + .map(|(_, decision)| decision) + } +} + +/// Per-slot analysis under one candidate `section_segment`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum SlotAnalysis { + /// Templatable: unit segment `varying` tracks the derived section, and root + /// pages agreed on `section_root`. + Templatable { + varying: usize, + section_root: String, + }, + /// The unit path never varied, so nothing about `{section}` was proven. + Static, + /// Cannot be represented; carries the operator-facing reason. + Refuse(String), + /// Unit segment `varying` tracks the derived section on every page this slot + /// was seen on, but none of those pages lacked the section segment, so the + /// slot witnessed no `section_root` of its own. + /// + /// Carries `varying` because such a slot is still templatable *when another + /// slot witnessed the config-level `section_root`*: a placement that only + /// exists on section pages (a sidebar, an in-article unit) never renders on a + /// path where `{section}` would fall back to the root. + RootUnwitnessed { varying: usize }, +} + +/// Infers unit-path templates for every slot in `table`. +/// +/// `network_id` is the resolved GAM network id; `{network_id}` is only ever +/// bound to a unit segment that already equals it. +pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> InferenceOutcome { + let slots: Vec<&SlotEvidence> = table.slots().collect(); + let mut diagnostics = Vec::new(); + + // Evaluate every candidate index independently; ambiguity between two that + // both fit is a refusal, not a preference for the smaller one. + let mut qualifying: Vec<(usize, String, BTreeMap)> = Vec::new(); + let mut root_witness_missing = false; + let mut root_unwitnessed_stems = BTreeSet::new(); + for segment in 0..=MAX_SECTION_SEGMENT { + let analyses: BTreeMap = slots + .iter() + .map(|slot| (slot.div_id.clone(), analyse_slot(slot, network_id, segment))) + .collect(); + + let roots: BTreeSet<&str> = analyses + .values() + .filter_map(|analysis| match analysis { + SlotAnalysis::Templatable { section_root, .. } => Some(section_root.as_str()), + _ => None, + }) + .collect(); + // Slots must agree: `section_root` is one config-level value, so two + // slots claiming different roots means this index is not the real one. + let Some(root) = roots.iter().next().copied() else { + // Distinguish "nothing tracks the section" from "everything does but + // no crawled page lacked the section segment": the second is a crawl + // gap the operator can close, and the generic literal-path refusal + // below does not say so. + root_witness_missing |= analyses + .values() + .any(|analysis| matches!(analysis, SlotAnalysis::RootUnwitnessed { .. })); + root_unwitnessed_stems.extend( + analyses + .iter() + .filter(|(_, analysis)| { + matches!(analysis, SlotAnalysis::RootUnwitnessed { .. }) + }) + .map(|(stem, _)| stem.clone()), + ); + continue; + }; + if roots.len() > 1 { + continue; + } + qualifying.push((segment, root.to_string(), analyses)); + } + + let chosen = match qualifying.len() { + 0 => None, + 1 => qualifying.into_iter().next(), + _ => { + let indices: Vec = qualifying + .iter() + .map(|(segment, _, _)| segment.to_string()) + .collect(); + diagnostics.push(format!( + "more than one section_segment ({}) explains the observed ad-unit paths \ + equally well, so no template can be chosen safely; slots without one safe literal path are omitted", + indices.join(", ") + )); + None + } + }; + + let Some((section_segment, section_root, analyses)) = chosen else { + // Only a crawl gap justifies rewriting the per-slot reasons. When + // inference stopped on segment ambiguity instead, that pushed its own + // diagnostic, and blaming the crawl here would send the operator to + // widen it when the remedy is pinning `section_segment`. + let root_gap = diagnostics.is_empty() && root_witness_missing; + if diagnostics.is_empty() { + diagnostics.push(if root_witness_missing { + "the ad-unit paths do track the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed and no {section} \ + template can be written; include the site root in the crawl (or set \ + section_root by hand) to template these slots" + .to_string() + } else { + "no ad-unit path varied by page section across the crawl, so paths were kept \ + literal; crawl more sections to enable a {section} template" + .to_string() + }); + } + let mut decisions = literal_decisions(&slots); + if root_gap { + for (stem, decision) in &mut decisions { + if root_unwitnessed_stems.contains(stem) + && let SlotDecision::Refuse { reasons } = decision + { + *reasons = vec![ + "the paths tracked the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed" + .to_string(), + ]; + } + } + } + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + borrowed_section_root: Vec::new(), + }; + }; + + let mut decisions = Vec::with_capacity(slots.len()); + let mut borrowed_section_root = Vec::new(); + let mut templated = 0_usize; + for slot in &slots { + let analysis = analyses + .get(&slot.div_id) + .cloned() + .unwrap_or(SlotAnalysis::Static); + let templatable = match analysis { + SlotAnalysis::Templatable { varying, .. } => Some((varying, true)), + // The config-level `section_root` is witnessed by another slot on the + // same property, and this slot's page patterns are derived from the + // paths it was seen on — all of which carry a section segment — so + // `{section}` never falls back to the root for it. Refusing here cost + // real inventory: a sidebar or in-article unit that simply does not + // exist on the site root was omitted from the config entirely. + SlotAnalysis::RootUnwitnessed { varying } => Some((varying, false)), + SlotAnalysis::Static | SlotAnalysis::Refuse(_) => None, + }; + let decision = match (templatable, analysis) { + (Some((varying, witnessed_root)), _) => { + let template = build_template(slot, varying); + match verify_round_trip(&template, slot, network_id, §ion_root, section_segment) + { + Ok(()) => { + templated += 1; + if !witnessed_root { + borrowed_section_root.push(slot.div_id.clone()); + diagnostics.push(format!( + "slot `{}` was never observed on a page without a section \ + segment, so its `{{section}}` template relies on the \ + config-level section_root `{section_root}` witnessed by other \ + slots; it is only rendered for the paths this slot was seen on", + slot.id + )); + } + SlotDecision::Template(template) + } + Err(reason) => { + diagnostics.push(format!( + "slot `{}` template `{template}` did not reproduce the observed \ + ad-unit paths ({reason}); refusing any unsafe fallback", + slot.id + )); + literal_decision(slot) + } + } + } + (None, SlotAnalysis::Refuse(reason)) => SlotDecision::Refuse { + reasons: vec![reason], + }, + (None, _) => literal_decision(slot), + }; + decisions.push((slot.div_id.clone(), decision)); + } + + if templated == 0 { + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + borrowed_section_root: Vec::new(), + }; + } + + diagnostics.push(format!( + "inferred section_segment = {section_segment} and section_root = \"{section_root}\" \ + from {} page(s); {templated} slot(s) templated", + table.pages().len() + )); + InferenceOutcome { + policy: Some(SectionPolicy { + section_root, + section_segment, + }), + decisions, + diagnostics, + borrowed_section_root, + } +} + +/// Checks the properties of a slot's observations that do not depend on which +/// `section_segment` is being considered. +/// +/// Kept separate because these refusals are final: no candidate index can +/// rescue a slot whose observations are not one template with a single hole in +/// them, and the operator needs the specific reason rather than a generic one. +/// +/// Returns the single varying unit segment, `None` when nothing varied, or the +/// reason the observations cannot be represented at all. +fn structural_check(slot: &SlotEvidence) -> Result, String> { + // One page reporting two different ad-unit paths for the same slot means the + // unit varies along something the request path cannot express — a device or + // geo split, or two profiles disagreeing. Nothing here can represent that. + let mut per_path: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for row in &slot.rows { + per_path + .entry(row.path.as_str()) + .or_default() + .insert(row.unit_path.as_str()); + } + if let Some((path, units)) = per_path.iter().find(|(_, units)| units.len() > 1) { + let observed: Vec<&str> = units.iter().copied().collect(); + return Err(format!( + "page `{path}` requested more than one ad-unit path for this slot ({}); \ + the unit varies by something the request path cannot derive", + observed.join(", ") + )); + } + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + let Some(first) = split.first() else { + return Ok(None); + }; + // Differing shapes are not one template with a hole in it. + if split.iter().any(|parts| parts.len() != first.len()) { + return Err( + "the observed ad-unit paths have different segment counts, so they are not \ + one template" + .to_string(), + ); + } + + let varying: Vec = (0..first.len()) + .filter(|index| { + split + .iter() + .map(|parts| parts[*index]) + .collect::>() + .len() + > 1 + }) + .collect(); + match varying.len() { + 0 => Ok(None), + 1 if varying[0] == 0 => { + Err("the network-id segment of the ad-unit path varied across pages".to_string()) + } + 1 => Ok(Some(varying[0])), + count => Err(format!( + "{count} ad-unit segments vary across pages, so the path does not track the \ + page section alone" + )), + } +} + +/// Analyses one slot under a candidate `section_segment`. +/// +/// [`structural_check`] has already established that a templatable candidate +/// contains more than one observed unit path. Therefore a successful derived +/// section match here is itself the required variation witness; a second +/// witness predicate would only restate that invariant. +fn analyse_slot(slot: &SlotEvidence, network_id: &str, section_segment: usize) -> SlotAnalysis { + let varying = match structural_check(slot) { + Err(reason) => return SlotAnalysis::Refuse(reason), + Ok(None) => return SlotAnalysis::Static, + Ok(Some(varying)) => varying, + }; + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + // `{network_id}` binds positionally and only to the resolved id. Substring + // replacement would rewrite an unrelated segment that merely contains it. + if split.first().and_then(|parts| parts.first()) != Some(&network_id) { + return SlotAnalysis::Static; + } + + // Partition observations into pages that have a section segment and pages + // that do not; the latter are what determine `section_root`. + let mut root_values = BTreeSet::new(); + for (row, parts) in slot.rows.iter().zip(split.iter()) { + let observed = parts[varying]; + if path_segments(&row.path).len() > section_segment { + // The empty root is unused here: the path has this segment. + if derive_section(&row.path, "", section_segment) != observed { + return SlotAnalysis::Static; + } + } else { + root_values.insert(observed); + } + } + + let mut roots = root_values.into_iter(); + let Some(section_root) = roots.next() else { + // Without a root observation, `section_root` would be a guess that + // silently mis-renders every short path. + return SlotAnalysis::RootUnwitnessed { varying }; + }; + if roots.next().is_some() { + return SlotAnalysis::Static; + } + // A root that is not `[A-Za-z0-9_-]+` makes any `{section}` template fail + // config load; catch it here rather than at push time. + if section_root.is_empty() + || !section_root + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + { + return SlotAnalysis::Static; + } + + SlotAnalysis::Templatable { + varying, + section_root: section_root.to_string(), + } +} + +/// Builds the template text by substituting the two proven placeholders. +fn build_template(slot: &SlotEvidence, varying: usize) -> String { + let first = slot + .rows + .iter() + .next() + .map(|row| row.unit_path.as_str()) + .unwrap_or_default(); + let rendered: Vec = segments(first) + .into_iter() + .enumerate() + .map(|(index, value)| { + if index == 0 { + "{network_id}".to_string() + } else if index == varying { + "{section}".to_string() + } else { + value.to_string() + } + }) + .collect(); + format!("/{}", rendered.join("/")) +} + +/// Replays `template` through the runtime renderer against every observation. +/// +/// Defense in depth rather than the primary gate: [`analyse_slot`] already +/// refuses to call a slot templatable when the derived section and the observed +/// segment disagree — a publisher whose `/site-news` pages request +/// `.../sitenews`, say — so a mismatch reaching here would mean inference and +/// the runtime renderer disagree. The template is then dropped instead of +/// written, and the diagnostic names the paths that did not reproduce. +fn verify_round_trip( + template: &str, + slot: &SlotEvidence, + network_id: &str, + section_root: &str, + section_segment: usize, +) -> Result<(), String> { + let probe = probe_slot(template)?; + for row in &slot.rows { + let section = derive_section(&row.path, section_root, section_segment); + match probe.render_gam_unit_path(network_id, §ion) { + Some(rendered) if rendered == row.unit_path => {} + Some(rendered) => { + return Err(format!( + "on `{}` it renders `{rendered}` but the page requested `{}`", + row.path, row.unit_path + )); + } + None => { + return Err(format!( + "on `{}` it renders past the GAM ad-unit path byte limit", + row.path + )); + } + } + } + Ok(()) +} + +/// Builds a throwaway slot carrying `template`, for rendering only. +/// +/// Deserializing is how the runtime itself builds slots, so this exercises the +/// same template parsing rather than a parallel implementation. +fn probe_slot(template: &str) -> Result { + let document = format!( + "id = \"probe\"\ngam_unit_path = {}\npage_patterns = [\"/\"]\n\ + formats = [{{ width = 1, height = 1 }}]\n", + toml_string(template) + ); + toml::from_str::(&document) + .map_err(|error| format!("template is not representable in config: {error}")) +} + +/// The decision for a slot no template was proven for. +/// +/// A structural refusal wins over the generic "several paths" message, so the +/// operator sees *why* the slot could not be represented (a device split, an +/// extra varying dimension) rather than only that it could not. +fn literal_decision(slot: &SlotEvidence) -> SlotDecision { + if let Err(reason) = structural_check(slot) { + return SlotDecision::Refuse { + reasons: vec![reason], + }; + } + let units = slot.unit_paths(); + let mut found = units.iter(); + match (found.next(), found.next()) { + (Some(only), None) => SlotDecision::Literal((*only).to_string()), + (Some(_), Some(_)) => SlotDecision::Refuse { + reasons: vec![format!( + "the slot used several ad-unit paths ({}) and none generalized, so no \ + single literal path is correct", + units.into_iter().collect::>().join(", ") + )], + }, + _ => SlotDecision::Refuse { + reasons: vec!["no ad-unit path was observed for this slot".to_string()], + }, + } +} + +fn literal_decisions(slots: &[&SlotEvidence]) -> Vec<(String, SlotDecision)> { + slots + .iter() + .map(|slot| (slot.div_id.clone(), literal_decision(slot))) + .collect() +} + +/// Non-empty path segments of an ad-unit path. +fn segments(unit_path: &str) -> Vec<&str> { + unit_path + .split('/') + .filter(|part| !part.is_empty()) + .collect() +} + +/// Non-empty path segments of a request path. +fn path_segments(path: &str) -> Vec<&str> { + path.split('/').filter(|part| !part.is_empty()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// Folds `(path, unit_path)` observations for one div into a table. + fn table_for(div_id: &str, observations: &[(&str, &str)]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, unit_path) in observations { + let registry = vec![CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: div_id.to_string(), + sizes: vec![(728, 90)], + }]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + /// Folds pages carrying different slot sets into one table. + /// + /// Each entry is `(request path, [(div id, ad-unit path)])`. + fn table_for_pages(pages: &[(&str, &[(&str, &str)])]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + fn only_decision(outcome: &InferenceOutcome) -> &SlotDecision { + assert_eq!(outcome.decisions.len(), 1, "fixture should have one slot"); + &outcome.decisions[0].1 + } + + #[test] + fn templates_a_section_varying_unit_path() { + // The shape the operator writes by hand today. + let table = table_for( + "ad-header", + &[ + ("/", "/123456789/publisher/homepage"), + ("/news/story-abc", "/123456789/publisher/news"), + ("/deals/thing", "/123456789/publisher/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123456789"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }) + ); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/publisher/{section}".to_string()) + ); + } + + #[test] + fn a_single_page_never_templates() { + // Literal, {network_id}-only and {section} all reproduce one observation, + // so only variation can distinguish them. This is the witness rule. + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/news".to_string()) + ); + } + + #[test] + fn a_static_unit_path_across_sections_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/fixed"), + ("/news/story", "/123/site/fixed"), + ("/deals/x", "/123/site/fixed"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None, "nothing varied, so nothing is proven"); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/fixed".to_string()) + ); + } + + #[test] + fn a_device_split_is_refused_rather_than_guessed() { + // Two units for the SAME path: the desktop/mobile cross-check surfaces + // here, and the request path cannot express the difference. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/news/story", "/123/mobile/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!( + "a device split must refuse, got {:?}", + only_decision(&outcome) + ); + }; + assert!( + reasons[0].contains("more than one ad-unit path"), + "reason should name the conflict, got {reasons:?}" + ); + } + + #[test] + fn two_varying_segments_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/deals/x", "/123/mobile/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two varying dimensions must refuse"); + }; + assert!( + reasons[0].contains("segments vary"), + "reason should name the extra dimension, got {reasons:?}" + ); + } + + #[test] + fn a_slug_the_path_cannot_reproduce_is_refused() { + // `/site-news` requests `.../sitenews`: the derived section and + // the observed segment differ, so the template would render the wrong + // unit. Candidate analysis rejects the inconsistent section mapping. + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news"), + ("/site-news/x", "/123/site/sitenews"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "a section whose slug is not derivable must not template" + ); + assert!(matches!( + only_decision(&outcome), + SlotDecision::Refuse { .. } + )); + } + + #[test] + fn an_unwitnessed_root_is_refused() { + // Every crawled page had a section, so `section_root` would be a guess + // that silently mis-renders the homepage. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/site/news"), + ("/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two literal paths and no template is not representable as one literal"); + }; + assert!( + reasons + .iter() + .any(|reason| reason.contains("section_root") && reason.contains("witnessed")), + "the per-slot reason should name the crawl gap; got {reasons:?}" + ); + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("section_root` could not be witnessed")), + "the crawl gap, not \"nothing generalized\", is the reason; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn a_slot_absent_from_the_root_templates_from_the_witnessed_policy() { + // The live shape behind the `ad-atf_sidebar-0` refusal: a header on the + // root and every section witnesses `section_root`, while a sidebar exists + // only on section pages. The sidebar's unit path tracks the section just + // as well, and its page patterns never cover the root, so refusing it + // dropped real inventory from the config. + let mut table = EvidenceTable::default(); + let pages: &[(&str, &[(&str, &str)])] = &[ + ("/", &[("ad-header", "/123/site/homepage")]), + ( + "/news/story", + &[ + ("ad-header", "/123/site/news"), + ("ad-sidebar", "/123/site/news"), + ], + ), + ( + "/deals/x", + &[ + ("ad-header", "/123/site/deals"), + ("ad-sidebar", "/123/site/deals"), + ], + ), + ]; + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }), + "the header witnesses the config-level policy" + ); + assert_eq!( + outcome.decision("ad-sidebar"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )), + "a slot that only exists on section pages is still templatable" + ); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.borrowed_section_root, + ["ad-sidebar".to_string()], + "the outcome should identify templates whose safety depends on derived patterns" + ); + assert!( + outcome.diagnostics.iter().any(|note| note + .contains("`ad-sidebar` was never observed on a page without a section segment")), + "the borrowed section_root should be stated; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn segment_ambiguity_does_not_blame_the_crawl_for_an_unwitnessed_root() { + // `ad-header` fits section_segment 0 and `ad-locale` fits 1, so + // inference stops on ambiguity. `ad-deep` is separately + // `RootUnwitnessed` at segment 2. Its refusal must not tell the + // operator to widen the crawl when the remedy is pinning + // `section_segment`. + let table = table_for_pages(&[ + ("/", &[("ad-header", "/99/site/home")]), + ("/news", &[("ad-header", "/99/site/news")]), + ("/en", &[("ad-locale", "/99/site/en-root")]), + ("/en/news", &[("ad-locale", "/99/site/news")]), + ("/a/b/news", &[("ad-deep", "/99/site/news")]), + ("/a/b/deals", &[("ad-deep", "/99/site/deals")]), + ]); + + let outcome = infer_unit_templates(&table, "99"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("more than one section_segment")), + "the fixture should stop on ambiguity; got {:?}", + outcome.diagnostics + ); + assert!( + !outcome + .diagnostics + .iter() + .any(|note| note.contains("include the site root in the crawl")), + "an ambiguous run must not also blame the crawl; got {:?}", + outcome.diagnostics + ); + let Some(SlotDecision::Refuse { reasons }) = outcome.decision("ad-deep") else { + panic!("expected a refusal, got {:?}", outcome.decision("ad-deep")); + }; + assert!( + reasons + .iter() + .all(|reason| !reason.contains("no crawled page lacked a section segment")), + "the crawl-gap reason belongs only to a run that stopped on the crawl gap; got {reasons:?}" + ); + } + + #[test] + fn a_locale_prefixed_site_infers_the_deeper_segment() { + let table = table_for( + "ad-header", + &[ + ("/en", "/123/site/homepage"), + ("/en/news/story", "/123/site/news"), + ("/en/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }), + "the locale prefix should push the section one segment deeper" + ); + } + + #[test] + fn network_id_is_bound_positionally_not_by_substring() { + // `sports123` merely contains the network id; substring replacement + // would corrupt it into `sports{network_id}`. + let table = table_for( + "ad-header", + &[ + ("/", "/123/sports123/homepage"), + ("/news/story", "/123/sports123/news"), + ("/deals/x", "/123/sports123/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/sports123/{section}".to_string()), + "only segment 0 may become {{network_id}}" + ); + } + + #[test] + fn a_unit_path_not_starting_with_the_network_id_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/999/site/homepage"), + ("/news/story", "/999/site/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "segment 0 must equal the resolved network id" + ); + } + + #[test] + fn differing_segment_counts_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news/extra"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("differing shapes are not one template"); + }; + assert!( + reasons[0].contains("segment counts"), + "reason should name the shape mismatch, got {reasons:?}" + ); + } + + #[test] + fn a_static_slot_stays_literal_alongside_a_templated_one() { + let mut table = EvidenceTable::default(); + for (path, section_unit) in [ + ("/", "homepage"), + ("/news/story", "news"), + ("/deals/x", "deals"), + ] { + let registry = vec![ + CollectedGptSlot { + gam_unit_path: format!("/123/site/{section_unit}"), + div_id: "ad-header".to_string(), + sizes: vec![(728, 90)], + }, + CollectedGptSlot { + gam_unit_path: "/123/site/sticky".to_string(), + div_id: "ad-sticky".to_string(), + sizes: vec![(300, 250)], + }, + ]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert!(outcome.policy.is_some(), "the varying slot should template"); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.decision("ad-sticky"), + Some(&SlotDecision::Literal("/123/site/sticky".to_string())), + "a genuinely static slot must not be dragged into the template" + ); + } + + #[test] + fn diagnostics_explain_why_nothing_templated() { + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("crawl more sections")), + "the operator should learn why, got {:?}", + outcome.diagnostics + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/validate.rs b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs new file mode 100644 index 000000000..2fca1dca9 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs @@ -0,0 +1,134 @@ +//! Write-side validation for generated ad-template config. +//! +//! Everything the generator writes is derived from a live, page-controlled ad +//! stack, so the candidate document must pass source-config validation +//! before it replaces the operator's file. A config the +//! runtime rejects is not a degraded ad stack — `build_state` fails and the +//! adapter answers every route from the startup error router, so an unloadable +//! `trusted-server.toml` is a full-site outage once pushed. + +use trusted_server_core::config::TrustedServerAppConfig; + +use crate::error::{CliResult, cli_error}; + +/// Validates the candidate config text the generator is about to persist. +/// +/// Runs [`TrustedServerAppConfig::new`], the push-time validation path, after +/// deserializing the source TOML. This checks slot and template compilation, +/// provider configuration, and secret key references without I/O. Secret values +/// are resolved and validated separately at runtime; treating source key names +/// as resolved secrets would incorrectly reject otherwise valid baselines. +/// +/// `baseline` is the config as it was read from disk. When the baseline is +/// *already* unloadable, this run cannot be blamed for it: the candidate is +/// accepted and the pre-existing error is returned as a warning instead. Without +/// that escape hatch a freshly bootstrapped config carrying placeholder secrets +/// could never be updated by `generate`. +/// +/// # Errors +/// +/// Returns a user-facing error when the candidate fails to load and the baseline +/// loaded cleanly — that is, when this run introduced the failure. +pub(super) fn check_candidate(candidate: &str, baseline: &str) -> CliResult> { + let Err(candidate_error) = validate_source_config(candidate) else { + return Ok(Vec::new()); + }; + + if let Err(baseline_error) = validate_source_config(baseline) { + return Ok(vec![format!( + "target config was already invalid before this run, so the generated \ + result could not be verified: {baseline_error}" + )]); + } + + cli_error(format!( + "refusing to write: the generated config would fail to load, which would \ + take the service down once pushed: {candidate_error}" + )) +} + +fn validate_source_config(source: &str) -> CliResult<()> { + let config: TrustedServerAppConfig = + toml::from_str(source).map_err(|error| error.to_string())?; + TrustedServerAppConfig::new(config.into_settings()) + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A source config containing secret references, as an operator would edit it. + fn baseline() -> String { + crate::commands::config::init::EXAMPLE_CONFIG + .replace("\"example.com\"", "\"publisher.example.com\"") + .replace("\".example.com\"", "\".publisher.example.com\"") + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ) + .replace( + "https://origin.example.com", + "https://origin.publisher.example.com", + ) + } + + #[test] + fn valid_candidate_passes_without_warnings() { + let config = baseline(); + + let warnings = check_candidate(&config, &config).expect("should accept valid candidate"); + + assert!( + warnings.is_empty(), + "a clean candidate should not warn, got {warnings:?}" + ); + } + + #[test] + fn candidate_this_run_broke_is_refused() { + let good = baseline(); + // An empty div_id override is exactly what a div id normalized down to + // nothing would produce, and `validate_runtime` rejects it. + let broken = format!( + "{good}\n[[creative_opportunities.slot]]\n\ + id = \"broken\"\ndiv_id = \"\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n" + ); + + let error = check_candidate(&broken, &good).expect_err("should refuse a broken candidate"); + + assert!( + format!("{error:?}").contains("refusing to write"), + "error should name the refusal, got {error:?}" + ); + } + + #[test] + fn pre_existing_breakage_downgrades_to_a_warning() { + // The operator's file was already unloadable; `generate` must still be + // able to update it rather than blaming this run for the old error. + let broken_baseline = "[creative_opportunities]\n"; + let broken_candidate = "[creative_opportunities]\n"; + + let warnings = check_candidate(broken_candidate, broken_baseline) + .expect("a pre-existing failure should not block the write"); + + assert_eq!(warnings.len(), 1, "should surface exactly one warning"); + assert!( + warnings[0].contains("already invalid"), + "warning should name the pre-existing failure, got {:?}", + warnings[0] + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 8f37e36fc..211060662 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -1,1343 +1,611 @@ -mod analyzer; -pub(crate) mod browser_collector; -pub(crate) mod collector; - -use std::collections::BTreeSet; -use std::fmt::Write as _; -use std::fs; -use std::io::Write; -use std::path::{Path, PathBuf}; - -use rand::RngCore as _; - -use serde::Serialize; -use url::Url; - -use crate::commands::audit::collector::AuditCollector; -use crate::commands::config::init::EXAMPLE_CONFIG; -use crate::error::{CliResult, cli_error, report_error}; - -use analyzer::{analyze_collected_page, extract_gtm_container_id}; - -/// Arguments for the `ts audit` command. -#[derive(Debug, clap::Args)] +//! Browser-backed `ts audit` command namespace. +//! +//! `ts audit page ` is the generic page audit; `ts audit ad-templates verify +//! ...` is the ad-template verifier; `ts audit generate ` bootstraps a +//! draft config from a live page (issue #800). `ts audit ` is a hidden +//! compatibility alias for `ts audit generate `. + +pub mod ad_templates; +pub mod browser; +mod browser_scroll; +pub mod collector; +pub mod generate; +pub mod page; + +use clap::{Args, Subcommand}; + +use crate::app_config::AppConfigArgs; +use crate::commands::audit::collector::{BrowserOpts, GenerateBrowserOpts}; +use crate::commands::audit::page::PageAuditArgs; +use crate::error::{CliResult, cli_error}; +use crate::run::RunOutcome; + +/// Parses and validates an `http`/`https` URL, rejecting all other schemes. +/// +/// # Errors +/// +/// Returns a user-facing string when the input is not a valid `http`/`https` URL. +pub(crate) fn parse_http_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + match url.scheme() { + "http" | "https" => Ok(url), + other => Err(format!( + "unsupported URL scheme `{other}` (expected http or https)" + )), + } +} + +/// Parses a `name=value` cookie argument into its `(name, value)` parts. +/// +/// Splits on the first `=` so cookie values may themselves contain `=`. The name +/// must be non-empty; the value may be empty. +/// +/// # Errors +/// +/// Returns a user-facing string when the input has no `=` or an empty name. +pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { + let (name, value) = raw + .split_once('=') + .ok_or_else(|| format!("invalid cookie `{raw}` (expected NAME=VALUE)"))?; + if name.is_empty() { + return Err(format!("invalid cookie `{raw}` (empty name)")); + } + Ok((name.to_string(), value.to_string())) +} + +/// `ts audit` arguments: an optional subcommand plus a hidden legacy URL positional. +#[derive(Debug, Args)] +#[command(arg_required_else_help = true)] pub(crate) struct AuditArgs { - /// Public HTTP(S) URL to audit. - pub(crate) url: String, + #[command(subcommand)] + pub(crate) command: Option, + /// Hidden compatibility alias: `ts audit ` behaves like `ts audit generate `. + /// + /// The hidden flags below all `requires` this positional, so putting one + /// before a subcommand (`ts audit --chrome X generate `) is rejected + /// rather than silently dropped. `value_name` keeps that rejection from + /// naming the field: an operator told to supply `` cannot find + /// it in `--help`, because the alias is deliberately undocumented. + #[arg(value_parser = parse_http_url, hide = true, value_name = "URL")] + pub(crate) legacy_url: Option, + #[command(flatten)] + pub(crate) legacy_generate: LegacyGenerateArgs, +} + +/// Hidden generation flags retained for the legacy `ts audit ` form. +#[derive(Debug, Default, Args)] +pub(crate) struct LegacyGenerateArgs { /// JavaScript asset audit output path. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) js_assets: Option, /// Draft Trusted Server config output path. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) config: Option, /// Do not write the JavaScript asset audit file. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) no_js_assets: bool, /// Do not write the draft Trusted Server config file. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) no_config: bool, /// Overwrite existing output files. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) force: bool, -} - -const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; -const DEFAULT_CONFIG_PATH: &str = "trusted-server.toml"; - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum AssetParty { - FirstParty, - ThirdParty, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct AuditedAsset { - pub(crate) kind: String, - pub(crate) url: String, - pub(crate) host: String, - pub(crate) party: AssetParty, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) integration: Option, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct DetectedIntegration { - pub(crate) id: String, - pub(crate) evidence: String, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct AuditArtifact { - pub(crate) audited_url: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) page_title: Option, - pub(crate) js_asset_count: usize, - pub(crate) third_party_asset_count: usize, - pub(crate) detected_integrations: Vec, - pub(crate) assets: Vec, - pub(crate) warnings: Vec, -} - -#[derive(Debug, Clone)] -pub(crate) struct AuditOutputs { - pub(crate) artifact: AuditArtifact, - pub(crate) js_assets_toml: String, - pub(crate) draft_config_toml: String, - pub(crate) js_asset_proxy_candidate_count: usize, -} - -#[derive(Debug, Clone)] -struct DraftConfig { - toml: String, - js_asset_proxy_candidate_count: usize, -} - -#[derive(Debug, Clone)] -struct JsAssetProxySection { - toml: String, - candidate_count: usize, -} - -#[derive(Debug, Default)] -struct JsAssetProxySkipCounts { - first_party: usize, - malformed_url: usize, - non_https: usize, - duplicate_url: usize, - non_script: usize, -} - -#[derive(Debug)] -struct JsAssetProxyCandidate<'a> { - origin_url: String, - integration: Option<&'a str>, -} - -trait OpaqueAssetPathGenerator { - fn next_path(&mut self) -> String; -} - -#[derive(Debug, Default)] -struct RandomOpaqueAssetPathGenerator; - -impl OpaqueAssetPathGenerator for RandomOpaqueAssetPathGenerator { - fn next_path(&mut self) -> String { - let mut bytes = [0_u8; 12]; - rand::rngs::OsRng.fill_bytes(&mut bytes); - format!("/assets/{}.js", lowercase_hex(&bytes)) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct AuditOutputPlan { - js_assets_path: Option, - config_path: Option, -} - -pub(crate) fn run_audit( - args: &AuditArgs, - collector: &dyn AuditCollector, - out: &mut dyn Write, -) -> CliResult<()> { - let target_url = parse_audit_url(&args.url)?; - let plan = resolve_output_plan(args)?; - let collected = collector.collect_page(&target_url)?; - let outputs = build_audit_outputs(&collected)?; - let wrote_config = plan.config_path.is_some(); - let written = write_audit_outputs(&outputs, &plan)?; - write_success_summary(&outputs, &written, wrote_config, out) -} - -fn parse_audit_url(value: &str) -> CliResult { - let url = Url::parse(value) - .map_err(|error| report_error(format!("invalid audit URL `{value}`: {error}")))?; - if !matches!(url.scheme(), "http" | "https") { - return cli_error(format!( - "`ts audit` only supports http/https URLs, got `{}`", - url.scheme() - )); - } - Ok(url) -} - -fn resolve_output_plan(args: &AuditArgs) -> CliResult { - if args.no_js_assets && args.no_config { - return cli_error("nothing to do: both --no-js-assets and --no-config were set"); - } - - let js_assets_path = if args.no_js_assets { - None - } else { - Some(resolve_output_path( - args.js_assets.as_deref(), - DEFAULT_JS_ASSETS_PATH, - )?) - }; - let config_path = if args.no_config { - None - } else { - Some(resolve_output_path( - args.config.as_deref(), - DEFAULT_CONFIG_PATH, - )?) - }; - - if js_assets_path.is_some() && js_assets_path == config_path { - return cli_error("audit output paths must be distinct"); - } - - for path in [&js_assets_path, &config_path].into_iter().flatten() { - if path.exists() && !args.force { - return cli_error(format!( - "refusing to overwrite existing file `{}`; re-run with --force", - path.display() - )); + /// Cookie to send with the page request, as `name=value`. Repeatable. + #[arg( + long = "cookie", + value_name = "NAME=VALUE", + value_parser = parse_cookie, + hide = true, + requires = "legacy_url" + )] + pub(crate) cookies: Vec<(String, String)>, + #[command(flatten)] + pub(crate) browser: LegacyBrowserOpts, +} + +/// Hidden browser flags retained for the legacy `ts audit ` form. +#[derive(Debug, Args)] +pub(crate) struct LegacyBrowserOpts { + /// Path to the Chrome/Chromium executable. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) no_assume_consent: bool, + /// Route the browser through this proxy. + #[arg(long, value_name = "HOST:PORT", hide = true, requires = "legacy_url")] + pub(crate) browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg( + long, + default_value_t = crate::commands::audit::collector::GENERATE_SETTLE_QUIET_MS, + hide = true, + requires = "legacy_url" + )] + pub(crate) settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg( + long, + default_value_t = crate::commands::audit::collector::GENERATE_SETTLE_MAX_MS, + hide = true, + requires = "legacy_url" + )] + pub(crate) settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) danger_accept_invalid_certs: bool, +} + +impl Default for LegacyBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: crate::commands::audit::collector::GENERATE_SETTLE_QUIET_MS, + settle_max_ms: crate::commands::audit::collector::GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, } } - - Ok(AuditOutputPlan { - js_assets_path, - config_path, - }) -} - -fn resolve_output_path(path: Option<&Path>, default: &str) -> CliResult { - let candidate = path.unwrap_or_else(|| Path::new(default)); - if candidate.is_absolute() { - Ok(candidate.to_path_buf()) - } else { - Ok(std::env::current_dir() - .map_err(|error| report_error(format!("failed to read current directory: {error}")))? - .join(candidate)) - } } -fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult { - let artifact = analyze_collected_page(collected)?; - let final_url = collected - .final_url() - .map_err(|error| report_error(format!("invalid final URL: {error}")))?; - let js_assets_toml = toml::to_string_pretty(&artifact) - .map_err(|error| report_error(format!("failed to serialize audit artifact: {error}")))?; - let mut path_generator = RandomOpaqueAssetPathGenerator; - let draft_config = - build_draft_config_with_generator(&final_url, &artifact, &mut path_generator)?; - - Ok(AuditOutputs { - artifact, - js_assets_toml, - draft_config_toml: draft_config.toml, - js_asset_proxy_candidate_count: draft_config.js_asset_proxy_candidate_count, - }) -} - -fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliResult> { - let selected_paths = [&plan.js_assets_path, &plan.config_path] - .into_iter() - .flatten() - .collect::>(); - for path in &selected_paths { - if let Some(parent) = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - { - fs::create_dir_all(parent).map_err(|error| { - report_error(format!( - "failed to create parent directory {}: {error}", - parent.display() - )) - })?; +impl From<&LegacyBrowserOpts> for GenerateBrowserOpts { + fn from(options: &LegacyBrowserOpts) -> Self { + Self { + chrome: options.chrome.clone(), + headful: options.headful, + no_assume_consent: options.no_assume_consent, + browser_proxy: options.browser_proxy.clone(), + settle_quiet_ms: options.settle_quiet_ms, + settle_max_ms: options.settle_max_ms, + danger_accept_invalid_certs: options.danger_accept_invalid_certs, } } - - let mut written_paths = Vec::new(); - if let Some(path) = &plan.js_assets_path { - fs::write(path, &outputs.js_assets_toml).map_err(|error| { - report_error(format!( - "failed to write JS asset audit {}: {error}", - path.display() - )) - })?; - written_paths.push(path.display().to_string()); - } - if let Some(path) = &plan.config_path { - fs::write(path, &outputs.draft_config_toml).map_err(|error| { - report_error(format!( - "failed to write draft config {}: {error}", - path.display() - )) - })?; - written_paths.push(path.display().to_string()); - } - - Ok(written_paths) -} - -fn write_success_summary( - outputs: &AuditOutputs, - written: &[String], - wrote_config: bool, - out: &mut dyn Write, -) -> CliResult<()> { - let integrations = outputs - .artifact - .detected_integrations - .iter() - .map(|integration| integration.id.as_str()) - .collect::>(); - let draft_note = if wrote_config { - "\nDraft config: review before validation and push" - } else { - "" - }; - let asset_proxy_note = if wrote_config && outputs.js_asset_proxy_candidate_count > 0 { - format!( - "{} disabled entries written to draft config", - outputs.js_asset_proxy_candidate_count - ) - } else if wrote_config { - "none".to_string() - } else { - "not written (--no-config)".to_string() - }; - writeln!( - out, - "Audited {}\nTitle: {}\nJS assets: {}\nThird-party assets: {}\nDetected integrations: {}\nJS asset proxy candidates: {}\nWrote: {}{}", - outputs.artifact.audited_url, - outputs - .artifact - .page_title - .as_deref() - .unwrap_or(""), - outputs.artifact.js_asset_count, - outputs.artifact.third_party_asset_count, - if integrations.is_empty() { - "none".to_string() - } else { - integrations.join(", ") - }, - asset_proxy_note, - if written.is_empty() { - "none".to_string() - } else { - written.join(", ") - }, - draft_note - ) - .map_err(|error| report_error(format!("failed to write command output: {error}"))) } -fn build_draft_config_with_generator( - target_url: &Url, - artifact: &AuditArtifact, - path_generator: &mut dyn OpaqueAssetPathGenerator, -) -> CliResult { - let host = target_url - .host_str() - .ok_or_else(|| report_error("audited URL is missing a host"))?; - let origin = target_url.origin().ascii_serialization(); - let mut draft = EXAMPLE_CONFIG.to_string(); - - draft = replace_key_in_section( - &draft, - "publisher", - "domain", - &format!("domain = \"{host}\""), - )?; - draft = replace_key_in_section( - &draft, - "publisher", - "cookie_domain", - &format!("cookie_domain = \".{host}\""), - )?; - draft = replace_key_in_section( - &draft, - "publisher", - "origin_url", - &format!("origin_url = \"{origin}\""), - )?; - - let detected = artifact - .detected_integrations - .iter() - .map(|integration| integration.id.as_str()) - .collect::>(); - - if detected.contains("gpt") { - draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; - } - if detected.contains("didomi") { - draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; - } - if detected.contains("datadome") { - draft = - replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; - } - - let asset_proxy_section = build_js_asset_proxy_section(artifact, path_generator)?; - draft = replace_js_asset_proxy_section(&draft, &asset_proxy_section.toml)?; - - let mut manual_review = Vec::new(); - if detected.contains("google_tag_manager") { - if let Some(gtm_id) = extract_gtm_container_id(artifact) { - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "enabled", - "enabled = true", - )?; - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "container_id", - &format!("container_id = \"{gtm_id}\""), - )?; - } else { - manual_review.push("google_tag_manager"); - } - } - - for integration in detected { - if !matches!( - integration, - "gpt" | "didomi" | "datadome" | "google_tag_manager" - ) { - manual_review.push(integration); - } - } - - if !manual_review.is_empty() { - if !draft.ends_with('\n') { - draft.push('\n'); - } - draft.push_str("\n# Audit findings requiring manual review\n"); - for integration in manual_review { - draft.push_str(&format!( - "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" - )); +/// `ts audit` subcommands. +#[derive(Debug, Subcommand)] +pub(crate) enum AuditSubcommand { + /// Audit a single page and print a read-only summary. + Page(PageAuditArgs), + /// Verify configured ad-template slots against live page evidence. + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), + /// Bootstrap a draft Trusted Server config + JS asset audit from a live page. + Generate(generate::GenerateArgs), +} + +/// `ts audit ad-templates` subcommands. +#[derive(Debug, Subcommand)] +pub(crate) enum AuditAdTemplatesCommand { + /// Scrape a live page's GPT slots and update the config's + /// `[creative_opportunities]` slots in place. + Generate(AuditAdTemplatesGenerateArgs), + /// Verify ad-template slots for one or more live URLs. + Verify(AuditAdTemplatesVerifyArgs), +} + +/// Arguments for `ts audit ad-templates generate `. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesGenerateArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page URL to scrape for GPT slots (http or https). + #[arg(value_parser = parse_http_url)] + pub url: url::Url, + /// Glob applied to every slot discovered this run (e.g. `/`, `/news/*`). + /// Repeatable. Defaults to the scraped URL's path. Re-running with a + /// different pattern unions it into slots already in the config. + #[arg(long = "page-pattern", value_name = "GLOB")] + pub page_patterns: Vec, + /// Replace all existing slots instead of merging this run into them. + #[arg(long)] + pub replace: bool, + /// Preview the updated config on stdout instead of writing it. + #[arg(long)] + pub dry_run: bool, + /// Perform a deterministic scroll pass after each page initially settles. + #[arg(long)] + pub scroll: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, + /// Maximum site sections to sample. Each contributes a landing page and an + /// article, so this bounds how much of the publisher's taxonomy is covered. + #[arg(long, default_value_t = 8)] + pub max_sections: usize, + /// Maximum pages to load in total, including the requested page. + /// + /// Set to 1 to restore single-page behavior: no crawl, no section + /// discovery, and the audited path as the only page pattern. + #[arg(long, default_value_t = 17)] + pub max_pages: usize, + /// Device profiles to audit, comma-separated: `desktop`, `mobile`. + /// + /// Defaults to `desktop`. Publishers often serve different GAM ad units per + /// device, which a single-profile crawl cannot see — it would infer a + /// template correct for the profile it used and silently wrong elsewhere. + /// Passing both crawls each page twice and refuses to write an ad-unit path + /// for any slot where the profiles disagree. + #[arg(long, value_delimiter = ',', default_value = "desktop")] + pub profiles: Vec, + /// Pause in milliseconds between page loads during the crawl. + /// + /// A crawl issues a dozen navigations in a row. Firing them back to back is + /// discourteous to the origin, and request pacing is one of the signals bot + /// protection scores, so an unpaced crawl can trigger the challenge that + /// empties the rest of the run. + #[arg(long, default_value_t = 750)] + pub page_delay_ms: u64, + /// Browser and consent options shared with `ts audit generate`. + #[command(flatten)] + pub browser: GenerateBrowserOpts, +} + +impl AuditAdTemplatesGenerateArgs { + /// The crawl bounds these arguments describe. + pub(crate) fn budget(&self) -> generate::CrawlBudget { + generate::CrawlBudget { + max_sections: self.max_sections, + max_pages: self.max_pages, } } - Ok(DraftConfig { - toml: draft, - js_asset_proxy_candidate_count: asset_proxy_section.candidate_count, - }) -} - -fn build_js_asset_proxy_section( - artifact: &AuditArtifact, - path_generator: &mut dyn OpaqueAssetPathGenerator, -) -> CliResult { - let (candidates, skipped) = select_js_asset_proxy_candidates(artifact); - let mut used_paths = BTreeSet::new(); - let mut toml = String::new(); - - toml.push_str("[integrations.js_asset_proxy]\n"); - toml.push_str("enabled = false\n"); - toml.push_str("# Uncomment to override upstream cache headers for every asset below.\n"); - toml.push_str("# This replaces upstream directives, including private and no-store.\n"); - toml.push_str("# Use only when each asset's bytes are identical for every visitor.\n"); - toml.push_str("# cache_ttl_seconds = 3600\n"); - toml.push_str( - "# Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets\n", - ); - toml.push_str( - "# that vary by browser User-Agent or use integrity hashes for UA-specific bytes.\n\n", - ); - toml.push_str("# Generated by `ts audit`; review before enabling.\n"); - toml.push_str( - "# Audit note: some discovered scripts may be runtime-injected and may not appear\n", - ); - toml.push_str( - "# in origin HTML. JS Asset Proxy rewrites only matching script src URLs present in\n", - ); - toml.push_str("# HTML processed by Trusted Server.\n"); - - if candidates.is_empty() { - toml.push_str( - "# No eligible third-party HTTPS script assets were detected by `ts audit`.\n", - ); - } - - for candidate in &candidates { - let generated_path = generate_unique_asset_path(path_generator, &mut used_paths)?; - toml.push('\n'); - toml.push_str("# Generated by `ts audit`; review before enabling.\n"); - if let Some(integration) = candidate.integration { - let integration = sanitized_comment_value(integration); - toml.push_str(&format!("# Detected integration: {integration}\n")); - toml.push_str(&format!( - "# Native integration may be preferable: [integrations.{integration}]\n" - )); + /// The device profiles to audit, deduplicated in the order given. + /// + /// # Errors + /// + /// Returns an error when a name is not a known profile, or when none were + /// given. + pub(crate) fn profiles(&self) -> Result, String> { + let mut profiles: Vec = Vec::new(); + for raw in &self.profiles { + let profile = generate::DeviceProfile::parse(raw)?; + if !profiles.contains(&profile) { + profiles.push(profile); + } } - toml.push_str("[[integrations.js_asset_proxy.assets]]\n"); - toml.push_str(&format!("path = {}\n", toml_quoted_string(&generated_path))); - toml.push_str(&format!( - "origin_url = {}\n", - toml_quoted_string(&candidate.origin_url) - )); - if Url::parse(&candidate.origin_url).is_ok_and(|url| url.query().is_some()) { - toml.push_str( - "# This URL includes a query string and must remain stable for proxy matching.\n", - ); + if profiles.is_empty() { + return Err("--profiles needs at least one of: desktop, mobile".to_string()); } - toml.push_str("proxy = \"disabled\"\n"); + Ok(profiles) } - - append_js_asset_proxy_skip_comments(&mut toml, &skipped); - toml.push('\n'); - - Ok(JsAssetProxySection { - toml, - candidate_count: candidates.len(), - }) } -fn select_js_asset_proxy_candidates( - artifact: &AuditArtifact, -) -> (Vec>, JsAssetProxySkipCounts) { - let mut candidates = Vec::new(); - let mut skipped = JsAssetProxySkipCounts::default(); - let mut seen_origin_urls = BTreeSet::new(); - - for asset in &artifact.assets { - if asset.kind != "script" { - skipped.non_script += 1; - continue; - } - if asset.party != AssetParty::ThirdParty { - skipped.first_party += 1; - continue; - } - - let Ok(url) = Url::parse(&asset.url) else { - skipped.malformed_url += 1; - continue; - }; - if url.host_str().is_none() { - skipped.malformed_url += 1; - continue; - } - if url.scheme() != "https" { - skipped.non_https += 1; - continue; - } - - let origin_url = url.to_string(); - if !seen_origin_urls.insert(origin_url.clone()) { - skipped.duplicate_url += 1; - continue; - } - - candidates.push(JsAssetProxyCandidate { - origin_url, - integration: asset.integration.as_deref(), - }); - } - - (candidates, skipped) -} - -fn generate_unique_asset_path( - path_generator: &mut dyn OpaqueAssetPathGenerator, - used_paths: &mut BTreeSet, -) -> CliResult { - for _ in 0..128 { - let path = path_generator.next_path(); - if !is_valid_generated_asset_path(&path) { - return cli_error(format!( - "generated JS asset proxy path `{path}` is invalid; expected /assets/.js" - )); - } - if used_paths.insert(path.clone()) { - return Ok(path); +/// Arguments for `ts audit ad-templates verify ...`. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesVerifyArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// One or more page URLs to verify (http or https). + #[arg(required = true, value_parser = parse_http_url)] + pub urls: Vec, + /// Exit non-zero when a matched slot is missing or only partially confirmed. + #[arg(long)] + pub strict: bool, + /// Emit machine-readable JSON instead of human output. + #[arg(long)] + pub json: bool, + /// Perform a deterministic scroll pass after the initial settle. + #[arg(long)] + pub scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + /// + /// Off by default: slots are matched on the post-redirect path, so an + /// off-origin page could otherwise satisfy `--strict`. Enable only for a + /// known redirect between your own properties (e.g. apex to `www`). + #[arg(long)] + pub allow_cross_origin_redirect: bool, + /// Cookie to send with each page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, + #[command(flatten)] + pub browser: BrowserOpts, +} + +/// Dispatches a `ts audit` invocation. +/// +/// `legacy_url` (if present) routes to artifact generation, while the `page` +/// subcommand routes to the generic read-only page audit. +/// +/// # Errors +/// +/// Returns a user-facing string when no URL or subcommand is provided, or when +/// the underlying command fails. +pub(crate) fn run_audit(args: &AuditArgs) -> Result { + match &args.command { + Some(AuditSubcommand::Page(page_args)) => { + page::run_page(page_args).map(|()| RunOutcome::Success) } - } - - cli_error("failed to generate a unique JS asset proxy path after 128 attempts") -} - -fn is_valid_generated_asset_path(path: &str) -> bool { - let Some(opaque_id) = path - .strip_prefix("/assets/") - .and_then(|value| value.strip_suffix(".js")) - else { - return false; - }; - - !opaque_id.is_empty() - && opaque_id - .chars() - .all(|ch| ch.is_ascii_hexdigit() && !ch.is_ascii_uppercase()) -} - -fn replace_js_asset_proxy_section(document: &str, replacement: &str) -> CliResult { - let lines = document.lines().collect::>(); - let start = lines - .iter() - .position(|line| line.trim() == "[integrations.js_asset_proxy]") - .ok_or_else(|| { - report_error( - "failed to update starter config because section `[integrations.js_asset_proxy]` was not found", + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { + gen_args.browser.validate()?; + let app_config_path = crate::app_config::resolve_app_config_file(&gen_args.config)?; + let raw_config = std::fs::read_to_string(&app_config_path).map_err(|error| { + format!("failed to read {}: {error}", app_config_path.display()) + })?; + let existing_creative = creative_config(&raw_config, &app_config_path)?; + let profiles = gen_args.profiles()?; + let collectors: Vec = profiles + .iter() + .map(|profile| { + generate::browser_collector::BrowserAuditCollector::with_profile(*profile) + .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) + .with_browser_options(&gen_args.browser) + .with_scroll(gen_args.scroll) + }) + .collect(); + let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles + .iter() + .zip(collectors.iter()) + .map(|(profile, collector)| { + ( + profile.label(), + collector as &dyn generate::collector::AuditCollector, + ) + }) + .collect(); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let stderr = std::io::stderr(); + let mut err = stderr.lock(); + generate::run_update_slots( + &generate::UpdateSlotsRequest { + url: gen_args.url.as_str(), + config_path: &app_config_path, + existing_creative: existing_creative.as_ref(), + page_patterns: &gen_args.page_patterns, + replace: gen_args.replace, + cookies: &gen_args.cookies, + dry_run: gen_args.dry_run, + scroll: gen_args.scroll, + budget: gen_args.budget(), + }, + &selected, + &mut out, + &mut err, ) - })?; - let mut end = start + 1; - - while end < lines.len() { - let trimmed = lines[end].trim(); - if trimmed.starts_with('[') - && trimmed.ends_with(']') - && trimmed != "[[integrations.js_asset_proxy.assets]]" - { - break; + .map(|()| RunOutcome::Success) } - end += 1; - } - - // Blank lines and comments directly above the next section header document - // that section, not this one, so leave them in the draft. - while end > start + 1 { - let trimmed = lines[end - 1].trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - end -= 1; - } else { - break; + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Verify(verify_args))) => { + ad_templates::run_verify(verify_args) } - } - - let mut output_lines = Vec::new(); - output_lines.extend_from_slice(&lines[..start]); - output_lines.extend(replacement.trim_end_matches('\n').lines()); - if end < lines.len() && !lines[end].trim().is_empty() { - output_lines.push(""); - } - output_lines.extend_from_slice(&lines[end..]); - - let mut output = output_lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); - } - Ok(output) -} - -fn append_js_asset_proxy_skip_comments(toml: &mut String, skipped: &JsAssetProxySkipCounts) { - if skipped.first_party == 0 - && skipped.malformed_url == 0 - && skipped.non_https == 0 - && skipped.duplicate_url == 0 - && skipped.non_script == 0 - { - return; - } - - toml.push('\n'); - toml.push_str("# Skipped JS Asset Proxy audit candidates:\n"); - append_skip_count(toml, skipped.first_party, "first-party script"); - append_skip_count(toml, skipped.malformed_url, "malformed script URL"); - append_skip_count(toml, skipped.non_https, "non-HTTPS third-party script"); - append_skip_count(toml, skipped.duplicate_url, "duplicate script URL"); - append_skip_count(toml, skipped.non_script, "non-script asset"); -} - -fn append_skip_count(toml: &mut String, count: usize, label: &str) { - if count == 0 { - return; - } - - let plural = if count == 1 { "" } else { "s" }; - toml.push_str(&format!("# - {count} {label}{plural}\n")); -} - -fn sanitized_comment_value(value: &str) -> String { - value - .chars() - .map(|ch| if ch.is_control() { ' ' } else { ch }) - .collect() -} - -fn toml_quoted_string(value: &str) -> String { - let mut quoted = String::from("\""); - for ch in value.chars() { - match ch { - '\\' => quoted.push_str("\\\\"), - '"' => quoted.push_str("\\\""), - '\n' => quoted.push_str("\\n"), - '\r' => quoted.push_str("\\r"), - '\t' => quoted.push_str("\\t"), - ch if ch.is_control() => { - write!(&mut quoted, "\\u{:04X}", ch as u32).expect("should write to string"); - } - ch => quoted.push(ch), + Some(AuditSubcommand::Generate(generate_args)) => { + generate_args.browser.validate()?; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); + generate::run_generate(generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) } + None => match args.legacy_url.as_ref() { + Some(url) => { + let generate_args = legacy_generate_args(args, url); + generate_args.browser.validate()?; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); + generate::run_generate(&generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) + } + None => Err( + "provide a URL or a subcommand (`generate`, `page`, `ad-templates`)".to_string(), + ), + }, } - quoted.push('"'); - quoted -} - -fn lowercase_hex(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut encoded = String::with_capacity(bytes.len() * 2); - for byte in bytes { - encoded.push(HEX[(byte >> 4) as usize] as char); - encoded.push(HEX[(byte & 0x0f) as usize] as char); - } - encoded } -fn replace_key_in_section( +/// Reads the config's `[creative_opportunities]` section, when it has one. +/// +/// An unrelated invalid setting elsewhere in the document must not hide the +/// section — the runtime rejects such a file, but the operator still has to be +/// able to update slots in it — so the document is read as plain TOML rather +/// than through [`Settings`](trusted_server_core::settings::Settings). +/// +/// A section that is present but unreadable is *not* treated as absent. +/// `CreativeOpportunitiesConfig` uses `deny_unknown_fields`, so one mistyped key +/// would otherwise leave the merge with nothing to merge into and replace the +/// operator's entire slot array. +/// +/// # Errors +/// +/// Returns a user-facing error when the document is malformed or the section is +/// present but cannot be deserialized. +fn creative_config( document: &str, - section: &str, - key: &str, - replacement_line: &str, -) -> CliResult { - let section_header = format!("[{section}]"); - let mut in_section = false; - let mut replaced = false; - let mut saw_section = false; - let mut lines = Vec::new(); - - for line in document.lines() { - let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_section = trimmed == section_header; - saw_section |= in_section; - } - - if in_section && !replaced && is_key_line(trimmed, key) { - lines.push(replacement_line.to_string()); - replaced = true; - } else { - lines.push(line.to_string()); - } - } - - if !saw_section { - return cli_error(format!( - "failed to update starter config because section `{section_header}` was not found" - )); - } - if !replaced { - return cli_error(format!( - "failed to update starter config because key `{key}` was not found in `{section_header}`" - )); - } - - let mut output = lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); + path: &std::path::Path, +) -> CliResult> { + // Plain `format!`, not `report_error`: the top-level `[ts]` printer already + // logs whatever is returned here, and this message embeds a multi-line + // `toml::de::Error`, so logging it here too would print the whole block + // twice. The guidance leads so the parse error can trail unbroken. + let value = toml::from_str::(document).map_err(|error| { + format!( + "failed to parse {} before generating slots; fix the TOML syntax and re-run:\n{error}", + path.display() + ) + })?; + let Some(section) = value.get("creative_opportunities").cloned() else { + return Ok(None); + }; + match section.try_into() { + Ok(config) => Ok(Some(config)), + Err(error) => cli_error(format!( + "failed to read the existing `[creative_opportunities]` section, so generating \ + slots would discard the configured ones: {error}. Fix the section (or delete it) \ + and re-run" + )), } - Ok(output) } -fn is_key_line(trimmed_line: &str, key: &str) -> bool { - trimmed_line - .strip_prefix(key) - .and_then(|remaining| remaining.trim_start().strip_prefix('=')) - .is_some() +fn legacy_generate_args(args: &AuditArgs, url: &url::Url) -> generate::GenerateArgs { + generate::GenerateArgs { + url: url.to_string(), + js_assets: args.legacy_generate.js_assets.clone(), + config: args.legacy_generate.config.clone(), + no_js_assets: args.legacy_generate.no_js_assets, + no_config: args.legacy_generate.no_config, + force: args.legacy_generate.force, + cookies: args.legacy_generate.cookies.clone(), + browser: GenerateBrowserOpts::from(&args.legacy_generate.browser), + } } #[cfg(test)] mod tests { - use std::cell::Cell; - use std::collections::VecDeque; - - use tempfile::TempDir; - use super::*; - use crate::commands::audit::collector::{CollectedPage, CollectedRequest, CollectedScriptTag}; - - struct FakeCollector { - collected: CollectedPage, - calls: Cell, - } - - struct FixedPathGenerator { - paths: VecDeque, - } - - impl FixedPathGenerator { - fn new(paths: &[&str]) -> Self { - Self { - paths: paths.iter().map(|path| (*path).to_string()).collect(), - } - } - } - - impl OpaqueAssetPathGenerator for FixedPathGenerator { - fn next_path(&mut self) -> String { - self.paths - .pop_front() - .expect("should have a fixed generated asset path") - } - } - - impl FakeCollector { - fn new(collected: CollectedPage) -> Self { - Self { - collected, - calls: Cell::new(0), - } - } - } - - impl AuditCollector for FakeCollector { - fn collect_page(&self, _target_url: &Url) -> CliResult { - self.calls.set(self.calls.get() + 1); - Ok(self.collected.clone()) - } - } - - fn collected_page() -> CollectedPage { - CollectedPage { - requested_url: "https://publisher.example/page".to_string(), - final_url: "https://publisher.example/page".to_string(), - page_title: Some("Example Publisher".to_string()), - html: r#"Example Publisher"#.to_string(), - script_tags: vec![ - CollectedScriptTag { - src: Some("https://www.googletagmanager.com/gtm.js?id=GTM-ABC123".to_string()), - inline_text: None, - }, - CollectedScriptTag { - src: Some("https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string()), - inline_text: None, - }, - ], - network_requests: vec![CollectedRequest { - url: "https://cdn.publisher.example/app.js".to_string(), - resource_type: Some("script".to_string()), - }], - warnings: Vec::new(), - } - } - - fn audit_args(url: &str) -> AuditArgs { - AuditArgs { - url: url.to_string(), - js_assets: None, - config: None, - no_js_assets: false, - no_config: false, - force: false, - } - } - - fn audited_asset(url: &str, party: AssetParty, integration: Option<&str>) -> AuditedAsset { - AuditedAsset { - kind: "script".to_string(), - url: url.to_string(), - host: Url::parse(url) - .ok() - .and_then(|parsed| parsed.host_str().map(str::to_string)) - .unwrap_or_default(), - party, - integration: integration.map(str::to_string), - } - } - - #[test] - fn parse_audit_url_accepts_http_and_https() { - assert!(parse_audit_url("http://publisher.example").is_ok()); - assert!(parse_audit_url("https://publisher.example").is_ok()); - } #[test] - fn parse_audit_url_rejects_non_http_schemes() { - for url in [ - "file:///etc/passwd", - "data:text/html,hello", - "chrome://version", - ] { - let error = parse_audit_url(url).expect_err("should reject non-http URL"); - assert!( - format!("{error:?}").contains("only supports http/https"), - "should explain scheme restriction" - ); - } - } - - #[test] - fn resolve_output_plan_rejects_no_outputs() { - let mut args = audit_args("https://publisher.example"); - args.no_js_assets = true; - args.no_config = true; - - let error = resolve_output_plan(&args).expect_err("should reject empty output set"); - - assert!( - format!("{error:?}").contains("nothing to do"), - "should explain no-output error" + fn parse_cookie_splits_on_first_equals() { + let (name, value) = parse_cookie("datadome=abc=def~ghi").expect("should parse cookie"); + assert_eq!(name, "datadome", "name should be the pre-`=` portion"); + assert_eq!( + value, "abc=def~ghi", + "value should keep later `=` characters" ); } #[test] - fn resolve_output_plan_rejects_existing_files_without_force() { - let temp = TempDir::new().expect("should create temp dir"); - let path = temp.path().join("js-assets.toml"); - fs::write(&path, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example"); - args.js_assets = Some(path); - args.no_config = true; - - let error = resolve_output_plan(&args).expect_err("should reject overwrite"); - - assert!( - format!("{error:?}").contains("refusing to overwrite"), - "should explain overwrite refusal" - ); + fn parse_cookie_allows_empty_value() { + let (name, value) = parse_cookie("session=").expect("should parse empty value"); + assert_eq!(name, "session"); + assert!(value.is_empty(), "empty value should be allowed"); } #[test] - fn resolve_output_plan_allows_existing_files_with_force() { - let temp = TempDir::new().expect("should create temp dir"); - let path = temp.path().join("js-assets.toml"); - fs::write(&path, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example"); - args.js_assets = Some(path.clone()); - args.no_config = true; - args.force = true; + fn invalid_setting_outside_the_section_still_yields_creative_config() { + let document = "unknown_runtime_key = true\n\ + [creative_opportunities]\ngam_network_id = \"123\"\n"; - let plan = resolve_output_plan(&args).expect("should allow forced overwrite"); + let creative = creative_config(document, std::path::Path::new("trusted-server.toml")) + .expect("an unrelated invalid setting must not hide creative config") + .expect("the section is present"); - assert_eq!(plan.js_assets_path.as_deref(), Some(path.as_path())); + assert_eq!(creative.gam_network_id, "123"); } #[test] - fn run_audit_writes_selected_outputs_and_summary() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("audit/js-assets.toml"); - let config = temp.path().join("audit/trusted-server.toml"); - let args = AuditArgs { - url: "https://publisher.example/page".to_string(), - js_assets: Some(js_assets.clone()), - config: Some(config.clone()), - no_js_assets: false, - no_config: false, - force: false, - }; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); - - assert_eq!(collector.calls.get(), 1, "should collect page once"); - assert!(js_assets.exists(), "should write JS assets"); - assert!(config.exists(), "should write draft config"); - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("Audited https://publisher.example/page")); - assert!(summary.contains("Detected integrations: google_tag_manager, gpt")); - assert!(summary.contains("Draft config: review before validation and push")); - } - - #[test] - fn run_audit_respects_no_config() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets.clone()); - args.no_config = true; - let collector = FakeCollector::new(collected_page()); - - run_audit(&args, &collector, &mut Vec::new()).expect("should run audit"); + fn absent_section_reads_as_absent() { + let creative = creative_config( + "[auction]\nenabled = true\n", + std::path::Path::new("trusted-server.toml"), + ) + .expect("should read the document"); - assert!(js_assets.exists(), "should write assets"); assert!( - !temp.path().join("trusted-server.toml").exists(), - "should not write config" + creative.is_none(), + "a document with no `[creative_opportunities]` has no configured slots" ); } #[test] - fn run_audit_respects_no_js_assets() { - let temp = TempDir::new().expect("should create temp dir"); - let config = temp.path().join("trusted-server.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.config = Some(config.clone()); - args.no_js_assets = true; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); + fn malformed_document_is_rejected_before_creative_config_extraction() { + let error = creative_config( + "[creative_opportunities\ngam_network_id = \"123\"\n", + std::path::Path::new("/tmp/example/trusted-server.toml"), + ) + .expect_err("should reject malformed TOML"); - assert!(config.exists(), "should write config"); assert!( - !temp.path().join("js-assets.toml").exists(), - "should not write JS assets" + error.contains("failed to parse /tmp/example/trusted-server.toml"), + "error should name the config file it could not parse, got {error}" ); - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("Draft config: review before validation and push")); - } - - #[test] - fn run_audit_writes_collector_warnings_to_asset_artifact() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets.clone()); - args.no_config = true; - let mut collected = collected_page(); - collected.warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - let collector = FakeCollector::new(collected); - - run_audit(&args, &collector, &mut Vec::new()).expect("should run audit"); - - let artifact = fs::read_to_string(js_assets).expect("should read artifact"); assert!( - artifact.contains("results may be partial"), - "should persist collector warning" + error.contains("fix the TOML syntax and re-run:\n"), + "the guidance should lead so the multi-line parse error trails it, got {error}" ); } #[test] - fn run_audit_conflict_prevents_collection() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - fs::write(&js_assets, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets); - args.no_config = true; - let collector = FakeCollector::new(collected_page()); - - let error = run_audit(&args, &collector, &mut Vec::new()) - .expect_err("should reject existing output"); + fn unreadable_section_is_refused_rather_than_read_as_absent() { + // `deny_unknown_fields` makes one mistyped key inside the section fail + // to deserialize. Reading that as "no slots configured" would let a + // merge replace the operator's entire slot array. + let document = "[creative_opportunities]\n\ + gam_network_id = \"123\"\n\ + gam_netwrok_id = \"123\"\n\ + [[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"ad-header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + + let error = creative_config(document, std::path::Path::new("trusted-server.toml")) + .expect_err("should refuse an unreadable section"); - assert_eq!(collector.calls.get(), 0, "should not collect page"); assert!( - format!("{error:?}").contains("refusing to overwrite"), - "should report overwrite conflict" + error.contains("would discard the configured ones"), + "error should say what merging would cost, got {error}" ); } #[test] - fn build_draft_config_writes_disabled_js_asset_proxy_candidates() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: Some("Example".to_string()), - js_asset_count: 2, - third_party_asset_count: 2, - detected_integrations: vec![DetectedIntegration { - id: "gpt".to_string(), - evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), - }], - assets: vec![ - audited_asset( - "https://cdn.vendor.example/sdk.js", - AssetParty::ThirdParty, - None, - ), - audited_asset( - "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - AssetParty::ThirdParty, - Some("gpt"), - ), - ], - warnings: Vec::new(), - }; - let mut generator = FixedPathGenerator::new(&[ - "/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js", - "/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js", - ]); - - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - - assert_eq!( - draft.js_asset_proxy_candidate_count, 2, - "should report generated disabled entries" - ); - assert!( - draft - .toml - .contains("[integrations.js_asset_proxy]\nenabled = false") - ); - assert!(draft.toml.contains("/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js")); - assert!(draft.toml.contains("/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js")); + fn parse_cookie_rejects_missing_equals() { + let err = parse_cookie("datadome").expect_err("should reject missing `=`"); assert!( - draft - .toml - .contains("origin_url = \"https://cdn.vendor.example/sdk.js\"") - ); - assert!(draft.toml.contains("proxy = \"disabled\"")); - assert!(draft.toml.contains("Detected integration: gpt")); - assert!( - draft - .toml - .contains("Native integration may be preferable: [integrations.gpt]") - ); - assert!( - !draft.toml.contains("example-vendor-loader"), - "should remove starter-template placeholder asset" - ); - assert!( - draft.toml.contains( - "# Proxy behavior and first-party asset routing. Kept active with defaults.\n[proxy]" - ), - "should preserve documentation for the section following the replaced block" - ); - let parsed = - toml::from_str::(&draft.toml).expect("draft should parse as TOML"); - assert!( - parsed["integrations"]["js_asset_proxy"] - .get("cache_ttl_seconds") - .is_none(), - "generated config should inherit upstream cache headers by default" + err.contains("NAME=VALUE"), + "error should show expected form" ); } #[test] - fn generated_asset_proxy_paths_are_opaque() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 1, - third_party_asset_count: 1, - detected_integrations: Vec::new(), - assets: vec![audited_asset( - "https://cdn.vendor.example/vendor-loader.js", - AssetParty::ThirdParty, - None, - )], - warnings: Vec::new(), - }; - let mut generator = FixedPathGenerator::new(&["/assets/0123456789abcdef01234567.js"]); - - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - let path_line = draft - .toml - .lines() - .find(|line| line.starts_with("path = ") && line.contains("0123456789abcdef")) - .expect("should include generated path"); - - assert!(path_line.contains("/assets/0123456789abcdef01234567.js")); - assert!( - !path_line.contains("vendor") - && !path_line.contains("cdn") - && !path_line.contains("loader"), - "generated path should not include vendor, domain, or filename semantics" - ); + fn parse_cookie_rejects_empty_name() { + let err = parse_cookie("=value").expect_err("should reject empty name"); + assert!(err.contains("empty name"), "error should name the problem"); } #[test] - fn asset_proxy_generation_deduplicates_and_summarizes_skips() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 4, - third_party_asset_count: 3, - detected_integrations: Vec::new(), - assets: vec![ - audited_asset( - "https://cdn.vendor.example/sdk.js", - AssetParty::ThirdParty, - None, - ), - audited_asset( - "https://cdn.vendor.example/sdk.js", - AssetParty::ThirdParty, - None, - ), - audited_asset( - "https://publisher.example/app.js", - AssetParty::FirstParty, - None, - ), - audited_asset( - "http://cdn.vendor.example/insecure.js", - AssetParty::ThirdParty, - None, - ), - ], - warnings: Vec::new(), + fn legacy_url_builds_artifact_generation_args() { + let args = AuditArgs { + command: None, + legacy_url: Some( + url::Url::parse("https://www.example.com/").expect("should parse URL"), + ), + legacy_generate: LegacyGenerateArgs { + js_assets: Some("audit/assets.toml".into()), + config: Some("audit/config.toml".into()), + no_js_assets: false, + no_config: false, + force: true, + cookies: vec![("session".to_string(), "example".to_string())], + browser: LegacyBrowserOpts { + headful: true, + ..LegacyBrowserOpts::default() + }, + }, }; - let mut generator = FixedPathGenerator::new(&["/assets/111111111111111111111111.js"]); - - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - assert_eq!(draft.js_asset_proxy_candidate_count, 1); - assert_eq!( - draft - .toml - .matches("[[integrations.js_asset_proxy.assets]]") - .count(), - 1, - "should only emit one candidate entry" + let generate = legacy_generate_args( + &args, + args.legacy_url.as_ref().expect("should have legacy URL"), ); - assert!(draft.toml.contains("# - 1 first-party script")); - assert!(draft.toml.contains("# - 1 non-HTTPS third-party script")); - assert!(draft.toml.contains("# - 1 duplicate script URL")); - } - - #[test] - fn asset_proxy_generation_warns_about_query_string_candidates() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 2, - third_party_asset_count: 2, - detected_integrations: Vec::new(), - assets: vec![ - audited_asset( - "https://cdn.vendor.example/sdk.js?v=one", - AssetParty::ThirdParty, - None, - ), - audited_asset( - "https://cdn.vendor.example/sdk.js?v=two", - AssetParty::ThirdParty, - None, - ), - ], - warnings: Vec::new(), - }; - let mut generator = FixedPathGenerator::new(&[ - "/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js", - "/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js", - ]); - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - - assert_eq!(draft.js_asset_proxy_candidate_count, 2); + assert_eq!(generate.url, "https://www.example.com/"); assert_eq!( - draft - .toml - .matches("This URL includes a query string and must remain stable") - .count(), - 2, - "each query-string candidate should explain exact-match behavior" + generate.js_assets.as_deref(), + Some(std::path::Path::new("audit/assets.toml")) ); - } - - #[test] - fn asset_proxy_generation_with_no_candidates_removes_placeholder_asset() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 1, - third_party_asset_count: 0, - detected_integrations: Vec::new(), - assets: vec![audited_asset( - "https://publisher.example/app.js", - AssetParty::FirstParty, - None, - )], - warnings: Vec::new(), - }; - let mut generator = FixedPathGenerator::new(&[]); - - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - - assert_eq!(draft.js_asset_proxy_candidate_count, 0); - assert!( - draft - .toml - .contains("No eligible third-party HTTPS script assets") + assert_eq!( + generate.config.as_deref(), + Some(std::path::Path::new("audit/config.toml")) ); - assert!( - !draft - .toml - .contains("[[integrations.js_asset_proxy.assets]]"), - "should not emit asset array entries without candidates" + assert!(generate.force); + assert_eq!( + generate.cookies, + [("session".to_string(), "example".to_string())] ); assert!( - !draft.toml.contains("example-vendor-loader"), - "should remove starter-template placeholder asset" + generate.browser.headful, + "browser flags passed to the legacy form should reach generation" ); - toml::from_str::(&draft.toml).expect("draft should parse as TOML"); - } - - #[test] - fn run_audit_summary_reports_written_asset_proxy_candidates() { - let temp = TempDir::new().expect("should create temp dir"); - let config = temp.path().join("trusted-server.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.config = Some(config); - args.no_js_assets = true; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); - - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("JS asset proxy candidates:")); - assert!(summary.contains("disabled entries written to draft config")); - } - - #[test] - fn build_draft_config_uses_final_url_and_detected_integrations() { - let url = Url::parse("https://www.publisher.example:8443/path").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: Some("Example".to_string()), - js_asset_count: 2, - third_party_asset_count: 2, - detected_integrations: vec![ - DetectedIntegration { - id: "google_tag_manager".to_string(), - evidence: "GTM-ABC123".to_string(), - }, - DetectedIntegration { - id: "gpt".to_string(), - evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), - }, - DetectedIntegration { - id: "prebid".to_string(), - evidence: "inline script matched `prebid`".to_string(), - }, - ], - assets: Vec::new(), - warnings: Vec::new(), - }; - - let mut generator = FixedPathGenerator::new(&[]); - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config") - .toml; - - assert!(draft.contains("domain = \"www.publisher.example\"")); - assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); - assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); - assert!(draft.contains("[integrations.gpt]\nenabled = true")); - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); - assert!(draft.contains("container_id = \"GTM-ABC123\"")); - assert!(draft.contains("Detected prebid")); - toml::from_str::(&draft).expect("draft should parse as TOML"); - } - - #[test] - fn build_draft_config_does_not_enable_gtm_without_container_id() { - let url = Url::parse("https://publisher.example/path").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 1, - third_party_asset_count: 1, - detected_integrations: vec![DetectedIntegration { - id: "google_tag_manager".to_string(), - evidence: "https://www.googletagmanager.com/gtm.js".to_string(), - }], - assets: Vec::new(), - warnings: Vec::new(), - }; - - let mut generator = FixedPathGenerator::new(&[]); - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config") - .toml; - - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); - assert!(draft.contains("Detected google_tag_manager")); } } diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs new file mode 100644 index 000000000..31cbf4b1b --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -0,0 +1,157 @@ +//! Generic `ts audit page ` command: a read-only page summary. + +use std::io::{self, Write}; + +use clap::Args; + +use crate::ad_templates::output::escape_terminal_text; +use crate::commands::audit::browser::BrowserCollector; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, CollectedPage, +}; + +/// Arguments for `ts audit page `. +#[derive(Debug, Args)] +pub(crate) struct PageAuditArgs { + /// The page URL to audit (http or https). + #[arg(value_parser = crate::commands::audit::parse_http_url)] + pub url: url::Url, + /// Perform a deterministic scroll pass after the initial settle. + #[arg(long)] + pub scroll: bool, + #[command(flatten)] + pub browser: BrowserOpts, +} + +/// Runs the generic page audit for the `page` subcommand. +/// +/// # Errors +/// +/// Returns a user-facing string when the browser cannot collect the page. +pub(crate) fn run_page(args: &PageAuditArgs) -> Result<(), String> { + args.browser.validate()?; + run_with_collector( + &BrowserCollector::from_opts(&args.browser), + &args.url, + args.scroll, + ) +} + +fn run_with_collector( + collector: &dyn AuditCollector, + url: &url::Url, + scroll: bool, +) -> Result<(), String> { + let page = collector.collect_page(BrowserCollectRequest { + url: url.clone(), + init_scripts: Vec::new(), + scroll, + collect_ad_evidence: false, + cookies: Vec::new(), + })?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + write_summary(&mut out, url, &page) +} + +fn write_summary(out: &mut dyn Write, url: &url::Url, page: &CollectedPage) -> Result<(), String> { + let to_err = |error: io::Error| format!("failed to write command output: {error}"); + writeln!(out, "url: {url}").map_err(to_err)?; + // The final URL, title, and collector warning messages are page-controlled, + // so escape control characters before they reach the operator's terminal. + writeln!( + out, + "final url: {}", + escape_terminal_text(page.final_url.as_str()) + ) + .map_err(to_err)?; + writeln!(out, "title: {}", escape_terminal_text(&page.title)).map_err(to_err)?; + writeln!(out, "scripts: {}", page.script_count).map_err(to_err)?; + writeln!(out, "resources: {}", page.resource_count).map_err(to_err)?; + for warning in &page.warnings { + writeln!( + out, + "warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(to_err)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::output::Warning; + + fn collected(final_url: &str, title: &str, warnings: Vec) -> CollectedPage { + CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse fixture URL"), + title: title.to_string(), + script_count: 3, + resource_count: 42, + warnings, + ad_evidence: None, + } + } + + fn summary(page: &CollectedPage, requested: &str) -> String { + let url = url::Url::parse(requested).expect("should parse requested URL"); + let mut out = Vec::new(); + write_summary(&mut out, &url, page).expect("should write summary"); + String::from_utf8(out).expect("summary should be UTF-8") + } + + #[test] + fn summary_reports_the_requested_and_final_urls_with_counts() { + let page = collected( + "https://publisher.example/news/story", + "Example Publisher", + Vec::new(), + ); + + let out = summary(&page, "https://publisher.example/news"); + + assert!( + out.contains("url: https://publisher.example/news\n"), + "should echo the requested URL, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/news/story\n"), + "should report the post-redirect URL, got {out:?}" + ); + assert!(out.contains("scripts: 3"), "got {out:?}"); + assert!(out.contains("resources: 42"), "got {out:?}"); + } + + #[test] + fn page_controlled_text_is_escaped_before_it_reaches_the_terminal() { + // Title and warning text are page-controlled and can contain raw + // terminal controls. URL percent-encoding is asserted separately. + let page = collected( + "https://publisher.example/a%1B%5B2Jb", + "Example\u{1b}[2J", + vec![Warning { + code: "page_\u{1b}[31m".to_string(), + message: "message\u{1b}[0m".to_string(), + }], + ); + + let out = summary(&page, "https://publisher.example/"); + + assert!( + !out.contains('\u{1b}'), + "no escape sequence may reach the terminal, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/a%1B%5B2Jb\n"), + "the final URL should retain URL's percent encoding, got {out:?}" + ); + assert!( + out.contains("warning [page_"), + "warnings should still be reported, got {out:?}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs new file mode 100644 index 000000000..481e07d5c --- /dev/null +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -0,0 +1,835 @@ +use std::collections::BTreeSet; +use std::io::{self, Write}; + +use crate::ad_templates::expected::normalize_path_or_url; +use crate::ad_templates::output::escape_terminal_text; +use crate::app_config::{AppConfigArgs, load_settings}; +use clap::{ArgGroup, Args, Subcommand}; +use http::Method; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, AdStackGateName, CreativeOpportunityFormat, CreativeOpportunitySlot, + RuntimeAdStackExpected, evaluate_ad_stack_gate, match_slots, validate_page_pattern, +}; + +use crate::run::RunOutcome; + +enum CheckFailure { + Tool(String), + Assertion(String), +} + +#[derive(Debug, Subcommand)] +pub enum AdTemplatesCommand { + /// Validate ad-template config and summarize deploy-time implications. + Lint(AdTemplatesLintArgs), + /// Show creative opportunity slots matching a page path or URL. + Match(AdTemplatesMatchArgs), + /// Assert that a page path or URL matches the expected slot set. + Check(AdTemplatesCheckArgs), + /// Explain why a page path or URL would or would not run the ad stack. + Explain(AdTemplatesExplainArgs), +} + +#[derive(Debug, Args)] +pub struct AdTemplatesLintArgs { + #[command(flatten)] + pub config: AppConfigArgs, +} + +#[derive(Debug, Args)] +pub struct AdTemplatesMatchArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// Include slot div, GAM path, formats, and providers. + #[arg(long)] + pub details: bool, +} + +#[derive(Debug, Args)] +#[command(group( + ArgGroup::new("expectation") + .required(true) + .args(["expected_slots", "expect_no_slots"]) +))] +pub struct AdTemplatesCheckArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// Expected slot id. Repeat for multiple slots. + #[arg(long = "expected-slot", value_name = "ID")] + pub expected_slots: Vec, + /// Assert that no slots match the page path or URL. + #[arg(long)] + pub expect_no_slots: bool, + /// Allow additional matched slots beyond --expected-slot values. + #[arg(long, conflicts_with = "expect_no_slots")] + pub allow_extra_slots: bool, +} + +#[derive(Debug, Args)] +pub struct AdTemplatesExplainArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// HTTP method to model. + #[arg(long, default_value = "GET", value_parser = parse_http_method)] + pub method: Method, + /// Model a non-navigation request. + #[arg(long)] + pub non_navigation: bool, + /// Model a prefetch request. + #[arg(long)] + pub prefetch: bool, + /// Model a known crawler user agent. + #[arg(long)] + pub bot: bool, + /// Model consent denying server-side auction. + #[arg(long)] + pub consent_denied: bool, +} + +fn parse_http_method(raw: &str) -> Result { + let normalized = raw.to_ascii_uppercase(); + Method::from_bytes(normalized.as_bytes()) + .map_err(|error| format!("invalid HTTP method `{raw}`: {error}")) +} + +/// Run an ad-template CLI command. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading, matching, or assertion +/// checks fail. +pub fn run_ad_templates(args: &AdTemplatesCommand) -> Result { + let stdout = io::stdout(); + let mut out = stdout.lock(); + if let AdTemplatesCommand::Check(args) = args { + return match run_check_classified(args, &mut out) { + Ok(()) => Ok(RunOutcome::Success), + Err(CheckFailure::Tool(error)) => Err(error), + Err(CheckFailure::Assertion(message)) => { + let stderr = io::stderr(); + let mut err = stderr.lock(); + writeln!(err, "{message}").map_err(output_error)?; + Ok(RunOutcome::AssertionFailed) + } + }; + } + run_ad_templates_with_writer(args, &mut out).map(|()| RunOutcome::Success) +} + +fn run_ad_templates_with_writer( + args: &AdTemplatesCommand, + out: &mut dyn Write, +) -> Result<(), String> { + match args { + AdTemplatesCommand::Lint(args) => run_lint(args, out), + AdTemplatesCommand::Match(args) => run_match(args, out), + AdTemplatesCommand::Check(args) => run_check(args, out), + AdTemplatesCommand::Explain(args) => run_explain(args, out), + } +} + +fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + writeln!(out, "app config: {}", loaded.app_config_path.display()).map_err(output_error)?; + + let Some(config) = &loaded.settings.creative_opportunities else { + writeln!(out, "server-side ad templates: not configured").map_err(output_error)?; + return Ok(()); + }; + + writeln!( + out, + "server-side ad templates: configured ({} slot{})", + config.slot.len(), + plural(config.slot.len()) + ) + .map_err(output_error)?; + writeln!( + out, + "gam_network_id: {}", + escape_terminal_text(&config.gam_network_id) + ) + .map_err(output_error)?; + writeln!( + out, + "auction_timeout_ms: {}", + config + .auction_timeout_ms + .unwrap_or(loaded.settings.auction.timeout_ms) + ) + .map_err(output_error)?; + writeln!( + out, + "creative_opportunities.enabled: {}", + if config.enabled { "true" } else { "false" } + ) + .map_err(output_error)?; + writeln!( + out, + "auction.enabled: {}", + if loaded.settings.auction.enabled { + "true" + } else { + "false" + } + ) + .map_err(output_error)?; + writeln!( + out, + "auction.providers: {}", + if loaded.settings.auction.providers.is_empty() { + "(none)".to_string() + } else { + loaded + .settings + .auction + .providers + .keys() + .map(|id| escape_terminal_text(id.as_str()).into_owned()) + .collect::>() + .join(", ") + } + ) + .map_err(output_error)?; + + if config.slot.is_empty() { + writeln!(out, "status: disabled because no slots are configured").map_err(output_error)?; + } else if !config.enabled { + writeln!( + out, + "status: slots are configured, but [creative_opportunities].enabled is false" + ) + .map_err(output_error)?; + } else if !loaded.settings.auction.enabled { + writeln!( + out, + "status: slots are configured, but [auction].enabled is false" + ) + .map_err(output_error)?; + } else if loaded.settings.auction.providers.is_empty() { + writeln!( + out, + "status: slots are configured, but [auction].providers is empty" + ) + .map_err(output_error)?; + } else { + writeln!(out, "status: eligible for legacy-path server-side auctions") + .map_err(output_error)?; + } + + for slot in &config.slot { + for pattern in &slot.page_patterns { + if let Err(error) = validate_page_pattern(pattern) { + writeln!( + out, + "invalid page pattern for slot `{}`: {}", + escape_terminal_text(&slot.id), + escape_terminal_text(&error), + ) + .map_err(output_error)?; + } + } + } + + Ok(()) +} + +fn run_match(args: &AdTemplatesMatchArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + let path = normalize_path_or_url(&args.path_or_url)?; + let Some(config) = &loaded.settings.creative_opportunities else { + writeln!( + out, + "{path}: no slots matched (creative_opportunities not configured)" + ) + .map_err(output_error)?; + return Ok(()); + }; + let matched = match_slots(&config.slot, &path); + + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + args.details, + ) +} + +fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), String> { + run_check_classified(args, out).map_err(|failure| match failure { + CheckFailure::Tool(error) | CheckFailure::Assertion(error) => error, + }) +} + +fn run_check_classified( + args: &AdTemplatesCheckArgs, + out: &mut dyn Write, +) -> Result<(), CheckFailure> { + let loaded = load_settings(&args.config).map_err(CheckFailure::Tool)?; + let path = normalize_path_or_url(&args.path_or_url).map_err(CheckFailure::Tool)?; + let matched = loaded + .settings + .creative_opportunities + .as_ref() + .map(|config| match_slots(&config.slot, &path)) + .unwrap_or_default(); + let actual: BTreeSet<&str> = matched.iter().map(|slot| slot.id.as_str()).collect(); + + if args.expect_no_slots { + if actual.is_empty() { + writeln!(out, "{path}: OK, no slots matched") + .map_err(output_error) + .map_err(CheckFailure::Tool)?; + return Ok(()); + } + return Err(CheckFailure::Assertion(format!( + "{path}: expected no slots, matched {}", + join_set(&actual) + ))); + } + + let expected: BTreeSet<&str> = args.expected_slots.iter().map(String::as_str).collect(); + let missing: BTreeSet<&str> = expected.difference(&actual).copied().collect(); + let extra: BTreeSet<&str> = actual.difference(&expected).copied().collect(); + + if missing.is_empty() && (args.allow_extra_slots || extra.is_empty()) { + writeln!(out, "{path}: OK, matched {}", join_set(&actual)) + .map_err(output_error) + .map_err(CheckFailure::Tool)?; + return Ok(()); + } + + let mut problems = Vec::new(); + if !missing.is_empty() { + problems.push(format!("missing {}", join_set(&missing))); + } + if !args.allow_extra_slots && !extra.is_empty() { + problems.push(format!("unexpected {}", join_set(&extra))); + } + Err(CheckFailure::Assertion(format!( + "{path}: {}", + problems.join("; ") + ))) +} + +fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + let path = normalize_path_or_url(&args.path_or_url)?; + writeln!(out, "path: {path}").map_err(output_error)?; + + let has_matches = if let Some(config) = &loaded.settings.creative_opportunities { + let matched = match_slots(&config.slot, &path); + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + true, + )?; + !matched.is_empty() + } else { + writeln!(out, "creative_opportunities: not configured").map_err(output_error)?; + false + }; + + let method_pass = args.method == Method::GET; + let navigation_pass = !args.non_navigation; + let consent_pass = !args.consent_denied; + let auction_enabled = loaded.settings.auction.enabled; + let ad_templates_enabled = loaded + .settings + .creative_opportunities + .as_ref() + .is_some_and(|config| config.enabled); + let providers_configured = !loaded.settings.auction.providers.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: method_pass, + navigation: navigation_pass, + prefetch: args.prefetch, + bot: args.bot, + matched_slots: has_matches, + consent_allows_auction: Some(consent_pass), + auction_enabled, + ad_templates_enabled, + }); + let blocked: Vec = gate.blocking_gates().collect(); + write_gate( + out, + "method GET", + !blocked.contains(&AdStackGateName::MethodGet), + )?; + write_gate( + out, + "navigation", + !blocked.contains(&AdStackGateName::Navigation), + )?; + write_gate( + out, + "not prefetch", + !blocked.contains(&AdStackGateName::NotPrefetch), + )?; + write_gate(out, "not bot", !blocked.contains(&AdStackGateName::NotBot))?; + write_gate( + out, + "consent allows auction", + !blocked.contains(&AdStackGateName::ConsentAllowsAuction), + )?; + write_gate( + out, + "auction.enabled", + !blocked.contains(&AdStackGateName::AuctionEnabled), + )?; + write_gate( + out, + "creative_opportunities.enabled", + !blocked.contains(&AdStackGateName::AdTemplatesEnabled), + )?; + write_gate( + out, + "matched slots", + !blocked.contains(&AdStackGateName::MatchedSlots), + )?; + writeln!( + out, + "advisory auction providers configured: {}", + if providers_configured { "yes" } else { "no" } + ) + .map_err(output_error)?; + writeln!( + out, + "server-side ad stack: {}", + match gate.expected { + RuntimeAdStackExpected::Yes => "yes", + RuntimeAdStackExpected::No => "no", + // `explain` always supplies a consent decision, which is the only + // input that yields `Unknown`; the arm is here for exhaustiveness. + RuntimeAdStackExpected::Unknown => "unknown", + } + ) + .map_err(output_error)?; + + Ok(()) +} + +fn write_match_result( + out: &mut dyn Write, + path: &str, + matched: &[&CreativeOpportunitySlot], + gam_network_id: &str, + section: &str, + details: bool, +) -> Result<(), String> { + if matched.is_empty() { + writeln!(out, "{}: no slots matched", escape_terminal_text(path)).map_err(output_error)?; + return Ok(()); + } + + let ids = matched + .iter() + .map(|slot| escape_terminal_text(&slot.id).into_owned()) + .collect::>() + .join(", "); + writeln!(out, "{}: matched {ids}", escape_terminal_text(path)).map_err(output_error)?; + + if details { + for slot in matched { + writeln!(out, "- {}", format_slot(slot, gam_network_id, section)) + .map_err(output_error)?; + } + } + + Ok(()) +} + +fn write_gate(out: &mut dyn Write, label: &str, pass: bool) -> Result<(), String> { + writeln!(out, "gate {label}: {}", if pass { "pass" } else { "block" }).map_err(output_error) +} + +/// Formats one matched slot for `--details` output. +/// +/// `section` is the value the runtime derives from the evaluated path, so a +/// `{section}` template renders the same unit path the live request would use. +fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str, section: &str) -> String { + let formats = slot + .formats + .iter() + .map(format_format) + .collect::>() + .join(", "); + let providers = format_providers(slot); + // `None` means a dynamic template renders past GAM's unit-path byte limit — + // a config the runtime rejects, so surface it rather than printing a path. + let gam_unit_path = slot + .render_gam_unit_path(gam_network_id, section) + .unwrap_or_else(|| "".to_string()); + format!( + "{} div={} gam={} patterns=[{}] formats=[{}] providers=[{}]", + escape_terminal_text(&slot.id), + escape_terminal_text(slot.resolved_div_id()), + escape_terminal_text(&gam_unit_path), + escape_terminal_text(&slot.page_patterns.join(", ")), + formats, + providers, + ) +} + +fn format_format(format: &CreativeOpportunityFormat) -> String { + let media_type = match format.media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + }; + format!("{}x{} {media_type}", format.width, format.height) +} + +fn format_providers(slot: &CreativeOpportunitySlot) -> String { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps"); + } + if slot.providers.prebid.is_some() { + providers.push("prebid"); + } + if providers.is_empty() { + return "none".to_string(); + } + providers.join(", ") +} + +/// Renders a set of config-derived slot ids for the terminal. +/// +/// Config can arrive from a pushed blob or the env overlay, not only from a file +/// the operator read, so the ids are escaped before they reach a terminal — the +/// assertion-failure path prints them too. +fn join_set(set: &BTreeSet<&str>) -> String { + if set.is_empty() { + return "(none)".to_string(); + } + set.iter() + .map(|id| escape_terminal_text(id).into_owned()) + .collect::>() + .join(", ") +} + +fn plural(count: usize) -> &'static str { + if count == 1 { "" } else { "s" } +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn output_error(err: io::Error) -> String { + format!("failed to write command output: {err}") +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + const EXAMPLE_CONFIG: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../trusted-server.example.toml" + )); + + fn project_with_config(config: &str) -> (TempDir, AppConfigArgs) { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + fs::write(&config_path, config).expect("should write app config"); + ( + temp, + AppConfigArgs { + app_config: Some(config_path), + manifest: manifest_path, + no_env: true, + }, + ) + } + + fn config_with_slots() -> String { + let base_config = EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ); + format!( + "{base_config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"atf\"\n\ + page_patterns = [\"/news/*\", \"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + targeting = {{ zone = \"atf\" }}\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {{}}\n\ + \n\ + [[creative_opportunities.slot]]\n\ + id = \"sports-sidebar\"\n\ + div_id = \"sports-ad\"\n\ + page_patterns = [\"/sports/*\"]\n\ + formats = [{{ width = 300, height = 600 }}]\n" + ) + } + + #[test] + fn match_reports_slots_for_path() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Match(AdTemplatesMatchArgs { + config, + path_or_url: "https://example.com/news/story?utm=1".to_string(), + details: true, + }), + &mut out, + ) + .expect("should match slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("/news/story: matched atf"), + "should report matched slot" + ); + assert!( + output.contains("formats=[300x250 banner]"), + "should include details" + ); + } + + #[test] + fn check_rejects_unexpected_extra_slots_by_default() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let err = run_ad_templates_with_writer( + &AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + }), + &mut Vec::new(), + ) + .expect_err("should reject mismatch"); + + assert!( + err.contains("missing atf") && err.contains("unexpected sports-sidebar"), + "should describe missing and unexpected slots" + ); + } + + #[test] + fn check_accepts_no_slots() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/weather/today".to_string(), + expected_slots: Vec::new(), + expect_no_slots: true, + allow_extra_slots: false, + }), + &mut out, + ) + .expect("should accept no slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("/weather/today: OK, no slots matched"), + "should report no-slot assertion" + ); + } + + #[test] + fn explain_keeps_provider_state_separate_from_runtime_verdict() { + let config_text = config_with_slots().replacen( + "\nenabled = false\n# Rewrite", + "\nenabled = true\n# Rewrite", + 1, + ); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { + config, + path_or_url: "/news/story".to_string(), + method: Method::GET, + non_navigation: false, + prefetch: false, + bot: false, + consent_denied: false, + }), + &mut out, + ) + .expect("should explain path"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("server-side ad stack: yes"), + "runtime verdict should not include provider configuration: {output}" + ); + assert!( + output.contains("advisory auction providers configured: yes"), + "provider state should be a separate advisory: {output}" + ); + } + + #[test] + fn lint_reports_configured_slot_count_and_auction_state() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint configured slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("server-side ad templates: configured (2 slots)"), + "should report the configured slot count" + ); + assert!( + output.contains("auction.enabled:"), + "should report the auction kill-switch state" + ); + assert!( + output.contains("auction.providers: pbs-main"), + "should report provider map identifiers: {output}" + ); + assert!(!output.contains("legacy fallback")); + } + + #[test] + fn lint_and_explain_report_the_disabled_template_switch() { + // `[creative_opportunities].enabled = false` is a runtime kill switch: + // the publisher path matches no slots at all while it is off, so the + // diagnostics must not claim the ad stack would run. + let config_text = config_with_slots().replace("enabled = true", "enabled = false"); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { + config: config.clone(), + }), + &mut out, + ) + .expect("should lint a disabled template switch"); + let lint_output = String::from_utf8(out).expect("should be utf8"); + + assert!( + lint_output.contains("creative_opportunities.enabled: false"), + "lint should report the template switch state: {lint_output}" + ); + assert!( + lint_output.contains( + "status: slots are configured, but [creative_opportunities].enabled is false" + ), + "lint status should name the template switch: {lint_output}" + ); + + let mut out = Vec::new(); + run_ad_templates_with_writer( + &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { + config, + path_or_url: "/news/story".to_string(), + method: Method::GET, + non_navigation: false, + prefetch: false, + bot: false, + consent_denied: false, + }), + &mut out, + ) + .expect("should explain a disabled template switch"); + let explain_output = String::from_utf8(out).expect("should be utf8"); + + assert!( + explain_output.contains("gate creative_opportunities.enabled: block"), + "explain should fail the template-switch gate: {explain_output}" + ); + assert!( + explain_output.contains("server-side ad stack: no"), + "explain verdict should follow the switch: {explain_output}" + ); + } + + #[test] + fn lint_reports_page_patterns_the_runtime_drops() { + let config_text = config_with_slots().replace( + "page_patterns = [\"/news/*\", \"/\"]", + "page_patterns = [\"/news/*\", \"[\"]", + ); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint mixed valid and invalid patterns"); + let output = String::from_utf8(out).expect("should be utf8"); + + assert!( + output.contains("invalid page pattern for slot `atf`") + && output.contains("page pattern '[' is not a valid glob"), + "lint should surface the runtime-dropped pattern: {output}" + ); + } + + #[test] + fn public_check_reports_drift_as_assertion_outcome() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let outcome = run_ad_templates(&AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + })) + .expect("assertion drift should not be a tool error"); + + assert_eq!(outcome, RunOutcome::AssertionFailed); + } + + #[test] + fn http_method_parser_normalizes_standard_methods() { + assert_eq!( + parse_http_method("get").expect("should parse lowercase GET"), + Method::GET, + "lowercase GET must evaluate the same runtime gate as uppercase GET" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/mod.rs b/crates/trusted-server-cli/src/commands/config/mod.rs index 43763f10a..77af7de85 100644 --- a/crates/trusted-server-cli/src/commands/config/mod.rs +++ b/crates/trusted-server-cli/src/commands/config/mod.rs @@ -1 +1,2 @@ +pub mod ad_templates; pub mod init; diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index 405bc6187..a3352ba93 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -1,4 +1,8 @@ #[cfg(not(target_arch = "wasm32"))] +mod ad_templates; +#[cfg(not(target_arch = "wasm32"))] +mod app_config; +#[cfg(not(target_arch = "wasm32"))] mod error; #[cfg(not(target_arch = "wasm32"))] mod prebid_bundle; @@ -6,7 +10,7 @@ mod prebid_bundle; mod run; #[cfg(not(target_arch = "wasm32"))] -pub use run::run_from_env; +pub use run::{RunOutcome, run_from_env}; // Every `ts` subcommand's implementation lives under `commands/`. The // `ts dev` group is available on every host target; its only subcommand, diff --git a/crates/trusted-server-cli/src/main.rs b/crates/trusted-server-cli/src/main.rs index 7cee5b1ca..0a325bd55 100644 --- a/crates/trusted-server-cli/src/main.rs +++ b/crates/trusted-server-cli/src/main.rs @@ -2,10 +2,21 @@ fn main() { use std::process; + // Dependencies such as chromiumoxide instrument their internals with + // `tracing`. Without a subscriber, tracing's log-compatibility fallback + // forwards tolerated CDP decode warnings into the CLI's user-facing logger. + // Trusted Server uses `log` for intentional operator output, so install a + // no-op tracing subscriber to keep dependency diagnostics out of stdout and + // stderr without changing the process-wide `log` level. + let _ = tracing::subscriber::set_global_default(tracing::subscriber::NoSubscriber::default()); edgezero_cli::init_cli_logger(); - if let Err(err) = trusted_server_cli::run_from_env() { - log::error!("[ts] {err}"); - process::exit(2); + match trusted_server_cli::run_from_env() { + Ok(outcome) if outcome.exit_code() != 0 => process::exit(outcome.exit_code()), + Ok(_) => {} + Err(err) => { + log::error!("[ts] {err}"); + process::exit(2); + } } } diff --git a/crates/trusted-server-cli/src/prebid_bundle.rs b/crates/trusted-server-cli/src/prebid_bundle.rs index abc545926..7bf3b6267 100644 --- a/crates/trusted-server-cli/src/prebid_bundle.rs +++ b/crates/trusted-server-cli/src/prebid_bundle.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::env; use std::fs::{self, OpenOptions}; use std::io::Write; @@ -10,6 +11,7 @@ use toml_edit::{DocumentMut, Item, table, value}; pub(crate) type CliResult = Result; const NODE_MODULES_MISSING_HELP: &str = "Prebid bundling dependencies are missing. Run `cd crates/trusted-server-js/lib && npm ci`, then retry `ts prebid bundle`."; +const USER_ID_REGISTRY_RELATIVE_PATH: &str = "src/integrations/prebid/user_id_modules.json"; #[derive(Debug, clap::Args)] pub(crate) struct PrebidBundleArgs { @@ -33,6 +35,7 @@ fn cli_error(message: impl Into) -> CliResult { pub(crate) struct PrebidBundleConfig { pub adapters: Vec, pub user_id_modules: Option>, + pub managed_user_id_names: Vec, pub external_bundle_url: Option, } @@ -120,11 +123,115 @@ fn npm_prebid_bundle_args(request: &PrebidBundleGenerateRequest) -> Vec #[derive(Debug, Deserialize)] struct PrebidBundleManifest { + #[serde(rename = "userIdModules")] + user_id_modules: Vec, sha256: String, sri: String, filename: String, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PrebidUserIdModuleRegistry { + modules: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PrebidUserIdModuleRegistryEntry { + module_name: String, + config_names: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RequiredPrebidUserIdModule { + config_name: String, + module_name: String, +} + +fn load_user_id_registry(js_lib_dir: &Path) -> CliResult<(PathBuf, PrebidUserIdModuleRegistry)> { + let path = js_lib_dir.join(USER_ID_REGISTRY_RELATIVE_PATH); + let contents = fs::read_to_string(&path).map_err(|error| { + report_error(format!( + "failed to read Prebid User ID registry {}: {error}", + path.display() + )) + })?; + let registry = serde_json::from_str(&contents).map_err(|error| { + report_error(format!( + "failed to parse Prebid User ID registry {}: {error}", + path.display() + )) + })?; + Ok((path, registry)) +} + +/// Rejects managed User ID names that resolve to one Prebid submodule. +/// +/// Prebid registers a single submodule for a module's name and each of its +/// aliases, then selects the first matching `userSync.userIds` entry. Two +/// managed names sharing a module therefore silently drop one operator +/// configuration, so reject the pair instead of generating a bundle whose +/// behaviour does not match the configuration. +fn reject_managed_user_id_module_collisions( + resolved: &[RequiredPrebidUserIdModule], + registry_path: &Path, +) -> CliResult<()> { + let mut owners: HashMap<&str, &str> = HashMap::with_capacity(resolved.len()); + for entry in resolved { + let Some(previous) = owners.insert(&entry.module_name, &entry.config_name) else { + continue; + }; + return cli_error(format!( + "managed User ID names {previous:?} and {:?} both resolve to module {:?} in {}; \ + Prebid registers one submodule for those names and ignores every entry after the first", + entry.config_name, + entry.module_name, + registry_path.display() + )); + } + Ok(()) +} + +fn resolve_managed_user_id_modules( + managed_names: &[String], + registry: &PrebidUserIdModuleRegistry, + registry_path: &Path, +) -> CliResult> { + let resolved = managed_names + .iter() + .map(|config_name| { + let mut candidates = registry + .modules + .iter() + .filter(|entry| entry.config_names.iter().any(|name| name == config_name)) + .map(|entry| entry.module_name.clone()) + .collect::>(); + candidates.sort(); + candidates.dedup(); + + match candidates.as_slice() { + [] => cli_error(format!( + "managed User ID name {config_name:?} is not registered in {}", + registry_path.display() + )), + [module_name] => Ok(RequiredPrebidUserIdModule { + config_name: config_name.clone(), + module_name: module_name.clone(), + }), + _ => cli_error(format!( + "managed User ID name {config_name:?} is ambiguous in {}; candidate modules: {}", + registry_path.display(), + candidates.join(", ") + )), + } + }) + .collect::>>()?; + + reject_managed_user_id_module_collisions(&resolved, registry_path)?; + Ok(resolved) +} + pub(crate) fn run_bundle( args: &PrebidBundleArgs, generator: &mut dyn PrebidBundleGenerator, @@ -135,11 +242,51 @@ pub(crate) fn run_bundle( let current_dir = env::current_dir() .map_err(|error| report_error(format!("failed to read current directory: {error}")))?; let js_lib_dir = find_js_lib_dir(¤t_dir)?; - let out_dir = resolve_output_dir(¤t_dir, &args.out); + let (registry_path, registry) = load_user_id_registry(&js_lib_dir)?; + + run_bundle_with_context( + args, + config, + PrebidBundleRunContext { + current_dir: ¤t_dir, + js_lib_dir, + registry_path: ®istry_path, + registry: ®istry, + }, + generator, + out, + err, + ) +} + +struct PrebidBundleRunContext<'a> { + current_dir: &'a Path, + js_lib_dir: PathBuf, + registry_path: &'a Path, + registry: &'a PrebidUserIdModuleRegistry, +} + +fn run_bundle_with_context( + args: &PrebidBundleArgs, + config: PrebidBundleConfig, + context: PrebidBundleRunContext<'_>, + generator: &mut dyn PrebidBundleGenerator, + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()> { + let requirements = resolve_managed_user_id_modules( + &config.managed_user_id_names, + context.registry, + context.registry_path, + )?; + let out_dir = resolve_output_dir(context.current_dir, &args.out); ensure_output_dir_writable(&out_dir)?; + let manifest_path = out_dir.join("manifest.json"); + invalidate_manifest(&manifest_path)?; + let request = PrebidBundleGenerateRequest { - js_lib_dir, + js_lib_dir: context.js_lib_dir, out_dir: out_dir.clone(), adapters: config.adapters, user_id_modules: config.user_id_modules, @@ -147,8 +294,8 @@ pub(crate) fn run_bundle( generator.generate(&request, out, err)?; - let manifest_path = out_dir.join("manifest.json"); let manifest = load_manifest(&manifest_path)?; + validate_managed_user_id_modules(&requirements, &manifest, &args.config)?; patch_config_metadata(&args.config, &manifest.sha256, &manifest.sri)?; writeln!( @@ -179,6 +326,40 @@ pub(crate) fn run_bundle( Ok(()) } +fn validate_managed_user_id_modules( + requirements: &[RequiredPrebidUserIdModule], + manifest: &PrebidBundleManifest, + config_path: &Path, +) -> CliResult<()> { + for requirement in requirements { + if !manifest + .user_id_modules + .iter() + .any(|module| module == &requirement.module_name) + { + return cli_error(format!( + "{} configures managed User ID {:?}, which requires Prebid module {:?}, but the generated manifest omits it; add {:?} to integrations.prebid.bundle.user_id_modules and rerun `ts prebid bundle`", + config_path.display(), + requirement.config_name, + requirement.module_name, + requirement.module_name, + )); + } + } + Ok(()) +} + +fn invalidate_manifest(path: &Path) -> CliResult<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => cli_error(format!( + "failed to remove stale Prebid manifest {}: {error}", + path.display() + )), + } +} + pub(crate) fn load_bundle_config(config_path: &Path) -> CliResult { let contents = fs::read_to_string(config_path).map_err(|error| { report_error(format!( @@ -235,6 +416,8 @@ pub(crate) fn load_bundle_config(config_path: &Path) -> CliResult CliResult CliResult> { + let Some(value) = prebid.get("managed_user_ids") else { + return Ok(Vec::new()); + }; + let entries = value.as_array().ok_or_else(|| { + report_error(format!( + "{} integrations.prebid.managed_user_ids must be an array of tables", + config_path.display() + )) + })?; + + entries + .iter() + .enumerate() + .map(|(index, entry)| { + let table = entry.as_table().ok_or_else(|| { + report_error(format!( + "{} integrations.prebid.managed_user_ids[{index}] must be a table", + config_path.display() + )) + })?; + let field = format!("integrations.prebid.managed_user_ids[{index}].name"); + let name = table + .get("name") + .and_then(toml::Value::as_str) + .ok_or_else(|| { + report_error(format!( + "{} {field} must be a non-empty string", + config_path.display() + )) + })?; + if name.trim().is_empty() { + return cli_error(format!( + "{} {field} must be a non-empty string", + config_path.display() + )); + } + Ok(name.to_string()) + }) + .collect() +} + fn read_required_string_array( table: &toml::Value, key: &str, @@ -568,6 +794,65 @@ user_id_modules = ["sharedIdSystem", "uid2IdSystem"] .to_string() } + fn managed_identity_link_config() -> String { + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-old.js" +external_bundle_sha256 = "old-sha256" +external_bundle_sri = "sha384-old" + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +user_id_modules = ["identityLinkIdSystem"] +"# + .to_string() + } + + fn two_managed_ids_config() -> String { + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +external_bundle_sha256 = "old-sha256" +external_bundle_sri = "sha384-old" + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" + +[[integrations.prebid.managed_user_ids]] +name = "uid2" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +user_id_modules = ["identityLinkIdSystem", "uid2IdSystem"] +"# + .to_string() + } + + fn shared_aliases_config() -> String { + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "sharedId" + +[[integrations.prebid.managed_user_ids]] +name = "pubCommonId" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +user_id_modules = ["sharedIdSystem"] +"# + .to_string() + } + #[test] fn bundle_config_loader_accepts_valid_settings() { let (_temp, path) = write_config(&valid_config()); @@ -586,6 +871,7 @@ user_id_modules = ["sharedIdSystem", "uid2IdSystem"] config.external_bundle_url.as_deref(), Some("https://assets.example.com/prebid/trusted-prebid-old.js") ); + assert!(config.managed_user_id_names.is_empty()); } #[test] @@ -604,6 +890,260 @@ adapters = ["rubicon"] assert_eq!(config.adapters, ["rubicon"]); assert_eq!(config.user_id_modules, None); + assert!(config.managed_user_id_names.is_empty()); + } + + #[test] + fn bundle_config_loader_reads_managed_user_id_names_in_order() { + let (_temp, path) = write_config( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" + +[[integrations.prebid.managed_user_ids]] +name = "pubCommonId" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"#, + ); + + let config = load_bundle_config(&path).expect("should load managed names"); + + assert_eq!( + config.managed_user_id_names, + ["identityLink", "pubCommonId"], + "should preserve managed entry order" + ); + } + + #[test] + fn bundle_config_loader_rejects_non_array_managed_user_ids() { + for managed_user_ids in ["\"identityLink\"", "{ name = \"identityLink\" }"] { + let (_temp, path) = write_config(&format!( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +managed_user_ids = {managed_user_ids} + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"# + )); + + let error = load_bundle_config(&path).expect_err("should require an array"); + + assert!( + error.contains("integrations.prebid.managed_user_ids must be an array of tables"), + "should identify the malformed managed list: {error}" + ); + } + } + + #[test] + fn bundle_config_loader_rejects_non_table_managed_entry() { + let (_temp, path) = write_config( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +managed_user_ids = ["identityLink"] + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"#, + ); + + let error = load_bundle_config(&path).expect_err("should require managed tables"); + + assert!( + error.contains("integrations.prebid.managed_user_ids[0] must be a table"), + "should identify the malformed managed entry: {error}" + ); + } + + #[test] + fn bundle_config_loader_rejects_managed_entry_without_string_name() { + for entry in [ + "{ params = { pid = \"999\" } }", + "{ name = 123 }", + "{ name = \"\" }", + "{ name = \" \" }", + ] { + let (_temp, path) = write_config(&format!( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +managed_user_ids = [{entry}] + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"# + )); + + let error = load_bundle_config(&path).expect_err("should reject malformed name"); + + assert!( + error.contains("integrations.prebid.managed_user_ids[0].name"), + "should identify the malformed managed name: {error}" + ); + } + } + + #[test] + fn managed_name_resolves_an_alias_to_its_registered_module() { + let registry = PrebidUserIdModuleRegistry { + modules: vec![PrebidUserIdModuleRegistryEntry { + module_name: "sharedIdSystem".to_string(), + config_names: vec!["sharedId".to_string(), "pubCommonId".to_string()], + }], + }; + let registry_path = Path::new("user_id_modules.json"); + + let required = + resolve_managed_user_id_modules(&["pubCommonId".to_string()], ®istry, registry_path) + .expect("should resolve the alias"); + + assert_eq!( + required, + [RequiredPrebidUserIdModule { + config_name: "pubCommonId".to_string(), + module_name: "sharedIdSystem".to_string(), + }], + "should resolve an alias to its registered module" + ); + } + + #[test] + fn managed_names_sharing_one_module_are_rejected() { + let registry = PrebidUserIdModuleRegistry { + modules: vec![PrebidUserIdModuleRegistryEntry { + module_name: "sharedIdSystem".to_string(), + config_names: vec!["sharedId".to_string(), "pubCommonId".to_string()], + }], + }; + let registry_path = Path::new("registry/user_id_modules.json"); + + let error = resolve_managed_user_id_modules( + &["pubCommonId".to_string(), "sharedId".to_string()], + ®istry, + registry_path, + ) + .expect_err("should reject two names resolving to one module"); + + assert!( + error.contains("pubCommonId") && error.contains("sharedId"), + "should identify both managed names: {error}" + ); + assert!( + error.contains("sharedIdSystem"), + "should identify the shared module: {error}" + ); + assert!( + error.contains(®istry_path.display().to_string()), + "should identify the registry: {error}" + ); + } + + #[test] + fn checked_in_registry_resolves_identity_link() { + let current_dir = env::current_dir().expect("should read current directory"); + let js_lib_dir = find_js_lib_dir(¤t_dir).expect("should locate JS library"); + let (registry_path, registry) = + load_user_id_registry(&js_lib_dir).expect("should load checked-in registry"); + + let required = resolve_managed_user_id_modules( + &["identityLink".to_string()], + ®istry, + ®istry_path, + ) + .expect("should resolve checked-in identityLink entry"); + + assert_eq!( + required, + [RequiredPrebidUserIdModule { + config_name: "identityLink".to_string(), + module_name: "identityLinkIdSystem".to_string(), + }] + ); + } + + #[test] + fn unknown_managed_name_identifies_name_and_registry() { + let registry = PrebidUserIdModuleRegistry { + modules: Vec::new(), + }; + let registry_path = Path::new("registry/user_id_modules.json"); + + let error = + resolve_managed_user_id_modules(&["unknownId".to_string()], ®istry, registry_path) + .expect_err("should reject unknown name"); + + assert!( + error.contains("unknownId"), + "should identify the name: {error}" + ); + assert!( + error.contains(®istry_path.display().to_string()), + "should identify the registry: {error}" + ); + } + + #[test] + fn ambiguous_managed_name_lists_sorted_candidate_modules() { + let registry = PrebidUserIdModuleRegistry { + modules: vec![ + PrebidUserIdModuleRegistryEntry { + module_name: "zetaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + PrebidUserIdModuleRegistryEntry { + module_name: "alphaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + PrebidUserIdModuleRegistryEntry { + module_name: "zetaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + ], + }; + let registry_path = Path::new("registry/user_id_modules.json"); + + let error = + resolve_managed_user_id_modules(&["ambiguousId".to_string()], ®istry, registry_path) + .expect_err("should reject ambiguous name"); + + assert!( + error.contains("ambiguousId"), + "should identify the name: {error}" + ); + assert!( + error.contains("alphaIdSystem, zetaIdSystem"), + "should list sorted unique candidates: {error}" + ); + assert!( + error.contains(®istry_path.display().to_string()), + "should identify the registry: {error}" + ); + } + + #[test] + fn empty_managed_names_require_no_modules() { + let registry = PrebidUserIdModuleRegistry { + modules: Vec::new(), + }; + + let required = + resolve_managed_user_id_modules(&[], ®istry, Path::new("user_id_modules.json")) + .expect("should accept no managed names"); + + assert!(required.is_empty()); } #[test] @@ -792,7 +1332,18 @@ adapters = ["rubicon", 123] struct FakeGenerator { generate_error: Option, generate_calls: Vec, - write_manifest: bool, + manifest: Option, + } + + fn fake_manifest(user_id_modules: &serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "prebidVersion": "10.26.0", + "adapters": ["rubicon"], + "userIdModules": user_id_modules, + "sha256": "b".repeat(64), + "sri": "sha384-test", + "filename": format!("trusted-prebid-{}.js", "b".repeat(64)) + }) } impl PrebidBundleGenerator for FakeGenerator { @@ -809,21 +1360,10 @@ adapters = ["rubicon", 123] err.write_all(b"generator stderr\n") .expect("should capture generator stderr"); - if self.write_manifest { + if let Some(manifest) = &self.manifest { fs::create_dir_all(&request.out_dir).expect("should create output dir"); - fs::write( - request.out_dir.join("manifest.json"), - serde_json::json!({ - "prebidVersion": "10.26.0", - "adapters": request.adapters, - "userIdModules": request.user_id_modules.clone().unwrap_or_default(), - "sha256": "b".repeat(64), - "sri": "sha384-test", - "filename": format!("trusted-prebid-{}.js", "b".repeat(64)) - }) - .to_string(), - ) - .expect("should write fake manifest"); + fs::write(request.out_dir.join("manifest.json"), manifest.to_string()) + .expect("should write fake manifest"); } if let Some(error) = &self.generate_error { @@ -843,7 +1383,10 @@ adapters = ["rubicon", 123] let mut generator = FakeGenerator { generate_error: None, generate_calls: Vec::new(), - write_manifest: true, + manifest: Some(fake_manifest(&serde_json::json!([ + "sharedIdSystem", + "uid2IdSystem" + ]))), }; let mut out = Vec::new(); let mut err = Vec::new(); @@ -885,7 +1428,7 @@ adapters = ["rubicon", 123] let mut generator = FakeGenerator { generate_error: Some("builder failed".to_string()), generate_calls: Vec::new(), - write_manifest: false, + manifest: None, }; let mut out = Vec::new(); let mut err = Vec::new(); @@ -901,6 +1444,347 @@ adapters = ["rubicon", 123] assert!(fs::read_to_string(&args.config).expect("should read config") == original_config); } + #[test] + fn run_bundle_rejects_managed_name_when_manifest_omits_required_module() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["sharedIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject missing managed module"); + + assert!( + error.contains("identityLink"), + "should name managed config: {error}" + ); + assert!( + error.contains("identityLinkIdSystem"), + "should name required module: {error}" + ); + assert!( + error.contains("integrations.prebid.bundle.user_id_modules"), + "should identify corrective field: {error}" + ); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original, + "should not patch metadata after consistency failure" + ); + } + + #[test] + fn run_bundle_accepts_manifest_with_required_managed_module() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["identityLinkIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect("should accept required managed module"); + + assert_eq!(generator.generate_calls.len(), 1); + } + + #[test] + fn run_bundle_requires_every_managed_module() { + let (_temp, config_path) = write_config(&two_managed_ids_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["identityLinkIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should require every managed module"); + + assert!( + error.contains("uid2"), + "should identify omitted config: {error}" + ); + assert!( + error.contains("uid2IdSystem"), + "should identify omitted module: {error}" + ); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + + #[test] + fn run_bundle_rejects_two_aliases_backed_by_one_module() { + let (_temp, config_path) = write_config(&shared_aliases_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["sharedIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject aliases backed by one module"); + + assert!( + error.contains("sharedIdSystem"), + "should identify the shared module: {error}" + ); + assert!( + generator.generate_calls.is_empty(), + "should reject before generating a bundle" + ); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original, + "should leave the configured bundle metadata untouched" + ); + } + + #[test] + fn run_bundle_accepts_default_manifest_module_when_module_list_is_omitted() { + let config = managed_identity_link_config() + .replace("user_id_modules = [\"identityLinkIdSystem\"]\n", ""); + let (_temp, config_path) = write_config(&config); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["identityLinkIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect("should validate the generated default module set"); + + assert_eq!(generator.generate_calls[0].user_id_modules, None); + } + + #[test] + fn run_bundle_rejects_unknown_managed_name_before_generation() { + let config = managed_identity_link_config().replace("identityLink", "unknownId"); + let (_temp, config_path) = write_config(&config); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!([]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject unknown managed name"); + + assert!(error.contains("unknownId")); + assert!(generator.generate_calls.is_empty()); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + + #[test] + fn run_bundle_rejects_ambiguous_managed_name_before_generation() { + let config = managed_identity_link_config().replace("identityLink", "ambiguousId"); + let (_temp, config_path) = write_config(&config); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let registry = PrebidUserIdModuleRegistry { + modules: vec![ + PrebidUserIdModuleRegistryEntry { + module_name: "zetaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + PrebidUserIdModuleRegistryEntry { + module_name: "alphaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + ], + }; + let registry_path = Path::new("synthetic/user_id_modules.json"); + let args = PrebidBundleArgs { + config: config_path.clone(), + out: output_root.path().join("prebid"), + }; + let loaded = load_bundle_config(&config_path).expect("should load focused config"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!([]))), + }; + + let error = run_bundle_with_context( + &args, + loaded, + PrebidBundleRunContext { + current_dir: output_root.path(), + js_lib_dir: PathBuf::from("unused-js-lib"), + registry_path, + registry: ®istry, + }, + &mut generator, + &mut Vec::new(), + &mut Vec::new(), + ) + .expect_err("should reject ambiguous managed name"); + + assert!(error.contains("ambiguousId")); + assert!(error.contains("alphaIdSystem, zetaIdSystem")); + assert!(generator.generate_calls.is_empty()); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + + #[test] + fn run_bundle_rejects_malformed_managed_name_before_generation() { + let config = managed_identity_link_config() + .replace("name = \"identityLink\"", "params = { pid = \"999\" }"); + let (_temp, config_path) = write_config(&config); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!([]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject malformed managed name"); + + assert!(error.contains("managed_user_ids[0].name")); + assert!(generator.generate_calls.is_empty()); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + + #[test] + fn run_bundle_rejects_manifest_without_user_id_modules() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut manifest = fake_manifest(&serde_json::json!([])); + manifest + .as_object_mut() + .expect("should be an object") + .remove("userIdModules"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(manifest), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should require manifest userIdModules"); + + assert!( + error.contains("userIdModules"), + "should identify missing field: {error}" + ); + } + + #[test] + fn run_bundle_rejects_non_array_manifest_user_id_modules() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!("identityLinkIdSystem"))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should require array manifest userIdModules"); + + assert!( + error.contains("failed to parse generated Prebid manifest"), + "should identify manifest parsing: {error}" + ); + } + + #[test] + fn run_bundle_cannot_reuse_stale_manifest() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let out_dir = output_root.path().join("prebid"); + fs::create_dir_all(&out_dir).expect("should create output directory"); + let manifest_path = out_dir.join("manifest.json"); + fs::write( + &manifest_path, + fake_manifest(&serde_json::json!(["identityLinkIdSystem"])).to_string(), + ) + .expect("should write stale manifest"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: None, + }; + let args = PrebidBundleArgs { + config: config_path, + out: out_dir, + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject missing fresh manifest"); + + assert!( + error.contains("manifest"), + "should identify missing manifest: {error}" + ); + assert!(!manifest_path.exists(), "should remove stale manifest"); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + #[test] fn missing_node_modules_fails_with_npm_ci_instruction() { let temp = tempfile::TempDir::new().expect("should create temp dir"); diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 13009d448..ec56238c1 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -7,8 +7,8 @@ use edgezero_cli::args::{ }; use trusted_server_core::config::TrustedServerAppConfig; -use crate::commands::audit::AuditArgs; -use crate::commands::audit::browser_collector::BrowserAuditCollector; +use crate::commands::audit::{AuditArgs, run_audit}; +use crate::commands::config::ad_templates::{AdTemplatesCommand, run_ad_templates}; use crate::commands::config::init::{ConfigInitArgs, run_config_init}; use crate::prebid_bundle::{NpmPrebidBundleGenerator, PrebidBundleArgs, run_bundle}; @@ -23,8 +23,8 @@ struct Args { enum Command { /// Print the currently active deployment version for a target adapter. ActiveVersion(ActiveVersionArgs), - /// Audit a public page and write draft Trusted Server artifacts. - Audit(AuditArgs), + /// Browser-backed page and ad-template audits. + Audit(Box), /// Sign in / out / status against an `EdgeZero` adapter. Auth(AuthArgs), /// Build the project for a target adapter. @@ -51,6 +51,9 @@ enum Command { #[derive(Debug, Subcommand)] enum ConfigCommand { + /// Diagnose server-side ad-template configuration and path matching. + #[command(name = "ad-templates", subcommand)] + AdTemplates(AdTemplatesCommand), /// Initialize a Trusted Server config file from the example template. Init(ConfigInitArgs), /// Diff `trusted-server.toml` against the live `EdgeZero` config. @@ -75,44 +78,71 @@ enum PrebidCommand { Bundle(PrebidBundleArgs), } +/// Process-level outcome for commands that distinguish drift from tool errors. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + /// Command completed without drift. + Success, + /// Command ran successfully and found assertion drift. + AssertionFailed, +} + +impl RunOutcome { + /// Stable process exit code for this outcome. + #[must_use] + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} + /// Run the CLI using process arguments. /// /// # Errors /// /// Returns an error when command parsing, config validation, `EdgeZero` /// delegation, audit collection, config initialization, or Prebid bundle generation fails. -pub fn run_from_env() -> Result<(), String> { +pub fn run_from_env() -> Result { dispatch(Args::parse()) } -fn dispatch(args: Args) -> Result<(), String> { +fn dispatch(args: Args) -> Result { match args.command { - Command::ActiveVersion(args) => edgezero_cli::run_active_version(&args), - Command::Audit(args) => { - let stdout = std::io::stdout(); - let mut out = stdout.lock(); - let collector = BrowserAuditCollector; - crate::commands::audit::run_audit(&args, &collector, &mut out) + Command::ActiveVersion(args) => { + edgezero_cli::run_active_version(&args).map(|()| RunOutcome::Success) + } + Command::Auth(args) => edgezero_cli::run_auth(&args).map(|()| RunOutcome::Success), + Command::Audit(args) => run_audit(&args), + Command::Build(args) => edgezero_cli::run_build(&args).map(|()| RunOutcome::Success), + Command::Config(ConfigCommand::AdTemplates(args)) => run_ad_templates(&args), + Command::Config(ConfigCommand::Init(args)) => { + run_config_init(&args).map(|()| RunOutcome::Success) } - Command::Auth(args) => edgezero_cli::run_auth(&args), - Command::Build(args) => edgezero_cli::run_build(&args), - Command::Config(ConfigCommand::Init(args)) => run_config_init(&args), Command::Config(ConfigCommand::Diff(args)) => { match edgezero_cli::run_config_diff_typed::(&args) { - Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(()), + Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(RunOutcome::Success), + Ok(edgezero_cli::DiffExit { code: 1 }) => Ok(RunOutcome::AssertionFailed), Ok(edgezero_cli::DiffExit { code }) => process::exit(code), Err(err) => Err(err), } } - Command::Config(ConfigCommand::Gc(args)) => edgezero_cli::run_config_gc(&args), + Command::Config(ConfigCommand::Gc(args)) => { + edgezero_cli::run_config_gc(&args).map(|()| RunOutcome::Success) + } Command::Config(ConfigCommand::Push(args)) => { edgezero_cli::run_config_push_typed::(&args) + .map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Validate(args)) => { edgezero_cli::run_config_validate_typed::(&args) + .map(|()| RunOutcome::Success) + } + Command::Deploy(args) => edgezero_cli::run_deploy(&args).map(|()| RunOutcome::Success), + Command::Healthcheck(args) => { + edgezero_cli::run_healthcheck(&args).map(|()| RunOutcome::Success) } - Command::Deploy(args) => edgezero_cli::run_deploy(&args), - Command::Healthcheck(args) => edgezero_cli::run_healthcheck(&args), Command::Prebid(prebid) => { let mut generator = NpmPrebidBundleGenerator; let mut stdout = std::io::stdout(); @@ -120,13 +150,16 @@ fn dispatch(args: Args) -> Result<(), String> { match prebid.command { PrebidCommand::Bundle(args) => { run_bundle(&args, &mut generator, &mut stdout, &mut stderr) + .map(|()| RunOutcome::Success) } } } - Command::Provision(args) => edgezero_cli::run_provision(&args), - Command::Rollback(args) => edgezero_cli::run_rollback(&args), - Command::Serve(args) => edgezero_cli::run_serve(&args), - Command::Dev(command) => crate::commands::dev::run(command), + Command::Provision(args) => { + edgezero_cli::run_provision(&args).map(|()| RunOutcome::Success) + } + Command::Rollback(args) => edgezero_cli::run_rollback(&args).map(|()| RunOutcome::Success), + Command::Serve(args) => edgezero_cli::run_serve(&args).map(|()| RunOutcome::Success), + Command::Dev(command) => crate::commands::dev::run(command).map(|()| RunOutcome::Success), } } @@ -143,6 +176,12 @@ mod tests { Args::try_parse_from(args).expect("should parse args") } + #[test] + fn run_outcomes_use_documented_exit_codes() { + assert_eq!(RunOutcome::Success.exit_code(), 0); + assert_eq!(RunOutcome::AssertionFailed.exit_code(), 1); + } + #[test] fn top_level_version_flag_is_available() { let err = Args::try_parse_from(["ts", "--version"]) @@ -353,64 +392,6 @@ mod tests { ); } - #[test] - fn parses_audit_with_default_outputs() { - let args = parse(&["ts", "audit", "https://publisher.example"]); - let Command::Audit(audit) = args.command else { - panic!("expected audit command"); - }; - assert_eq!(audit.url, "https://publisher.example"); - assert_eq!(audit.js_assets, None); - assert_eq!(audit.config, None); - assert!(!audit.no_js_assets); - assert!(!audit.no_config); - assert!(!audit.force); - } - - #[test] - fn parses_audit_with_custom_outputs() { - let args = parse(&[ - "ts", - "audit", - "https://publisher.example", - "--js-assets", - "audit/js-assets.toml", - "--config", - "audit/trusted-server.toml", - "--no-js-assets", - "--no-config", - "--force", - ]); - let Command::Audit(audit) = args.command else { - panic!("expected audit command"); - }; - assert_eq!(audit.js_assets, Some(PathBuf::from("audit/js-assets.toml"))); - assert_eq!( - audit.config, - Some(PathBuf::from("audit/trusted-server.toml")) - ); - assert!(audit.no_js_assets); - assert!(audit.no_config); - assert!(audit.force); - } - - #[test] - fn audit_does_not_accept_adapter_option() { - let error = Args::try_parse_from([ - "ts", - "audit", - "https://publisher.example", - "--adapter", - "fastly", - ]) - .expect_err("should reject audit adapter option"); - assert!( - error.to_string().contains("unexpected argument") - || error.to_string().contains("Found argument"), - "error should explain unsupported option" - ); - } - #[test] fn parses_build_with_adapter_args() { let args = parse(&[ @@ -638,6 +619,420 @@ mod tests { assert_eq!(validate.manifest, default_validate.manifest); } + #[test] + fn config_ad_templates_match_parses_app_config_flags() { + let args = parse(&[ + "ts", + "config", + "ad-templates", + "match", + "--app-config", + "publisher-a.toml", + "--no-env", + "--details", + "/news/story", + ]); + let Command::Config(ConfigCommand::AdTemplates(AdTemplatesCommand::Match(match_args))) = + args.command + else { + panic!("expected ad-templates match command"); + }; + assert_eq!( + match_args.config.app_config, + Some(PathBuf::from("publisher-a.toml")) + ); + assert!(match_args.config.no_env); + assert!(match_args.details); + assert_eq!(match_args.path_or_url, "/news/story"); + } + + #[test] + fn config_ad_templates_check_parses_expected_slots() { + let args = parse(&[ + "ts", + "config", + "ad-templates", + "check", + "/sports/game", + "--expected-slot", + "atf", + "--expected-slot", + "sports-sidebar", + "--allow-extra-slots", + ]); + let Command::Config(ConfigCommand::AdTemplates(AdTemplatesCommand::Check(check_args))) = + args.command + else { + panic!("expected ad-templates check command"); + }; + assert_eq!(check_args.path_or_url, "/sports/game"); + assert_eq!(check_args.expected_slots, ["atf", "sports-sidebar"]); + assert!(check_args.allow_extra_slots); + assert!(!check_args.expect_no_slots); + } + + #[test] + fn config_ad_templates_check_requires_an_expectation_mode() { + assert!(Args::try_parse_from(["ts", "config", "ad-templates", "check", "/news"]).is_err()); + } + + #[test] + fn config_ad_templates_check_rejects_extra_slots_with_no_slots_mode() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "check", + "/news", + "--expect-no-slots", + "--allow-extra-slots", + ]) + .is_err() + ); + } + + #[test] + fn config_ad_templates_explain_rejects_removed_edgezero_model() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "explain", + "/news", + "--edgezero-enabled", + ]) + .is_err() + ); + } + + #[test] + fn cli_definition_is_valid() { + // clap validates `requires` / `conflicts_with` argument-id references + // only from an explicit `debug_assert`. Without this, renaming or + // typoing an id compiles and ships. + ::command().debug_assert(); + } + + #[test] + fn bare_audit_namespace_displays_help_as_an_error() { + let error = Args::try_parse_from(["ts", "audit"]).expect_err("should require audit mode"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::DisplayHelpOnMissingArgumentOrSubcommand + ); + } + + #[test] + fn audit_legacy_url_parses_with_artifact_generation_flags() { + let args = parse(&[ + "ts", + "audit", + "https://www.example.com/", + "--js-assets", + "audit/assets.toml", + "--config", + "audit/config.toml", + "--force", + "--cookie", + "session=example", + "--chrome", + "/tmp/test-chrome", + "--headful", + "--no-assume-consent", + "--browser-proxy", + "127.0.0.1:8080", + "--settle-quiet-ms", + "900", + "--settle-max-ms", + "13000", + "--danger-accept-invalid-certs", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + assert_eq!( + audit.legacy_generate.js_assets, + Some(PathBuf::from("audit/assets.toml")) + ); + assert_eq!( + audit.legacy_generate.config, + Some(PathBuf::from("audit/config.toml")) + ); + assert!(audit.legacy_generate.force); + assert_eq!( + audit.legacy_generate.cookies, + [("session".to_string(), "example".to_string())] + ); + assert_eq!( + audit.legacy_generate.browser.chrome, + Some(PathBuf::from("/tmp/test-chrome")) + ); + assert!(audit.legacy_generate.browser.headful); + assert!(audit.legacy_generate.browser.no_assume_consent); + assert_eq!( + audit.legacy_generate.browser.browser_proxy.as_deref(), + Some("127.0.0.1:8080") + ); + assert_eq!(audit.legacy_generate.browser.settle_quiet_ms, 900); + assert_eq!(audit.legacy_generate.browser.settle_max_ms, 13_000); + assert!(audit.legacy_generate.browser.danger_accept_invalid_certs); + } + + #[test] + fn audit_help_does_not_advertise_hidden_legacy_browser_flags() { + let error = + Args::try_parse_from(["ts", "audit", "--help"]).expect_err("should render audit help"); + let help = error.to_string(); + + for flag in [ + "--chrome", + "--headful", + "--no-assume-consent", + "--browser-proxy", + "--settle-quiet-ms", + "--settle-max-ms", + "--danger-accept-invalid-certs", + ] { + assert!( + !help.contains(flag), + "`{flag}` is a legacy-only alias flag and must stay hidden; got {help}" + ); + } + } + + #[test] + fn audit_rejects_parent_browser_flags_before_a_subcommand() { + // `is_err()` alone would also pass if `--chrome` were deleted from + // `LegacyBrowserOpts` (an `UnknownArgument`), which is the opposite of + // the invariant this pins: the flag exists but requires the legacy URL. + let error = Args::try_parse_from([ + "ts", + "audit", + "--chrome", + "/tmp/test-chrome", + "generate", + "https://www.example.com/", + ]) + .expect_err("a parent-level browser flag must not be silently ignored"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument, + "should reject the flag for lacking the legacy URL it requires" + ); + } + + #[test] + fn audit_page_subcommand_parses_with_page_settle_defaults() { + let args = parse(&["ts", "audit", "page", "https://www.example.com/"]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::Page(page)) = audit.command else { + panic!("expected audit page command"); + }; + assert_eq!(page.browser.settle_quiet_ms, 750); + assert_eq!(page.browser.settle_max_ms, 10_000); + } + + #[test] + fn audit_generate_subcommands_use_generation_settle_defaults() { + let args = parse(&["ts", "audit", "generate", "https://www.example.com/"]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::Generate(generate)) = audit.command + else { + panic!("expected audit generate command"); + }; + assert_eq!(generate.browser.settle_quiet_ms, 750); + assert_eq!(generate.browser.settle_max_ms, 12_000); + + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command + else { + panic!("expected audit ad-templates generate command"); + }; + assert_eq!(generate.browser.settle_quiet_ms, 750); + assert_eq!(generate.browser.settle_max_ms, 12_000); + assert!(!generate.scroll, "generation should not scroll by default"); + } + + #[test] + fn audit_ad_templates_generate_parses_scroll() { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--scroll", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command + else { + panic!("expected audit ad-templates generate command"); + }; + + assert!( + generate.scroll, + "--scroll should enable generation scrolling" + ); + } + + #[test] + fn audit_ad_templates_verify_parses() { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "verify", + "https://www.example.com/", + ]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_browser_options_are_shared_by_generate_and_verify() { + for mode in ["generate", "verify"] { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + mode, + "https://www.example.com/", + "--chrome", + "/tmp/test-chrome", + "--headful", + "--browser-proxy", + "127.0.0.1:8080", + "--no-assume-consent", + "--settle-quiet-ms", + "100", + "--settle-max-ms", + "200", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let (chrome, headful, no_assume_consent, browser_proxy, validation) = + match audit.command.expect("should parse audit subcommand") { + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Verify(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + _ => panic!("expected ad-template mode"), + }; + assert_eq!(chrome, Some(PathBuf::from("/tmp/test-chrome"))); + assert!(headful); + assert!(no_assume_consent); + assert_eq!(browser_proxy.as_deref(), Some("127.0.0.1:8080")); + validation.expect("should validate settle bounds"); + } + } + + #[test] + fn audit_generate_does_not_expose_the_ignored_browser_profile_flag() { + assert!( + Args::try_parse_from([ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--browser-profile", + "mobile", + ]) + .is_err(), + "generation device selection must use --profiles" + ); + } + + #[test] + fn browser_settle_quiet_cannot_exceed_maximum() { + let args = parse(&[ + "ts", + "audit", + "page", + "https://www.example.com/", + "--settle-quiet-ms", + "201", + "--settle-max-ms", + "200", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let crate::commands::audit::AuditSubcommand::Page(page) = + audit.command.expect("should parse page subcommand") + else { + panic!("expected page audit"); + }; + assert!(page.browser.validate().is_err()); + } + + #[test] + fn audit_ad_templates_without_verify_is_error() { + assert!(Args::try_parse_from(["ts", "audit", "ad-templates"]).is_err()); + } + + #[test] + fn audit_rejects_non_http_url() { + assert!(Args::try_parse_from(["ts", "audit", "ftp://www.example.com/"]).is_err()); + } + + #[test] + fn audit_does_not_accept_adapter_option() { + let error = Args::try_parse_from([ + "ts", + "audit", + "page", + "https://www.example.com/", + "--adapter", + "fastly", + ]) + .expect_err("should reject audit adapter option"); + assert!(error.to_string().contains("unexpected argument")); + } + #[test] fn prebid_bundle_defaults_match_spec() { let args = parse(&["ts", "prebid", "bundle"]); diff --git a/crates/trusted-server-core/examples/local_dev_config.rs b/crates/trusted-server-core/examples/local_dev_config.rs new file mode 100644 index 000000000..acbb61b6c --- /dev/null +++ b/crates/trusted-server-core/examples/local_dev_config.rs @@ -0,0 +1,129 @@ +//! Generate a ready-to-use local dev config envelope for the Axum adapter. +//! +//! Reads `trusted-server.example.toml`, replaces the placeholder secrets with +//! random values, flips the flags a local smoke test needs, validates the +//! result through [`trusted_server_core::settings::Settings::from_toml`], and +//! prints the blob envelope JSON that the Axum adapter's +//! `TRUSTED_SERVER_CONFIG_{STORE}_{KEY}` environment variable expects. With +//! the default store and key both named `trusted_server_config`, the +//! concrete variable resolves (not a typo) to +//! `TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG`. +//! +//! The random values are time-and-pid seeded, not cryptographic. This tool +//! exists for throwaway local test instances only; never use its output for a +//! deployed service. +//! +//! Usage: +//! +//! ```text +//! cargo run -p trusted-server-core --example local_dev_config \ +//! --target -- [origin-url] [--realistic] +//! ``` +//! +//! `origin-url` defaults to `https://www.example.com`. By default every +//! response is forced `Cache-Control: private, no-store` so the Server-Timing +//! header is visible on all routes; pass `--realistic` to keep the origin's +//! own cache policy instead. + +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Deliberately non-cryptographic generator for local placeholder secrets. +struct WeakRandom(u64); + +impl WeakRandom { + fn from_environment() -> Self { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .subsec_nanos() as u64; + let secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .as_secs(); + let pid = std::process::id() as u64; + Self(nanos ^ (secs << 20) ^ (pid << 40) ^ 0x9e37_79b9_7f4a_7c15) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn hex(&mut self, chars: usize) -> String { + let mut out = String::with_capacity(chars); + while out.len() < chars { + out.push_str(&format!("{:016x}", self.next())); + } + out.truncate(chars); + out + } +} + +#[allow(clippy::print_stdout, clippy::print_stderr)] +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let realistic = args.iter().any(|a| a == "--realistic"); + let origin = args + .iter() + .find(|a| !a.starts_with("--")) + .cloned() + .unwrap_or_else(|| "https://www.example.com".to_string()); + + let template = std::fs::read_to_string("trusted-server.example.toml") + .expect("should read trusted-server.example.toml from the repo root"); + + let mut random = WeakRandom::from_environment(); + let mut config = template + .replace( + "password = \"replace-with-admin-password-32-bytes\"", + &format!("password = \"{}\"", random.hex(48)), + ) + .replace( + "proxy_secret = \"change-me-proxy-secret\"", + &format!("proxy_secret = \"{}\"", random.hex(48)), + ) + .replace( + "passphrase = \"trusted-server-placeholder-secret\"", + &format!("passphrase = \"{}\"", random.hex(48)), + ) + .replace( + "server_timing_enabled = false", + "server_timing_enabled = true", + ); + + let origin_line = config + .lines() + .find(|line| line.starts_with("origin_url = ")) + .expect("should find the origin_url line in the template") + .to_string(); + config = config.replace(&origin_line, &format!("origin_url = \"{origin}\"")); + + if !realistic { + config = config.replace( + "# [response_headers]", + "[response_headers]\n\"Cache-Control\" = \"private, no-store\"", + ); + } + + let settings = trusted_server_core::settings::Settings::from_toml(&config) + .expect("should validate the generated local config"); + let data = serde_json::to_value(&settings).expect("should serialize settings"); + let generated_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .as_secs() + .to_string(); + let envelope = edgezero_core::blob_envelope::BlobEnvelope::new(data, generated_at); + println!( + "{}", + serde_json::to_string(&envelope).expect("should serialize the envelope") + ); + eprintln!( + "local dev envelope generated: origin={origin} force_private={}", + !realistic + ); +} diff --git a/crates/trusted-server-core/src/access_telemetry.rs b/crates/trusted-server-core/src/access_telemetry.rs new file mode 100644 index 000000000..5b1530768 --- /dev/null +++ b/crates/trusted-server-core/src/access_telemetry.rs @@ -0,0 +1,561 @@ +//! Access telemetry: route classification and the per-request access log row. +//! +//! Extends the reserved `access_logs_raw` Tinybird datasource with bounded, +//! content-free route identity (see [`RouteClass`] and +//! [`publisher_route_template`]) instead of the raw request path, which would +//! otherwise carry identifiers, search terms, and other user-generated +//! content into a 30-day dataset. See the design spec +//! `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md` +//! section 9. + +use serde_json::json; + +use crate::request_timing::{AuctionWaitPlacement, TimingSnapshot}; + +/// Normalizes an HTTP method token into the bounded set of values stored in +/// the `method` `LowCardinality` column. +/// +/// HTTP permits arbitrary extension-method tokens (`PROPFIND`, `MKCOL`, or +/// any client-supplied garbage), and the token on an inbound request is +/// entirely client controlled. Capturing one verbatim into a 30-day +/// `LowCardinality(String)` column would let a single caller inflate that +/// column's cardinality without bound and would violate this dataset's +/// bounded-dimension privacy rule (see the module doc). Every standard +/// method maps to its uppercase form; anything else maps to `"other"`. Runs +/// inside [`access_event_row`] rather than at each capture site, so every +/// row-building path is covered regardless of how `method` was populated. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::access_telemetry::normalize_method; +/// +/// assert_eq!(normalize_method("get"), "GET"); +/// assert_eq!(normalize_method("PROPFIND"), "other"); +/// assert_eq!(normalize_method(""), + "other", + "an unbounded client-controlled token must not reach the row verbatim" + ); + } + + #[test] + fn row_normalizes_method_even_when_snapshot_carries_a_raw_token() { + // The normalizer runs inside `access_event_row` so every row-building + // path is covered, regardless of what the snapshot's `method` field + // holds — a caller-controlled extension method must never leak into + // the row unnormalized. + let mut snapshot = unknown_snapshot(RouteClass::Other, "/other/*"); + snapshot.method = "PROPFIND".to_owned(); + let row = access_event_row(&snapshot, &TimingSnapshot::default(), 0); + let parsed: serde_json::Value = + serde_json::from_str(&row).expect("should serialize valid JSON"); + + assert_eq!(parsed["method"], "other"); + } +} diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index c0c0a7792..ab3585e3d 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -769,6 +769,7 @@ mod tests { "the endpoint must hand its snapshot to the request context" ); + ec_context.set_eid_sync_source(crate::ec::EidSyncSource::Auction); let mut response = http::Response::new(EdgeBody::empty()); crate::ec::finalize::ec_finalize_response( &settings, diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 571d9d484..255346ee4 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -51,10 +51,12 @@ pub struct AdRequest { /// `code` identifies the slot (e.g. `"atf_sidebar_ad"`) and becomes the /// impression ID in the outgoing `OpenRTB` request. /// -/// `bids` is optional. When absent or empty the PBS provider falls back to -/// a stored-request keyed by `code` (`imp.ext.prebid.storedrequest.id`). -/// When present, each entry's params are forwarded inline to PBS as -/// `imp.ext.prebid.bidder.`. +/// `bids` is optional. Absent or empty bids retain legacy PBS stored fallback +/// keyed by `code` (`imp.ext.prebid.storedrequest.id`). Bidder params route through +/// the server-owned auction plan. The reserved `trustedServer` entry accepts +/// `bidderParams`, `zone`, and boolean `storedRequest` inside its params. False +/// disables stored fallback, true permits it, and omission retains legacy +/// inference. Usable inline PBS params take precedence after provider overrides. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AdUnit { @@ -743,6 +745,24 @@ mod tests { .expect("should convert banner request") } + #[test] + fn tsjs_wire_stored_intent_survives_conversion_and_atomic_admission() { + for (intent, expected_inputs, malformed) in [ + (json!(true), 1, 0), + (json!(false), 0, 0), + (json!(null), 0, 1), + ] { + let body: AdRequest = serde_json::from_value(json!({ + "adUnits":[{"code":"example-slot","mediaTypes":{"banner":{"sizes":[[300,250]]}}, + "bids":[{"bidder":"trustedServer","params":{"bidderParams":{},"storedRequest":intent}}]}] + })).expect("should deserialize wire request"); + let request = convert_body_to_auction_request(&body, &make_settings()); + let routed = route_auction(request, &make_request(), &single_prebid_plan(), None); + assert_eq!(routed.inputs().len(), expected_inputs); + assert_eq!(routed.diagnostics().malformed_envelope_count(), malformed); + } + } + #[test] fn canonical_tsjs_request_without_bids_feeds_stored_request_router() { let body = AdRequest { diff --git a/crates/trusted-server-core/src/auction/openrtb.rs b/crates/trusted-server-core/src/auction/openrtb.rs index cf657a1ec..2cc11beb6 100644 --- a/crates/trusted-server-core/src/auction/openrtb.rs +++ b/crates/trusted-server-core/src/auction/openrtb.rs @@ -263,6 +263,9 @@ pub(crate) fn build_request( return Ok(OpenRtbBuildOutcome::NoImpressions); } policy.augment_request(&mut request, input, routed)?; + if request.imp.is_empty() { + return Ok(OpenRtbBuildOutcome::NoImpressions); + } finalize_request(&mut request, policy, finalization)?; Ok(OpenRtbBuildOutcome::Ready(request)) } @@ -482,35 +485,43 @@ fn apply_prebid( input.slots().len(), "should keep one impression per routed slot" ); - for (imp, slot) in request.imp.iter_mut().zip(input.slots()) { - let bidder = slot - .bidder_params() - .iter() - .filter_map(|(bidder, params)| { - let mut params = params.clone(); - plan.override_engine - .apply_routed(bidder.as_str(), slot.prebid_zone(), &mut params); - params - .as_object() - .is_some_and(|params| !params.is_empty()) - .then(|| (bidder.as_str().to_string(), params)) - }) - .collect::>(); - let mut prebid = Map::new(); - if !bidder.is_empty() { - prebid.insert("bidder".to_string(), Value::Object(bidder)); - } else if slot.has_trusted_stored_request() || !slot.bidder_params().is_empty() { - prebid.insert("storedrequest".to_string(), json!({"id": slot.slot().id})); - } - debug_assert!( - !prebid.is_empty(), - "should never route a demandless slot to prebid-server" - ); - imp.ext = Some(Map::from_iter([( - "prebid".to_string(), - Value::Object(prebid), - )])); - } + // Filter paired impressions and slots together so later demand keeps its slot ID. + request.imp = std::mem::take(&mut request.imp) + .into_iter() + .zip(input.slots()) + .filter_map(|(mut imp, slot)| { + let bidder = slot + .bidder_params() + .iter() + .filter_map(|(bidder, params)| { + let mut params = params.clone(); + plan.override_engine.apply_routed( + bidder.as_str(), + slot.prebid_zone(), + &mut params, + ); + params + .as_object() + .is_some_and(|params| !params.is_empty()) + .then(|| (bidder.as_str().to_string(), params)) + }) + .collect::>(); + let mut prebid = Map::new(); + if !bidder.is_empty() { + prebid.insert("bidder".to_string(), Value::Object(bidder)); + } else if slot.allows_stored_fallback() { + prebid.insert("storedrequest".to_string(), json!({"id": slot.slot().id})); + } + if prebid.is_empty() { + return None; + } + imp.ext = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid), + )])); + Some(imp) + }) + .collect(); let mut prebid_request = Map::new(); if plan.debug { prebid_request.insert("debug".to_string(), Value::Bool(true)); diff --git a/crates/trusted-server-core/src/auction/openrtb/tests.rs b/crates/trusted-server-core/src/auction/openrtb/tests.rs index c2d81088b..f47efa475 100644 --- a/crates/trusted-server-core/src/auction/openrtb/tests.rs +++ b/crates/trusted-server-core/src/auction/openrtb/tests.rs @@ -267,7 +267,6 @@ fn consent_matrix_preserves_pbs_standard_and_aps_policies() { fn pbs_body_consent_respects_source_and_forwarding_mode() { for (mode, source, expected) in [ ("cookies_only", ConsentSource::Cookie, false), - ("cookies_only", ConsentSource::KvStore, true), ("cookies_only", ConsentSource::PolicyDefault, true), ("openrtb_only", ConsentSource::Cookie, true), ("both", ConsentSource::Cookie, true), @@ -546,6 +545,180 @@ fn pbs_pairs_each_impression_with_its_routed_slot_params() { ); } +#[test] +fn pbs_stored_intent_is_applied_after_overrides_with_inline_first() { + for intent in [None, Some(false), Some(true)] { + for inline in [false, true] { + for fill_override in [false, true] { + let profile = if fill_override { + json!({"bid_param_overrides":{"exampleBidder":{"filled":1}}}) + } else { + json!({}) + }; + let plan = AuctionPlan::compile(config("prebid-server", profile)) + .expect("should compile plan"); + let mut request = canonical_parity_auction_request(); + let mut envelope = json!({"bidderParams":{"exampleBidder":if inline { json!({"original":1}) } else { json!({}) }}}); + if let Some(intent) = intent { + envelope["storedRequest"] = json!(intent); + } + request.slots[0].bidders = HashMap::from([("trustedServer".to_string(), envelope)]); + let inbound = Request::new(EdgeBody::empty()); + let routed = route_auction(request, &inbound, &plan, None); + let result = build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request"); + if !inline && !fill_override && intent == Some(false) { + assert!(matches!(result, OpenRtbBuildOutcome::NoImpressions)); + continue; + } + let OpenRtbBuildOutcome::Ready(request) = result else { + panic!("should retain demand") + }; + let wire = serde_json::to_value(request).expect("should serialize request"); + let prebid = &wire["imp"][0]["ext"]["prebid"]; + if inline || fill_override { + assert!(prebid.get("storedrequest").is_none()); + assert_eq!( + prebid["bidder"]["exampleBidder"].get("original"), + inline.then_some(&json!(1)) + ); + assert_eq!( + prebid["bidder"]["exampleBidder"].get("filled"), + fill_override.then_some(&json!(1)) + ); + } else { + assert_eq!(prebid["storedrequest"]["id"], "fictional-slot"); + } + } + } + } +} + +#[test] +fn pbs_filtering_keeps_slot_pairs_and_drops_demandless_trusted_routes() { + let plan = + AuctionPlan::compile(config("prebid-server", json!({}))).expect("should compile plan"); + let mut request = canonical_parity_auction_request(); + let template = request.slots[0].clone(); + request.slots = [ + ( + "drop-first", + json!({"storedRequest":false,"bidderParams":{"exampleBidder":{}}}), + ), + ( + "keep-inline", + json!({"storedRequest":false,"bidderParams":{"exampleBidder":{"id":2}}}), + ), + ( + "drop-trusted", + json!({"storedRequest":false,"bidderParams":{}}), + ), + ( + "keep-stored", + json!({"storedRequest":true,"bidderParams":{}}), + ), + ] + .into_iter() + .map(|(id, envelope)| AdSlot { + id: id.to_string(), + bidders: HashMap::from([("trustedServer".to_string(), envelope)]), + ..template.clone() + }) + .collect(); + let inbound = Request::new(EdgeBody::empty()); + let routes = crate::auction::routing::TrustedProviderRoutes::new(vec![ + vec![], + vec![], + vec![plan.providers()[0].id.clone()], + vec![], + ]); + let routed = crate::auction::routing::route_auction_with_trusted_routes( + request, &inbound, &plan, None, &routes, + ); + assert_eq!( + routed.inputs()[0].slots().len(), + 4, + "should preserve server-owned route admission" + ); + let result = build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request"); + let OpenRtbBuildOutcome::Ready(request) = result else { + panic!("should retain siblings") + }; + let wire = serde_json::to_value(request).expect("should serialize request"); + assert_eq!( + wire["imp"] + .as_array() + .expect("should have impressions") + .len(), + 2 + ); + assert_eq!(wire["imp"][0]["id"], "keep-inline"); + assert_eq!( + wire["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"id":2}) + ); + assert_eq!(wire["imp"][1]["id"], "keep-stored"); + assert_eq!( + wire["imp"][1]["ext"]["prebid"]["storedrequest"]["id"], + "keep-stored" + ); +} + +#[test] +fn pbs_disabled_empty_candidate_does_not_become_stored_demand() { + let mut raw = config("prebid-server", json!({})); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let mut request = canonical_parity_auction_request(); + request.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"bidderParams":{"exampleBidder":{}}, "storedRequest": false}), + )]); + let inbound = Request::builder() + .uri("https://publisher.example.com/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + assert_eq!( + routed.inputs().len(), + 1, + "should route candidate for overrides" + ); + assert!(matches!( + build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request"), + OpenRtbBuildOutcome::NoImpressions + )); +} + #[test] fn pbs_empty_params_without_matching_override_fall_back_to_stored_request() { let mut raw = config("prebid-server", json!({})); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 954869e38..c53b3c4d3 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -4171,7 +4171,8 @@ mod tests { .expect("current adapters should accept completed late mediator responses"); assert!( current_mediator.response_time_ms >= 50, - "mediator timing should preserve actual elapsed duration" + "mediator timing should preserve actual elapsed duration, got {} ms", + current_mediator.response_time_ms ); assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); @@ -5919,6 +5920,138 @@ mod tests { ); } + #[tokio::test] + async fn planned_prebid_stored_intent_filters_wire_demand_and_skips_empty_transports() { + for inline_providers in 0..=2 { + let http = Arc::new(StubHttpClient::new()); + // Providers launch in ID order: APS, then only PBS instances with usable demand. + http.push_response(204, Vec::new()); + for index in 0..inline_providers { + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid":[{"seat":"example-seat","bid":[{ + "id":format!("bid-{index}"), "impid":format!("inline-{index}"), + "price":2.0, "adm":"
example
", "w":300, "h":250 + }]}] + })) + .expect("should serialize PBS response"), + ); + } + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = planned_prebid_config(&[ + ( + "pbs-a", + serde_json::json!({}), + NotificationConfig::default(), + ), + ( + "pbs-b", + serde_json::json!({}), + NotificationConfig::default(), + ), + ]); + config.providers.extend(planned_aps_config().providers); + for (bidder, provider) in [("alpha", "pbs-a"), ("beta", "pbs-b")] { + config.bidders.insert( + bidder.parse().expect("should parse bidder"), + crate::auction::plan::BidderRouteConfig { + provider: provider.parse().expect("should parse provider"), + }, + ); + } + let plan = AuctionPlan::compile(config).expect("should compile plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let mut request = planned_request(); + let template = request.slots[0].clone(); + // Both PBS instances must evaluate candidates but omit them after overrides. + request.slots[0].bidders.insert( + "trustedServer".to_string(), + serde_json::json!({ + "storedRequest":false, "bidderParams":{"alpha":{},"beta":{}} + }), + ); + request.slots.push(AdSlot { + id: "synthetic-no-pbs".to_string(), + bidders: HashMap::from([( + "trustedServer".to_string(), + serde_json::json!({"storedRequest":false,"bidderParams":{}}), + )]), + ..template.clone() + }); + for index in 0..inline_providers { + let bidder = if index == 0 { "alpha" } else { "beta" }; + request.slots.push(AdSlot { + id:format!("inline-{index}"), + bidders:HashMap::from([("trustedServer".to_string(),serde_json::json!({"storedRequest":false,"bidderParams":{bidder:{"placement":index}}}))]), + ..template.clone() + }); + } + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://example.com/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should run auction"); + let bodies = http.recorded_request_bodies(); + assert_eq!( + bodies.len(), + 1 + inline_providers, + "should never transport an empty PBS request" + ); + let aps: serde_json::Value = + serde_json::from_slice(&bodies[0]).expect("should parse APS wire request"); + assert_eq!( + aps["imp"] + .as_array() + .expect("should have impressions") + .len(), + 2 + inline_providers + ); + assert_eq!(result.winning_bids.len(), inline_providers); + for index in 0..inline_providers { + let wire: serde_json::Value = serde_json::from_slice(&bodies[index + 1]) + .expect("should parse PBS wire request"); + assert_eq!( + wire["imp"] + .as_array() + .expect("should have impressions") + .len(), + 1 + ); + assert_eq!(wire["imp"][0]["id"], format!("inline-{index}")); + let prebid = &wire["imp"][0]["ext"]["prebid"]; + assert!(prebid.get("storedrequest").is_none()); + let bidder = if index == 0 { "alpha" } else { "beta" }; + assert_eq!( + prebid["bidder"], + serde_json::json!({bidder:{"placement":index}}) + ); + assert_eq!( + result.winning_bids[&format!("inline-{index}")] + .bid_id + .as_deref(), + Some(format!("bid-{index}").as_str()) + ); + } + } + } + #[tokio::test] async fn planned_prebid_instances_preserve_headers_metadata_suppression_and_identity() { let http = Arc::new(StubHttpClient::new()); diff --git a/crates/trusted-server-core/src/auction/routing.rs b/crates/trusted-server-core/src/auction/routing.rs index 61b2302cc..1b88c7c7a 100644 --- a/crates/trusted-server-core/src/auction/routing.rs +++ b/crates/trusted-server-core/src/auction/routing.rs @@ -13,6 +13,7 @@ use super::types::{AdSlot, AuctionRequest, MediaType}; const TRUSTED_SERVER_ENVELOPE: &str = "trustedServer"; const BIDDER_PARAMS_FIELD: &str = "bidderParams"; const ZONE_FIELD: &str = "zone"; +const STORED_REQUEST_FIELD: &str = "storedRequest"; /// Maximum bidder entries admitted from one browser `bidderParams` envelope. pub(crate) const MAX_BIDDER_ENTRIES: usize = 128; @@ -161,7 +162,7 @@ pub(crate) struct ProviderSlotInput { slot: AdSlot, bidder_params: BTreeMap, prebid_zone: Option, - trusted_stored_request: bool, + stored_request: StoredRequestIntent, } impl ProviderSlotInput { @@ -178,8 +179,14 @@ impl ProviderSlotInput { self.prebid_zone.as_deref() } + #[cfg(test)] pub(crate) fn has_trusted_stored_request(&self) -> bool { - self.trusted_stored_request + self.bidder_params.is_empty() && self.allows_stored_fallback() + } + + pub(crate) fn allows_stored_fallback(&self) -> bool { + self.stored_request + .allows_fallback(!self.bidder_params.is_empty()) } } @@ -244,10 +251,31 @@ impl TrustedProviderRoutes { } } +/// Stored fallback permission, retaining the original legacy admission shape. +#[derive(Debug, Clone, Copy, Default)] +enum StoredRequestIntent { + #[default] + Disabled, + Explicit, + Legacy { + empty_admission: bool, + }, +} + +impl StoredRequestIntent { + fn allows_fallback(self, has_candidates: bool) -> bool { + match self { + Self::Disabled => false, + Self::Explicit => true, + Self::Legacy { empty_admission } => empty_admission || has_candidates, + } + } +} + #[derive(Debug, Default)] struct NormalizedSlotDemand { bidder_params: BTreeMap, - stored_request: bool, + stored_request: StoredRequestIntent, prebid_zone: Option, } @@ -345,12 +373,13 @@ pub(crate) fn route_auction_with_trusted_routes( for (provider_index, builder) in builders.iter_mut().enumerate() { let bidder_params = std::mem::take(&mut routed_params[provider_index]); let trusted_route = trusted_provider_indices.contains(&provider_index); - let trusted_stored_request = - builder.is_prebid && demand.stored_request && bidder_params.is_empty(); - let include = builder.routing == RoutingMode::AllEligible - || !bidder_params.is_empty() - || trusted_stored_request - || trusted_route; + let include = !bidder_params.is_empty() + || trusted_route + || if builder.is_prebid { + demand.stored_request.allows_fallback(false) + } else { + builder.routing == RoutingMode::AllEligible + }; if !include { continue; } @@ -361,7 +390,7 @@ pub(crate) fn route_auction_with_trusted_routes( .is_prebid .then(|| demand.prebid_zone.clone()) .flatten(), - trusted_stored_request, + stored_request: demand.stored_request, }); } } @@ -422,16 +451,26 @@ fn normalize_slot_demand( ) -> NormalizedSlotDemand { if bidders.is_empty() { return NormalizedSlotDemand { - stored_request: true, + stored_request: StoredRequestIntent::Legacy { + empty_admission: true, + }, ..Default::default() }; } - let mut demand = NormalizedSlotDemand::default(); + let mut demand = NormalizedSlotDemand { + stored_request: StoredRequestIntent::Legacy { + empty_admission: false, + }, + ..Default::default() + }; if let Some(envelope) = bidders.get(TRUSTED_SERVER_ENVELOPE) { match normalize_envelope(envelope) { Some(normalized) => demand = normalized, - None => diagnostics.record_malformed_envelope(), + None => { + diagnostics.record_malformed_envelope(); + demand = NormalizedSlotDemand::default(); + } } } @@ -456,10 +495,12 @@ fn normalize_slot_demand( fn normalize_envelope(envelope: &Value) -> Option { let object = envelope.as_object()?; - if object - .keys() - .any(|key| !matches!(key.as_str(), BIDDER_PARAMS_FIELD | ZONE_FIELD)) - { + if object.keys().any(|key| { + !matches!( + key.as_str(), + BIDDER_PARAMS_FIELD | ZONE_FIELD | STORED_REQUEST_FIELD + ) + }) { return None; } let prebid_zone = match object.get(ZONE_FIELD) { @@ -467,28 +508,25 @@ fn normalize_envelope(envelope: &Value) -> Option { Some(Value::String(zone)) if zone.len() <= MAX_PREBID_ZONE_BYTES => Some(zone.clone()), Some(_) => return None, }; - let Some(raw_params) = object.get(BIDDER_PARAMS_FIELD) else { - return Some(NormalizedSlotDemand { - stored_request: true, - prebid_zone, - ..Default::default() - }); + let params = match object.get(BIDDER_PARAMS_FIELD) { + None | Some(Value::Null) => None, + Some(value) => Some(value.as_object()?), }; - if raw_params.is_null() { - return Some(NormalizedSlotDemand { - stored_request: true, - prebid_zone, - ..Default::default() - }); - } - let params = raw_params.as_object()?; - if params.is_empty() { + let stored_request = match object.get(STORED_REQUEST_FIELD) { + None => StoredRequestIntent::Legacy { + empty_admission: params.is_none_or(serde_json::Map::is_empty), + }, + Some(Value::Bool(false)) => StoredRequestIntent::Disabled, + Some(Value::Bool(true)) => StoredRequestIntent::Explicit, + Some(_) => return None, + }; + let Some(params) = params else { return Some(NormalizedSlotDemand { - stored_request: true, + stored_request, prebid_zone, ..Default::default() }); - } + }; if params.len() > MAX_BIDDER_ENTRIES { return None; } @@ -503,7 +541,7 @@ fn normalize_envelope(envelope: &Value) -> Option { } Some(NormalizedSlotDemand { bidder_params, - stored_request: false, + stored_request, prebid_zone, }) } @@ -673,6 +711,123 @@ mod tests { .expect("should find provider input") } + #[test] + fn pbs_admission_respects_intent_without_changing_aps_eligibility() { + for (params, pbs_ids) in [ + (json!({"bidderParams":{}, "storedRequest":false}), vec![]), + ( + json!({"bidderParams":{"unknown":{"id":1}}, "storedRequest":false}), + vec![], + ), + (json!({"bidderParams":{"unknown":{"id":1}}}), vec![]), + ( + json!({"bidderParams":{"alpha":{}}, "storedRequest":false}), + vec!["pbs-a"], + ), + (json!({"bidderParams":{"alpha":{}}}), vec!["pbs-a"]), + ( + json!({"bidderParams":{"alpha":{"id":1}}, "storedRequest":true}), + vec!["pbs-a", "pbs-b"], + ), + ( + json!({"bidderParams":{}, "storedRequest":true}), + vec!["pbs-a", "pbs-b"], + ), + (json!({"bidderParams":{}}), vec!["pbs-a", "pbs-b"]), + ] { + let routed = route_auction( + request(vec![slot(HashMap::from([( + "trustedServer".to_string(), + params, + )]))]), + &inbound(), + &plan(), + None, + ); + let mut expected = vec!["aps-primary"]; + expected.extend(pbs_ids); + assert_eq!( + routed + .inputs() + .iter() + .map(|input| input.provider_id().as_str()) + .collect::>(), + expected + ); + assert_eq!(routed.diagnostics().malformed_envelope_count(), 0); + } + } + + #[test] + fn malformed_stored_intent_rejects_envelope_atomically_but_preserves_direct_demand() { + for intent in [Value::Null, json!("false"), json!(0), json!([]), json!({})] { + for params in [ + None, + Some(Value::Null), + Some(json!({})), + Some(json!({"alpha":{"id":1}})), + ] { + for direct in [false, true] { + let mut value = json!({"storedRequest":intent, "zone":"example-zone"}); + if let Some(params) = params.clone() { + value["bidderParams"] = params; + } + let mut bidders = HashMap::from([("trustedServer".to_string(), value)]); + if direct { + bidders.insert("alpha".to_string(), json!({"direct":1})); + } + let routed = + route_auction(request(vec![slot(bidders)]), &inbound(), &plan(), None); + assert_eq!(routed.diagnostics().malformed_envelope_count(), 1); + assert_eq!(routed.inputs().len(), if direct { 2 } else { 1 }); + assert_eq!(routed.inputs()[0].provider_id().as_str(), "aps-primary"); + if direct { + let slot = &input(&routed, "pbs-a").slots()[0]; + assert!(!slot.allows_stored_fallback()); + assert_eq!(slot.prebid_zone(), None); + assert_eq!( + slot.bidder_params().values().collect::>(), + vec![&json!({"direct":1})] + ); + } + } + } + } + } + + #[test] + fn explicit_stored_intent_and_disabled_inline_are_admitted() { + for (params, expected) in [ + ( + json!({"bidderParams": {}, "storedRequest": true}), + vec!["aps-primary", "pbs-a", "pbs-b"], + ), + ( + json!({"bidderParams": {"alpha": {"placement": 1}}, "storedRequest": false}), + vec!["aps-primary", "pbs-a"], + ), + ] { + let routed = route_auction( + request(vec![slot(HashMap::from([( + "trustedServer".to_string(), + params, + )]))]), + &inbound(), + &plan(), + None, + ); + assert_eq!( + routed + .inputs() + .iter() + .map(|input| input.provider_id().as_str()) + .collect::>(), + expected + ); + assert_eq!(routed.diagnostics().malformed_envelope_count(), 0); + } + } + #[test] fn missing_null_and_empty_envelope_params_fan_out_stored_routes() { let cases = [ diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 6be0c9cfa..80d3f989e 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -182,6 +182,10 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { vec![optional_object("tinybird"), object("auction_token_secret")], true, ), + field( + vec![optional_object("tinybird"), object("access_token_secret")], + true, + ), field( vec![ optional_object("integrations"), @@ -421,7 +425,7 @@ fn validate_secret_key_references(settings: &Settings) -> Result<(), Report Result<(), Report("datadome")? { if datadome.enable_protection { @@ -765,6 +777,7 @@ formats = [{ width = 300, height = 250 }] ("handlers[*].password".to_owned(), false), ("trusted_client_ip.shared_secret".to_owned(), false), ("tinybird.auction_token_secret".to_owned(), true), + ("tinybird.access_token_secret".to_owned(), true), ( "integrations.datadome.server_side_key_secret_name".to_owned(), true, @@ -1187,10 +1200,14 @@ password = "production-admin-password-32-bytes" ); } - /// Integrations that default to disabled do not validate inactive fields. + /// `enabled` defaults to `false`, so a section that omits the flag resolves + /// to disabled and must not have its fields validated. #[test] fn deploy_validation_skips_field_validation_for_integrations_with_omitted_enabled() { let mut settings = valid_settings(); + // `endpoint` parses as a plain string but would fail the `url` + // validator, so this section only survives if validation is skipped for + // integrations that resolve to disabled. settings .integrations .insert_config( diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index 497d48b3e..ebe8f0f70 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -61,16 +61,30 @@ pub fn settings_from_config_blob( } fn remove_inactive_secret_references(data: &mut serde_json::Value) { - if data - .pointer("/tinybird/enabled") - .and_then(serde_json::Value::as_bool) - != Some(true) - && let Some(tinybird) = data - .get_mut("tinybird") - .and_then(serde_json::Value::as_object_mut) + if let Some(tinybird) = data + .get_mut("tinybird") + .and_then(serde_json::Value::as_object_mut) { - tinybird.remove("auction_token_secret"); - tinybird.remove("access_token_secret"); + let enabled = tinybird.get("enabled").and_then(serde_json::Value::as_bool) == Some(true); + if !enabled { + tinybird.remove("auction_token_secret"); + tinybird.remove("access_token_secret"); + } else { + if tinybird + .get("auction_enabled") + .and_then(serde_json::Value::as_bool) + == Some(false) + { + tinybird.remove("auction_token_secret"); + } + if tinybird + .get("access_enabled") + .and_then(serde_json::Value::as_bool) + != Some(true) + { + tinybird.remove("access_token_secret"); + } + } } if let Some(partners) = data @@ -127,7 +141,10 @@ fn json_bool_or_string_is_true(value: Option<&serde_json::Value>) -> bool { #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; + use crate::integrations::IntegrationRegistry; use crate::integrations::didomi::DidomiIntegrationConfig; use crate::platform::{PlatformError, StoreId}; use crate::redacted::Redacted; @@ -834,9 +851,15 @@ mod tests { #[test] fn runtime_blob_accepts_disabled_browser_bidder_ownership_overlap() { let original = settings_with_browser_bidder_overlap(false); + let reconstructed = load_settings(&envelope_json(&original)) + .expect("should decode dormant conflicting runtime blob"); + let plan = Arc::new( + crate::auction::compile_auction_plan(&reconstructed) + .expect("should compile decoded disabled auction plan"), + ); - load_settings(&envelope_json(&original)) - .expect("runtime should accept disabled browser bidder ownership overlap"); + IntegrationRegistry::with_plan(&reconstructed, plan) + .expect("runtime registry should accept disabled ownership overlap"); } #[test] diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index f205a8363..ba77366ca 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -8,10 +8,8 @@ //! auction pipeline and populates `OpenRTB` bid requests. //! //! Consent is interpreted from request cookies, headers, geolocation, and -//! publisher policy defaults. When the caller supplies an EC ID and a KV -//! store via [`ConsentPipelineInput`], the pipeline also loads persisted -//! consent as a fallback and persists cookie-sourced consent on change. EC -//! identity lifecycle state is managed separately by the EC identity graph. +//! publisher policy defaults. EC identity lifecycle and withdrawal state are +//! managed separately by the EC identity graph. //! //! # Supported signals //! @@ -38,7 +36,6 @@ pub mod tcf; pub mod types; pub mod us_privacy; -pub use crate::storage::kv_store as kv; pub use extraction::extract_consent_signals; pub use types::{ ConsentContext, ConsentSource, PrivacyFlag, RawConsentSignals, TcfConsent, UsPrivacy, @@ -76,17 +73,6 @@ pub struct ConsentPipelineInput<'a> { pub config: &'a ConsentConfig, /// Geolocation data from the request (for jurisdiction detection). pub geo: Option<&'a GeoInfo>, - /// EC ID for KV Store consent persistence. - /// - /// When set along with `kv_store`, enables: - /// - **Read fallback**: loads consent from KV when cookies are absent. - /// - **Write-on-change**: persists cookie-sourced consent to KV. - pub ec_id: Option<&'a str>, - /// KV store for consent persistence. - /// - /// `None` when consent persistence is not configured for this request, or - /// when the caller intentionally skips consent KV access. - pub kv_store: Option<&'a dyn crate::platform::PlatformKvStore>, } /// Extracts, decodes, and normalizes consent signals from a request. @@ -101,16 +87,8 @@ pub struct ConsentPipelineInput<'a> { /// 6. Builds a [`ConsentContext`] with both raw and decoded data. /// 7. Logs a summary for observability. /// -/// When [`ConsentPipelineInput::ec_id`] and [`ConsentPipelineInput::kv_store`] -/// are both set, the pipeline also: -/// -/// - **Read fallback**: loads consent persisted in KV for the EC ID when the -/// request carries no consent signals. -/// - **Write-on-change**: persists cookie-sourced consent to KV after the -/// context is built (skipping empty contexts and unchanged fingerprints). -/// -/// Without those inputs the returned context reflects request-local consent -/// signals plus policy defaults only. +/// The returned context reflects request-local consent signals plus policy +/// defaults only. /// /// Decoding failures are logged and the corresponding decoded field is set to /// `None` — the raw string is still preserved for proxy-mode forwarding. @@ -122,19 +100,6 @@ pub fn build_consent_context(input: &ConsentPipelineInput<'_>) -> ConsentContext log_missing_geo_warning_once(); } - // Read fallback: when the request carries no consent signals, fall back - // to consent persisted in KV for this EC ID (when persistence is wired). - if signals.is_empty() - && let (Some(ec_id), Some(store)) = (input.ec_id, input.kv_store) - && let Some(mut ctx) = kv::load_consent_from_kv(store, ec_id) - { - // Jurisdiction is request-local: derive it from the current - // geo rather than the value stored with the persisted entry. - ctx.jurisdiction = jurisdiction::detect_jurisdiction(input.geo, input.config); - log_consent_context(&ctx); - return ctx; - } - // In proxy mode, skip decoding entirely. if input.config.mode == ConsentMode::Proxy { let jur = jurisdiction::detect_jurisdiction(input.geo, input.config); @@ -168,13 +133,6 @@ pub fn build_consent_context(input: &ConsentPipelineInput<'_>) -> ConsentContext apply_expiration_check(&mut ctx, input.config); apply_gpc_us_privacy(&mut ctx, input.config); - // Write-on-change: persist cookie-sourced consent for this EC ID (when - // persistence is wired). The helper skips empty contexts and unchanged - // fingerprints internally. - if let (Some(ec_id), Some(store)) = (input.ec_id, input.kv_store) { - kv::save_consent_to_kv(store, ec_id, &ctx, input.config.max_consent_age_days); - } - log_consent_context(&ctx); ctx } @@ -703,7 +661,7 @@ mod tests { use super::{ ConsentPipelineInput, allows_ec_creation, apply_expiration_check, apply_tcf_conflict_resolution, build_consent_context, build_context_from_signals, - consent_allows_server_side_auction, has_explicit_ec_withdrawal, + consent_allows_server_side_auction, gate_eids_by_consent, has_explicit_ec_withdrawal, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -903,8 +861,6 @@ mod tests { req: &req, config: &config, geo: None, - ec_id: None, - kv_store: None, }); assert_eq!( @@ -932,8 +888,6 @@ mod tests { req: &req, config: &config, geo: None, - ec_id: None, - kv_store: None, }); assert!( @@ -962,8 +916,6 @@ mod tests { req: &req, config: &config, geo: None, - ec_id: None, - kv_store: None, }); assert!( @@ -1080,6 +1032,36 @@ mod tests { TcfBuilder::new().with_storage(has_storage).build() } + #[test] + fn gate_eids_by_consent_strips_every_eid_when_personalization_is_denied() { + let context = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some( + TcfBuilder::new() + .with_storage(true) + .with_personalized_ads(false) + .build(), + ), + ..ConsentContext::default() + }; + + // Gating is all-or-nothing across sources; LiveRamp is included here as + // the case that motivated this coverage, not as a special case. + let gated = gate_eids_by_consent( + Some(vec![ + ("liveramp.com", "opaque-test-envelope"), + ("sharedid.org", "shared-test-id"), + ]), + Some(&context), + ); + + assert!( + gated.is_none(), + "should remove every EID when personalization consent is denied" + ); + } + #[test] fn ec_allowed_gdpr_with_storage_consent() { let ctx = ConsentContext { @@ -1472,161 +1454,4 @@ mod tests { "GPP without US section should fall through to us_privacy" ); } - - // ----------------------------------------------------------------------- - // Consent KV read-fallback / write-on-change pipeline tests - // ----------------------------------------------------------------------- - - struct InMemoryKvStore { - entries: std::sync::Mutex>, - } - - impl InMemoryKvStore { - fn new() -> Self { - Self { - entries: std::sync::Mutex::new(std::collections::HashMap::new()), - } - } - } - - #[async_trait::async_trait(?Send)] - impl crate::platform::PlatformKvStore for InMemoryKvStore { - async fn get_bytes( - &self, - key: &str, - ) -> Result, crate::platform::KvError> { - Ok(self - .entries - .lock() - .expect("should lock entries") - .get(key) - .cloned()) - } - - async fn put_bytes( - &self, - key: &str, - value: bytes::Bytes, - ) -> Result<(), crate::platform::KvError> { - self.entries - .lock() - .expect("should lock entries") - .insert(key.to_owned(), value); - Ok(()) - } - - async fn put_bytes_with_ttl( - &self, - key: &str, - value: bytes::Bytes, - _ttl: std::time::Duration, - ) -> Result<(), crate::platform::KvError> { - self.put_bytes(key, value).await - } - - async fn delete(&self, key: &str) -> Result<(), crate::platform::KvError> { - self.entries - .lock() - .expect("should lock entries") - .remove(key); - Ok(()) - } - - async fn list_keys_page( - &self, - _prefix: &str, - _cursor: Option<&str>, - _limit: usize, - ) -> Result { - Ok(edgezero_core::key_value_store::KvPage::default()) - } - } - - #[test] - fn pipeline_persists_cookie_sourced_consent_to_kv() { - let jar = parse_cookies_to_jar("us_privacy=1YNN"); - let req = build_request(); - let config = ConsentConfig::default(); - let store = InMemoryKvStore::new(); - - let ctx = build_consent_context(&ConsentPipelineInput { - jar: Some(&jar), - req: &req, - config: &config, - geo: None, - ec_id: Some("test-ec-id"), - kv_store: Some(&store), - }); - - assert_eq!( - ctx.raw_us_privacy.as_deref(), - Some("1YNN"), - "should build cookie-sourced consent" - ); - let persisted = crate::consent::kv::load_consent_from_kv(&store, "test-ec-id") - .expect("should persist cookie-sourced consent to KV"); - assert_eq!( - persisted.raw_us_privacy.as_deref(), - Some("1YNN"), - "persisted consent should round-trip the cookie signal" - ); - } - - #[test] - fn pipeline_falls_back_to_kv_consent_when_request_has_no_signals() { - let config = ConsentConfig::default(); - let store = InMemoryKvStore::new(); - - // First request carries a consent cookie — persisted to KV. - let jar = parse_cookies_to_jar("us_privacy=1YNN"); - let req = build_request(); - build_consent_context(&ConsentPipelineInput { - jar: Some(&jar), - req: &req, - config: &config, - geo: None, - ec_id: Some("test-ec-id"), - kv_store: Some(&store), - }); - - // Second request has no consent signals — must fall back to KV. - let bare_req = build_request(); - let ctx = build_consent_context(&ConsentPipelineInput { - jar: None, - req: &bare_req, - config: &config, - geo: None, - ec_id: Some("test-ec-id"), - kv_store: Some(&store), - }); - - assert_eq!( - ctx.raw_us_privacy.as_deref(), - Some("1YNN"), - "should load persisted consent when the request carries no signals" - ); - } - - #[test] - fn pipeline_skips_kv_when_persistence_not_wired() { - let jar = parse_cookies_to_jar("us_privacy=1YNN"); - let req = build_request(); - let config = ConsentConfig::default(); - let store = InMemoryKvStore::new(); - - // ec_id is absent, so the pipeline must not touch the KV store. - build_consent_context(&ConsentPipelineInput { - jar: Some(&jar), - req: &req, - config: &config, - geo: None, - ec_id: None, - kv_store: Some(&store), - }); - - assert!( - crate::consent::kv::load_consent_from_kv(&store, "test-ec-id").is_none(), - "should not persist consent without an EC ID" - ); - } } diff --git a/crates/trusted-server-core/src/consent/types.rs b/crates/trusted-server-core/src/consent/types.rs index 73c2bbc3f..7d1b1902a 100644 --- a/crates/trusted-server-core/src/consent/types.rs +++ b/crates/trusted-server-core/src/consent/types.rs @@ -7,7 +7,7 @@ //! - [`UsPrivacy`] / [`PrivacyFlag`] — decoded US Privacy (CCPA) 4-char string //! - [`TcfConsent`] — decoded TCF v2 core consent data //! - [`GppConsent`] — decoded GPP consent data -//! - [`ConsentSource`] — how consent was sourced (cookie, KV store, etc.) +//! - [`ConsentSource`] — how consent was sourced (cookie or policy default) use core::fmt; @@ -379,8 +379,6 @@ impl fmt::Display for UsPrivacy { pub enum ConsentSource { /// Read from cookies on the incoming request. Cookie, - /// Loaded from KV store via Edge Cookie (EC) ID lookup. - KvStore, /// Applied from explicit publisher policy defaults. PolicyDefault, /// No consent data available. diff --git a/crates/trusted-server-core/src/consent_config.rs b/crates/trusted-server-core/src/consent_config.rs index 465629a51..fbef7efe8 100644 --- a/crates/trusted-server-core/src/consent_config.rs +++ b/crates/trusted-server-core/src/consent_config.rs @@ -73,11 +73,6 @@ pub struct ConsentConfig { /// but disagree on consent status. #[serde(default)] pub conflict_resolution: ConflictResolutionConfig, - /// When set, consent data is persisted per Edge Cookie (EC) ID so that - /// returning users without consent cookies can still have their - /// consent preferences applied. Set to `None` to disable. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub consent_store: Option, } impl Default for ConsentConfig { @@ -90,7 +85,6 @@ impl Default for ConsentConfig { us_states: UsStatesConfig::default(), us_privacy_defaults: UsPrivacyDefaultsConfig::default(), conflict_resolution: ConflictResolutionConfig::default(), - consent_store: None, } } } diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index d444238e8..8e39f2d19 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -1,6 +1,8 @@ use http::header::HeaderName; pub const COOKIE_TS_EC: &str = "ts-ec"; +/// Short-lived signed proof that the current EC row has every pull-partner UID. +pub const COOKIE_TS_EC_PULL_COMPLETE: &str = "ts-ec-pull-complete"; /// Cookie written by the Trusted Server JS SDK containing a standard-base64-encoded /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; @@ -36,6 +38,8 @@ pub const HEADER_X_TS_ENV: HeaderName = HeaderName::from_static("x-ts-env"); // Fastly environment variables pub const ENV_FASTLY_SERVICE_VERSION: &str = "FASTLY_SERVICE_VERSION"; pub const ENV_FASTLY_IS_STAGING: &str = "FASTLY_IS_STAGING"; +pub const ENV_FASTLY_SERVICE_ID: &str = "FASTLY_SERVICE_ID"; +pub const ENV_FASTLY_POP: &str = "FASTLY_POP"; // Common standard header names used across modules pub const HEADER_USER_AGENT: HeaderName = HeaderName::from_static("user-agent"); diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 2254c27d5..4b1405359 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -173,8 +173,12 @@ fn sanitize_section(segment: &str) -> String { /// The path is used **raw** (not percent-decoded) so this stays consistent with /// how [`page_patterns`](CreativeOpportunitySlot::page_patterns) glob-match the /// same path — e.g. `/new%20s` yields `new_20s`, never the decoded `new_s`. +/// +/// Public so operator tooling that *infers* a `{section}` template from observed +/// ad-unit paths can check its inference against the exact derivation the +/// runtime will perform, rather than reimplementing the sanitization rules. #[must_use] -fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { +pub fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { match path .split('/') .filter(|segment| !segment.is_empty()) @@ -701,15 +705,7 @@ impl CreativeOpportunitySlot { // skip `compile_patterns`). Re-compiles on every call. self.page_patterns .iter() - .any(|pattern| match Pattern::new(pattern) { - Ok(p) => p.matches(path), - Err(_) => { - let normalised = pattern.replace("**", "*"); - Pattern::new(&normalised) - .map(|p| p.matches(path)) - .unwrap_or(false) - } - }) + .any(|pattern| compile_page_pattern(pattern).is_ok_and(|p| p.matches(path))) } /// Compile [`page_patterns`](Self::page_patterns) into the @@ -726,22 +722,20 @@ impl CreativeOpportunitySlot { self.compiled_patterns = self .page_patterns .iter() - .filter_map(|pattern| { - match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { - Ok(compiled) => Some(compiled), - Err(_) => { - // Build-time validation only requires *one* valid pattern - // per slot, so a mixed valid/invalid set passes the build - // with the bad pattern silently dropped here. Warn so the - // operator can see the slot matches fewer pages than - // configured. - log::warn!( - "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", - self.id, - pattern - ); - None - } + .filter_map(|pattern| match compile_page_pattern(pattern) { + Ok(compiled) => Some(compiled), + Err(error) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}': {error}", + self.id, + pattern + ); + None } }) .collect(); @@ -994,6 +988,48 @@ pub struct PrebidSlotParams { pub bidders: HashMap, } +/// Compiles a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This is the single definition of what the runtime accepts as a page glob: +/// a direct [`Pattern::new`], falling back to the `**`→`*` rewrite that +/// [`CreativeOpportunitySlot::compile_patterns`] and +/// [`matches_path`](CreativeOpportunitySlot::matches_path) apply. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// normalisation. +pub(crate) fn compile_page_pattern(pattern: &str) -> Result { + Pattern::new(pattern) + .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) + .map_err(|error| format!("page pattern '{pattern}' is not a valid glob: {error}")) +} + +/// Validates a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This exposes validation without leaking the runtime's `glob::Pattern` type +/// into the public API. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// the runtime's `**` to `*` normalisation. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::creative_opportunities::validate_page_pattern; +/// +/// assert!(validate_page_pattern("/news/*").is_ok()); +/// assert!(validate_page_pattern("/20**").is_ok()); +/// assert!(validate_page_pattern("[").is_err()); +/// ``` +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) +} + /// Validates that a slot ID contains only safe characters. /// /// Allowed characters: ASCII alphanumerics, underscores (`_`), and hyphens (`-`). @@ -1026,6 +1062,151 @@ pub fn match_slots<'a>( slots.iter().filter(|s| s.matches_path(path)).collect() } +/// Three-state outcome of the server-side ad-stack gate. +/// +/// [`Yes`](RuntimeAdStackExpected::Yes) and [`No`](RuntimeAdStackExpected::No) +/// are decided purely from known inputs; [`Unknown`](RuntimeAdStackExpected::Unknown) +/// is reserved for callers (such as the operator CLI) that cannot prove the live +/// consent state and pass `None` for `consent_allows_auction`. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RuntimeAdStackExpected { + /// All known gates pass and consent is known to allow the auction. + Yes, + /// At least one known gate blocks the server-side ad stack. + No, + /// All known gates pass but consent is unproven. + Unknown, +} + +/// Identifies a single gate evaluated by [`evaluate_ad_stack_gate`]. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum AdStackGateName { + /// Request method is `GET`. + MethodGet, + /// Request is a top-level navigation. + Navigation, + /// Request is not a prefetch. + NotPrefetch, + /// Request is not from a known bot. + NotBot, + /// At least one configured slot matches the request path. + MatchedSlots, + /// Consent is known to allow the auction. + ConsentAllowsAuction, + /// The global `[auction].enabled` kill switch is on. + AuctionEnabled, + /// The `[creative_opportunities].enabled` template switch is on. + AdTemplatesEnabled, +} + +impl AdStackGateName { + const ALL: [Self; 8] = [ + Self::MethodGet, + Self::Navigation, + Self::NotPrefetch, + Self::NotBot, + Self::MatchedSlots, + Self::ConsentAllowsAuction, + Self::AuctionEnabled, + Self::AdTemplatesEnabled, + ]; + + fn blocks(self, input: AdStackGateInput) -> bool { + match self { + Self::MethodGet => !input.method_get, + Self::Navigation => !input.navigation, + Self::NotPrefetch => input.prefetch, + Self::NotBot => input.bot, + Self::MatchedSlots => !input.matched_slots, + Self::ConsentAllowsAuction => input.consent_allows_auction == Some(false), + Self::AuctionEnabled => !input.auction_enabled, + Self::AdTemplatesEnabled => !input.ad_templates_enabled, + } + } +} + +/// Inputs to [`evaluate_ad_stack_gate`]. +/// +/// `consent_allows_auction` is tri-state: `Some(true)` allows, `Some(false)` +/// blocks, and `None` means the caller cannot prove the consent state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct AdStackGateInput { + /// Request method is `GET`. + pub method_get: bool, + /// Request is a top-level navigation. + pub navigation: bool, + /// Request advertises itself as a prefetch. + pub prefetch: bool, + /// Request is from a known bot. + pub bot: bool, + /// At least one configured slot matches the request path. + pub matched_slots: bool, + /// Whether consent allows the auction. + /// + /// `Some(true)` allows the auction, `Some(false)` blocks it, and `None` + /// means the caller cannot prove either state. Unknown consent is not a + /// denial: it produces [`RuntimeAdStackExpected::Unknown`] when every known + /// boolean gate passes. + pub consent_allows_auction: Option, + /// The global `[auction].enabled` kill switch. + pub auction_enabled: bool, + /// The `[creative_opportunities].enabled` template switch. + /// + /// `false` whenever creative opportunities are absent from the + /// configuration, so an unconfigured publisher blocks here as well. + pub ad_templates_enabled: bool, +} + +/// Result of [`evaluate_ad_stack_gate`]: the three-state expectation plus the +/// original inputs used to derive per-gate diagnostics on demand. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AdStackGateResult { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, + input: AdStackGateInput, +} + +impl AdStackGateResult { + /// Returns the gates that blocked the server-side ad stack. + pub fn blocking_gates(&self) -> impl Iterator + '_ { + AdStackGateName::ALL + .into_iter() + .filter(|gate| gate.blocks(self.input)) + } +} + +/// Evaluates whether the server-side ad stack should run for a request. +/// +/// Any known gate that fails sets [`No`](RuntimeAdStackExpected::No) and is +/// recorded in [`AdStackGateResult::blocking_gates`]. When no known gate blocks, +/// the result is [`Yes`](RuntimeAdStackExpected::Yes) if consent is known to +/// allow the auction, or [`Unknown`](RuntimeAdStackExpected::Unknown) when +/// `consent_allows_auction` is `None`. +/// +/// Gate polarity mirrors the runtime publisher path: `method_get`, `navigation`, +/// `matched_slots`, `auction_enabled`, and `ad_templates_enabled` block when +/// `false`; `prefetch` and `bot` block when `true`. +#[must_use] +pub fn evaluate_ad_stack_gate(input: AdStackGateInput) -> AdStackGateResult { + let known_gate_blocks = !input.method_get + || !input.navigation + || input.prefetch + || input.bot + || !input.matched_slots + || input.consent_allows_auction == Some(false) + || !input.auction_enabled + || !input.ad_templates_enabled; + let expected = if known_gate_blocks { + RuntimeAdStackExpected::No + } else if input.consent_allows_auction.is_none() { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::Yes + }; + + AdStackGateResult { expected, input } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -1041,6 +1222,159 @@ mod tests { use crate::auction::routing::route_auction; use crate::auction::types::{AuctionRequest, PublisherInfo, UserInfo}; + #[test] + fn ad_stack_gate_passes_for_eligible_navigation() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Yes); + assert_eq!(result.blocking_gates().count(), 0); + } + + #[test] + fn ad_stack_gate_blocks_known_kill_switch() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: false, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::No); + assert!( + result + .blocking_gates() + .any(|gate| gate == AdStackGateName::AuctionEnabled) + ); + } + + #[test] + fn ad_stack_gate_blocks_disabled_ad_templates() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + ad_templates_enabled: false, + }); + + assert_eq!( + result.expected, + RuntimeAdStackExpected::No, + "a disabled [creative_opportunities].enabled switch should block the ad stack" + ); + assert!( + result + .blocking_gates() + .any(|gate| gate == AdStackGateName::AdTemplatesEnabled), + "the template switch should be named as the blocking gate" + ); + } + + #[test] + fn ad_stack_gate_is_unknown_when_consent_is_unknown() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: None, + auction_enabled: true, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Unknown); + } + + // Locks the spec §5.2 mirror invariant: with Some(consent) supplied for every + // input combination, `expected == Yes` must equal the legacy all-AND boolean. + #[test] + fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { + for bits in 0u16..256 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: Some(bits & 32 != 0), + auction_enabled: bits & 64 != 0, + ad_templates_enabled: bits & 128 != 0, + }; + // Legacy semantics: all positive gates true, both negative gates false. + let legacy = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.consent_allows_auction == Some(true) + && input.auction_enabled + && input.ad_templates_enabled; + let got = evaluate_ad_stack_gate(input).expected == RuntimeAdStackExpected::Yes; + assert_eq!(got, legacy, "gate mismatch for bits={bits}"); + } + } + + #[test] + fn ad_stack_gate_with_unknown_consent_matches_known_boolean_gates() { + for bits in 0u8..128 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: None, + auction_enabled: bits & 32 != 0, + ad_templates_enabled: bits & 64 != 0, + }; + let known_gates_pass = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.auction_enabled + && input.ad_templates_enabled; + let expected = if known_gates_pass { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::No + }; + + assert_eq!( + evaluate_ad_stack_gate(input).expected, + expected, + "should match unknown-consent gate semantics for bits={bits}" + ); + } + } + + #[test] + fn validate_page_pattern_preserves_specific_compile_error() { + let error = validate_page_pattern("[").expect_err("should reject invalid glob"); + + assert!( + error.contains("page pattern '[' is not a valid glob"), + "should retain the invalid pattern in the error: {error}" + ); + } + fn make_slot(id: &str, patterns: Vec<&str>) -> CreativeOpportunitySlot { CreativeOpportunitySlot { id: id.to_string(), diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 248e1dd27..0e50d548e 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -2,15 +2,18 @@ //! //! Partners send authenticated batch ID sync requests via Bearer token. //! Each mapping associates an `ec_id` (`{64hex}.{6alnum}`) -//! with the partner's user ID. Mappings are individually validated and -//! written to the KV identity graph, with per-mapping rejection reasons -//! reported in the response. +//! with the partner's user ID. Mappings are individually validated, then valid +//! mappings are grouped by normalized EC ID before one call to the KV update +//! path per group. +//! Responses still report outcomes per original mapping index. //! //! Mapping timestamps are retained in the request schema for client //! compatibility, but the EC identity graph no longer stores per-partner sync -//! timestamps. Valid mappings therefore use idempotent last-write-wins -//! semantics: unchanged UIDs are accepted without a write; different UIDs -//! replace the stored value regardless of timestamp. +//! timestamps. The last valid mapping in request order supplies each group's +//! UID; unchanged UIDs are accepted without a write, and different UIDs replace +//! the stored value regardless of timestamp. + +use std::collections::HashMap; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; @@ -88,6 +91,17 @@ struct MappingError { reason: &'static str, } +/// Valid mappings sharing one normalized EC ID. +/// +/// `indexes` stays in request order, while the groups themselves stay ordered +/// by first valid occurrence. The lookup map used to locate this structure is +/// never used for processing order. +struct MappingGroup { + ec_id: String, + partner_uid: String, + indexes: Vec, +} + // --------------------------------------------------------------------------- // Handler // --------------------------------------------------------------------------- @@ -179,19 +193,28 @@ fn content_length_exceeds_limit(req: &Request, max_body_size: usize) - .is_some_and(|content_length| content_length > max_body_size) } +/// Validates all mappings, then processes each normalized EC ID once. +/// +/// Successful and eligibility outcomes fan out to every valid input in a +/// group. On infrastructure failure, the failing group and all unprocessed +/// valid groups are rejected as unavailable; already validated invalid inputs +/// keep their specific errors. Errors are sorted by original input index. fn process_mappings( writer: &dyn BatchSyncWriter, partner_id: &str, mappings: &[SyncMapping], ) -> (usize, Vec) { - let mut accepted: usize = 0; let mut errors = Vec::new(); + let mut groups: Vec = Vec::new(); + let mut group_indexes: HashMap = HashMap::new(); - for (idx, mapping) in mappings.iter().enumerate() { + // Validate all inputs before beginning KV work. The vector preserves group + // order; the map only locates an existing group in constant time. + for (index, mapping) in mappings.iter().enumerate() { let ec_id = normalize_ec_id_for_kv(&mapping.ec_id); if !is_valid_ec_id(&ec_id) { errors.push(MappingError { - index: idx, + index, reason: REASON_INVALID_EC_ID, }); continue; @@ -199,42 +222,57 @@ fn process_mappings( if mapping.partner_uid.trim().is_empty() || mapping.partner_uid.len() > MAX_UID_LENGTH { errors.push(MappingError { - index: idx, + index, reason: REASON_INVALID_PARTNER_UID, }); continue; } - match writer.upsert_partner_id_if_exists(&ec_id, partner_id, &mapping.partner_uid) { + + if let Some(&group_index) = group_indexes.get(&ec_id) { + let group = &mut groups[group_index]; + group.partner_uid.clone_from(&mapping.partner_uid); + group.indexes.push(index); + } else { + group_indexes.insert(ec_id.clone(), groups.len()); + groups.push(MappingGroup { + ec_id, + partner_uid: mapping.partner_uid.clone(), + indexes: vec![index], + }); + } + } + + let mut accepted = 0; + for (group_index, group) in groups.iter().enumerate() { + match writer.upsert_partner_id_if_exists(&group.ec_id, partner_id, &group.partner_uid) { Ok(UpsertResult::Written | UpsertResult::Unchanged) => { - accepted += 1; + accepted += group.indexes.len(); } Ok(UpsertResult::NotFound | UpsertResult::ConsentWithdrawn) => { - errors.push(MappingError { - index: idx, + errors.extend(group.indexes.iter().map(|&index| MappingError { + index, reason: REASON_INELIGIBLE, - }); + })); } Err(err) => { log::warn!( - "Batch sync KV write failed for index {idx} (ec_id '{}'): {err:?}", - log_id(&mapping.ec_id), + "Batch sync KV write failed for group starting at index {} (ec_id '{}'): {err:?}", + group.indexes[0], + log_id(&group.ec_id), ); - errors.push(MappingError { - index: idx, - reason: REASON_KV_UNAVAILABLE, - }); - // Abort remaining mappings on infrastructure failure. - for remaining_idx in (idx + 1)..mappings.len() { - errors.push(MappingError { - index: remaining_idx, + for unavailable_group in &groups[group_index..] { + errors.extend(unavailable_group.indexes.iter().map(|&index| MappingError { + index, reason: REASON_KV_UNAVAILABLE, - }); + })); } break; } } } + errors.sort_by_key(|error| error.index); + debug_assert_eq!(accepted + errors.len(), mappings.len()); (accepted, errors) } @@ -301,29 +339,47 @@ mod tests { } } + #[derive(Clone, Debug, PartialEq, Eq)] + struct WriterCall { + ec_id: String, + partner_id: String, + uid: String, + } + struct MockWriter { results: std::cell::RefCell>>>, + calls: std::cell::RefCell>, } impl MockWriter { fn new(results: Vec>>) -> Self { Self { results: std::cell::RefCell::new(results.into()), + calls: std::cell::RefCell::new(Vec::new()), } } + + fn calls(&self) -> Vec { + self.calls.borrow().clone() + } } impl BatchSyncWriter for MockWriter { fn upsert_partner_id_if_exists( &self, - _ec_id: &str, - _partner_id: &str, - _uid: &str, + ec_id: &str, + partner_id: &str, + uid: &str, ) -> Result> { + self.calls.borrow_mut().push(WriterCall { + ec_id: ec_id.to_owned(), + partner_id: partner_id.to_owned(), + uid: uid.to_owned(), + }); self.results .borrow_mut() .pop_front() - .expect("should provide mock result for each mapping") + .expect("should provide mock result for each group") } } @@ -361,6 +417,14 @@ mod tests { .expect("should build authorized batch request") } + fn response_json(response: Response) -> serde_json::Value { + let body = response + .into_body() + .into_bytes() + .expect("should contain batch-sync response"); + serde_json::from_slice(&body).expect("should serialize batch-sync response") + } + fn test_registry() -> PartnerRegistry { let partners = vec![make_test_partner( "ssp.example.com", @@ -578,8 +642,12 @@ mod tests { Ok(UpsertResult::NotFound), Ok(UpsertResult::ConsentWithdrawn), ]); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mappings = vec![mapping(&ec_id, "uid-1", 100), mapping(&ec_id, "uid-2", 101)]; + let missing_ec_id = format!("{}.ABC123", "a".repeat(64)); + let withdrawn_ec_id = format!("{}.ABC123", "b".repeat(64)); + let mappings = vec![ + mapping(&missing_ec_id, "uid-1", 100), + mapping(&withdrawn_ec_id, "uid-2", 101), + ]; let (accepted, errors) = process_mappings(&writer, "partner", &mappings); @@ -589,26 +657,28 @@ mod tests { assert_eq!(errors[0].reason, REASON_INELIGIBLE); assert_eq!(errors[1].index, 1); assert_eq!(errors[1].reason, REASON_INELIGIBLE); + assert_eq!(writer.calls().len(), 2, "should exercise both outcomes"); } #[test] - fn process_mappings_counts_unchanged_as_accepted() { + fn process_mappings_fans_out_unchanged_to_group_members() { let writer = MockWriter::new(vec![Ok(UpsertResult::Unchanged)]); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mappings = vec![mapping(&ec_id, "uid-1", 100)]; + let mappings = vec![mapping(&ec_id, "uid-1", 100), mapping(&ec_id, "uid-1", 101)]; let (accepted, errors) = process_mappings(&writer, "partner", &mappings); - assert_eq!(accepted, 1, "should count unchanged mappings as accepted"); + assert_eq!(accepted, 2, "should accept every unchanged group member"); assert!( errors.is_empty(), "should report no errors for unchanged mappings" ); + assert_eq!(writer.calls().len(), 1, "should call once for the group"); } #[test] fn process_mappings_does_not_order_by_timestamp() { - let writer = MockWriter::new(vec![Ok(UpsertResult::Written), Ok(UpsertResult::Written)]); + let writer = MockWriter::new(vec![Ok(UpsertResult::Written)]); let ec_id = format!("{}.ABC123", "a".repeat(64)); let mappings = vec![ mapping(&ec_id, "uid-new", 200), @@ -622,5 +692,342 @@ mod tests { "timestamps are compatibility fields and should not reject older mappings" ); assert!(errors.is_empty(), "should accept valid mappings"); + assert_eq!( + writer.calls(), + vec![WriterCall { + ec_id, + partner_id: "partner".to_owned(), + uid: "uid-old".to_owned(), + }], + "should persist the last valid UID with one writer call" + ); + } + + #[test] + fn process_mappings_groups_normalized_ids_in_first_occurrence_order() { + let writer = MockWriter::new(vec![Ok(UpsertResult::Written), Ok(UpsertResult::Unchanged)]); + let ec_id_a = format!("{}.ABC123", "a".repeat(64)); + let ec_id_a_upper = format!("{}.ABC123", "A".repeat(64)); + let ec_id_b = format!("{}.ABC123", "b".repeat(64)); + let mappings = vec![ + mapping(&ec_id_a, "a-first", 1), + mapping(&ec_id_b, "b-only", 2), + mapping(&ec_id_a_upper, "a-last", 3), + ]; + + let (accepted, errors) = process_mappings(&writer, "partner", &mappings); + + assert_eq!(accepted, 3, "should accept every valid group member"); + assert!(errors.is_empty(), "should report no errors"); + assert_eq!( + writer.calls(), + vec![ + WriterCall { + ec_id: ec_id_a, + partner_id: "partner".to_owned(), + uid: "a-last".to_owned(), + }, + WriterCall { + ec_id: ec_id_b, + partner_id: "partner".to_owned(), + uid: "b-only".to_owned(), + }, + ], + "should make one ordered call per normalized EC ID" + ); + } + + #[test] + fn process_mappings_keeps_suffix_case_distinct() { + let writer = MockWriter::new(vec![Ok(UpsertResult::Written), Ok(UpsertResult::Written)]); + let upper_suffix = format!("{}.ABC123", "a".repeat(64)); + let mixed_suffix = format!("{}.AbC123", "a".repeat(64)); + let mappings = vec![ + mapping(&upper_suffix, "upper", 1), + mapping(&mixed_suffix, "mixed", 2), + ]; + + let (accepted, errors) = process_mappings(&writer, "partner", &mappings); + + assert_eq!(accepted, 2, "should accept both distinct EC IDs"); + assert!(errors.is_empty(), "should report no errors"); + assert_eq!( + writer.calls(), + vec![ + WriterCall { + ec_id: upper_suffix, + partner_id: "partner".to_owned(), + uid: "upper".to_owned(), + }, + WriterCall { + ec_id: mixed_suffix, + partner_id: "partner".to_owned(), + uid: "mixed".to_owned(), + }, + ], + "normalization must preserve suffix case" + ); + } + + #[test] + fn process_mappings_invalid_duplicate_does_not_replace_last_valid_uid() { + let writer = MockWriter::new(vec![Ok(UpsertResult::Written)]); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mappings = vec![ + mapping(&ec_id, "first", 1), + mapping(&ec_id, "last", 2), + mapping(&ec_id, " ", 3), + ]; + + let (accepted, errors) = process_mappings(&writer, "partner", &mappings); + + assert_eq!(accepted, 2, "should accept valid group members"); + assert_eq!(errors.len(), 1, "should retain the invalid UID error"); + assert_eq!(errors[0].index, 2, "should retain original error index"); + assert_eq!(errors[0].reason, REASON_INVALID_PARTNER_UID); + assert_eq!( + writer.calls()[0].uid, + "last", + "invalid duplicates must not replace the final valid UID" + ); + } + + #[test] + fn process_mappings_fans_out_ineligible_outcomes_to_group_members() { + let writer = MockWriter::new(vec![ + Ok(UpsertResult::NotFound), + Ok(UpsertResult::ConsentWithdrawn), + ]); + let ec_id_a = format!("{}.ABC123", "a".repeat(64)); + let ec_id_b = format!("{}.ABC123", "b".repeat(64)); + let mappings = vec![ + mapping(&ec_id_a, "a-1", 1), + mapping(&ec_id_b, "b-1", 2), + mapping(&ec_id_a, "a-2", 3), + mapping(&ec_id_b, "b-2", 4), + ]; + + let (accepted, errors) = process_mappings(&writer, "partner", &mappings); + + assert_eq!(accepted, 0, "should reject all ineligible group members"); + assert_eq!( + errors.len(), + mappings.len(), + "should account for every input" + ); + assert_eq!( + errors.iter().map(|error| error.index).collect::>(), + vec![0, 1, 2, 3], + "should sort errors by input index" + ); + assert!( + errors.iter().all(|error| error.reason == REASON_INELIGIBLE), + "should fan out ineligible outcomes" + ); + assert_eq!(writer.calls().len(), 2, "should call once per group"); + } + + #[test] + fn process_mappings_aborts_by_group_and_preserves_sorted_accounting() { + let writer = MockWriter::new(vec![ + Ok(UpsertResult::Written), + Err(Report::new(TrustedServerError::KvStore { + store_name: "ec_store".to_owned(), + message: "down".to_owned(), + })), + ]); + let ec_id_a = format!("{}.ABC123", "a".repeat(64)); + let ec_id_b = format!("{}.ABC123", "b".repeat(64)); + let ec_id_c = format!("{}.ABC123", "c".repeat(64)); + let mappings = vec![ + mapping("invalid", "bad-id", 1), + mapping(&ec_id_a, "a-first", 2), + mapping(&ec_id_b, "b-only", 3), + mapping(&ec_id_a, "a-last", 4), + mapping(&ec_id_c, "c-only", 5), + mapping(&ec_id_c, "", 6), + ]; + + let (accepted, errors) = process_mappings(&writer, "partner", &mappings); + + assert_eq!( + accepted, 2, + "successful groups should accept every member, including later duplicates" + ); + assert_eq!( + errors + .iter() + .map(|error| (error.index, error.reason)) + .collect::>(), + vec![ + (0, REASON_INVALID_EC_ID), + (2, REASON_KV_UNAVAILABLE), + (4, REASON_KV_UNAVAILABLE), + (5, REASON_INVALID_PARTNER_UID), + ], + "should preserve validation errors and fan out failed/unprocessed groups in input order" + ); + assert_eq!( + accepted + errors.len(), + mappings.len(), + "should account for every input exactly once" + ); + assert_eq!( + writer.calls().len(), + 2, + "should stop after the failing group" + ); + } + + #[test] + fn handle_batch_sync_reports_grouped_success_and_rejection_counts() { + let registry = test_registry(); + let limiter = MockRateLimiter { + should_exceed: false, + }; + let ec_id_a = format!("{}.ABC123", "a".repeat(64)); + let ec_id_b = format!("{}.ABC123", "b".repeat(64)); + let success_writer = MockWriter::new(vec![Ok(UpsertResult::Written)]); + let success_body = format!( + r#"{{"mappings":[{{"ec_id":"{ec_id_a}","partner_uid":"one","timestamp":1}},{{"ec_id":"{ec_id_a}","partner_uid":"two","timestamp":2}}]}}"# + ); + let success_response = handle_batch_sync_with_writer( + &success_writer, + ®istry, + &limiter, + authorized_batch_request(&success_body), + ) + .expect("should return success response"); + assert_eq!(success_response.status(), StatusCode::OK); + let success_body = success_response + .into_body() + .into_bytes() + .expect("should contain grouped success response"); + let success_json: serde_json::Value = serde_json::from_slice(&success_body) + .expect("should serialize grouped success response"); + assert_eq!(success_json["accepted"], 2); + assert_eq!(success_json["rejected"], 0); + + let rejected_writer = MockWriter::new(vec![Ok(UpsertResult::NotFound)]); + let rejected_body = format!( + r#"{{"mappings":[{{"ec_id":"{ec_id_b}","partner_uid":"one","timestamp":1}},{{"ec_id":"{ec_id_b}","partner_uid":"two","timestamp":2}}]}}"# + ); + let rejected_response = handle_batch_sync_with_writer( + &rejected_writer, + ®istry, + &limiter, + authorized_batch_request(&rejected_body), + ) + .expect("should return multi-status response"); + assert_eq!(rejected_response.status(), StatusCode::MULTI_STATUS); + let rejected_body = rejected_response + .into_body() + .into_bytes() + .expect("should contain grouped multi-status response"); + let rejected_json: serde_json::Value = serde_json::from_slice(&rejected_body) + .expect("should serialize grouped multi-status response"); + assert_eq!(rejected_json["accepted"], 0); + assert_eq!(rejected_json["rejected"], 2); + assert_eq!( + rejected_json["errors"], + serde_json::json!([ + {"index": 0, "reason": REASON_INELIGIBLE}, + {"index": 1, "reason": REASON_INELIGIBLE}, + ]) + ); + } + + #[test] + fn handle_batch_sync_reports_validation_errors_without_writer_calls() { + let writer = MockWriter::new(vec![]); + let registry = test_registry(); + let limiter = MockRateLimiter { + should_exceed: false, + }; + let valid_ec_id = format!("{}.ABC123", "a".repeat(64)); + let body = format!( + r#"{{"mappings":[{{"ec_id":"invalid","partner_uid":"one","timestamp":1}},{{"ec_id":"{valid_ec_id}","partner_uid":"","timestamp":2}}]}}"# + ); + + let response = handle_batch_sync_with_writer( + &writer, + ®istry, + &limiter, + authorized_batch_request(&body), + ) + .expect("should return validation response"); + + assert_eq!(response.status(), StatusCode::MULTI_STATUS); + let response = response_json(response); + assert_eq!(response["accepted"], 0); + assert_eq!(response["rejected"], 2); + assert_eq!( + response["errors"], + serde_json::json!([ + {"index": 0, "reason": REASON_INVALID_EC_ID}, + {"index": 1, "reason": REASON_INVALID_PARTNER_UID}, + ]) + ); + assert!( + writer.calls().is_empty(), + "invalid-only requests should not call the writer" + ); + } + + #[test] + fn handle_batch_sync_reports_grouped_infrastructure_failure() { + let writer = MockWriter::new(vec![ + Ok(UpsertResult::Written), + Err(Report::new(TrustedServerError::KvStore { + store_name: "ec_store".to_owned(), + message: "down".to_owned(), + })), + ]); + let registry = test_registry(); + let limiter = MockRateLimiter { + should_exceed: false, + }; + let ec_id_a = format!("{}.ABC123", "a".repeat(64)); + let ec_id_b = format!("{}.ABC123", "b".repeat(64)); + let ec_id_c = format!("{}.ABC123", "c".repeat(64)); + let body = format!( + r#"{{"mappings":[{{"ec_id":"{ec_id_a}","partner_uid":"a-first","timestamp":1}},{{"ec_id":"{ec_id_b}","partner_uid":"b","timestamp":2}},{{"ec_id":"{ec_id_a}","partner_uid":"a-last","timestamp":3}},{{"ec_id":"{ec_id_c}","partner_uid":"c","timestamp":4}}]}}"# + ); + + let response = handle_batch_sync_with_writer( + &writer, + ®istry, + &limiter, + authorized_batch_request(&body), + ) + .expect("should return infrastructure failure response"); + + assert_eq!(response.status(), StatusCode::MULTI_STATUS); + let response = response_json(response); + assert_eq!(response["accepted"], 2); + assert_eq!(response["rejected"], 2); + assert_eq!( + response["errors"], + serde_json::json!([ + {"index": 1, "reason": REASON_KV_UNAVAILABLE}, + {"index": 3, "reason": REASON_KV_UNAVAILABLE}, + ]) + ); + assert_eq!( + writer.calls(), + vec![ + WriterCall { + ec_id: ec_id_a, + partner_id: "ssp.example.com".to_owned(), + uid: "a-last".to_owned(), + }, + WriterCall { + ec_id: ec_id_b, + partner_id: "ssp.example.com".to_owned(), + uid: "b".to_owned(), + }, + ], + "should stop after the failing group and accept A's later duplicate" + ); } } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 1fe61fa1f..223c86930 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -6,23 +6,23 @@ use std::collections::HashSet; use edgezero_core::body::Body as EdgeBody; -use error_stack::Report; use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; -use crate::error::TrustedServerError; use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; use super::generation::{generate_ec_id, is_valid_ec_id}; use super::kv::{ - CreateIfAbsentOutcome, KvIdentityGraph, TombstoneOutcome, apply_partner_id_updates, + CreateIfAbsentOutcome, EidCookieSyncOutcome, KvIdentityGraph, PartnerIdUpdate, + apply_partner_id_updates, }; use super::kv_types::KvEntry; use super::prebid_eids::collect_eid_cookie_updates; +use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, current_timestamp, log_id}; +use super::{EcKvSnapshot, EidSyncSource, current_timestamp, log_id}; /// TS-managed response headers tied to EC identity output. const EC_RESPONSE_HEADERS: &[&str] = &[ @@ -52,10 +52,17 @@ pub fn ec_finalize_response( sharedid_cookie: Option<&str>, response: &mut Response, ) { + ec_context.validate_pull_sync_marker(settings, registry); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); if !consent_allows_ec { + // Expire the request-local marker independently of the EC cookie: a + // withdrawal must stop any pending pull-sync disclosure window. + if consent_withdrawn && ec_context.pull_sync_marker().was_present() { + expire_marker(ec_context.pull_sync_marker_mut(), response); + } + finalize_unusable_consent( settings, ec_context, @@ -70,13 +77,13 @@ pub fn ec_finalize_response( // Returning user: consent is granted and EC came from request. if ec_context.ec_was_present() && !ec_context.ec_generated() && consent_allows_ec { if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { - let updates = collect_eid_cookie_updates(eids_cookie, sharedid_cookie, registry); - let snapshot = graph.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - ec_context.kv_snapshot().clone(), - ); - ec_context.set_kv_snapshot(snapshot); + let source = ec_context.eid_sync_source(); + let updates = source + .map(|_| collect_eid_cookie_updates(eids_cookie, sharedid_cookie, registry)) + .unwrap_or_default(); + if let Some(source) = source { + sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, source); + } if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) && ec_context.recovery_eligible() { @@ -86,6 +93,8 @@ pub fn ec_finalize_response( } } + reconcile_pull_sync_marker(settings, registry, ec_context, response); + // Ordinary returning-user page views no longer refresh the browser // cookie, emit the EC header, or update KV TTL. return; @@ -97,22 +106,103 @@ pub fn ec_finalize_response( if ec_context.ec_generated() { let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { log::info!("Skipping generated EC response write because KV graph is unavailable"); + reconcile_pull_sync_marker(settings, registry, ec_context, response); return; }; let updates = collect_eid_cookie_updates(eids_cookie, sharedid_cookie, registry); - let snapshot = graph.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - ec_context.kv_snapshot().clone(), - ); - ec_context.set_kv_snapshot(snapshot); + sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, EidSyncSource::NewEc); if ec_context.kv_snapshot().entry_for(&ec_id).is_some() { set_ec_cookie_on_response(settings, ec_context, response); } else { log::warn!("Skipping generated EC cookie because backing row is not authoritative"); } } + + reconcile_pull_sync_marker(settings, registry, ec_context, response); +} + +fn sync_eid_cookie_updates( + graph: &KvIdentityGraph, + ec_context: &mut EcContext, + ec_id: &str, + updates: &[PartnerIdUpdate], + source: EidSyncSource, +) { + if updates.is_empty() { + return; + } + + let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( + ec_id, + updates, + ec_context.kv_snapshot().clone(), + ); + ec_context.set_kv_snapshot(snapshot); + record_eid_sync_terminal(source, outcome); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct EidSyncMeasurement { + source: EidSyncSource, + outcome: EidCookieSyncOutcome, + already_matched: u8, + written: u8, + conflict_duplicate: u8, + deferred: u8, +} + +impl EidSyncMeasurement { + fn new(source: EidSyncSource, outcome: EidCookieSyncOutcome) -> Self { + Self { + source, + outcome, + already_matched: u8::from(matches!(outcome, EidCookieSyncOutcome::AlreadyMatched)), + written: u8::from(matches!( + outcome, + EidCookieSyncOutcome::Written | EidCookieSyncOutcome::WrittenWithDeferredFreshness + )), + conflict_duplicate: u8::from(matches!(outcome, EidCookieSyncOutcome::ConflictMatched)), + deferred: u8::from(matches!( + outcome, + EidCookieSyncOutcome::WrittenWithDeferredFreshness + | EidCookieSyncOutcome::DeferredConflict + | EidCookieSyncOutcome::DeferredFreshness + )), + } + } +} + +fn record_eid_sync_terminal(source: EidSyncSource, outcome: EidCookieSyncOutcome) { + let measurement = EidSyncMeasurement::new(source, outcome); + log::info!( + "EID sync measurement: source={} outcome={} attempted=1 already_matched={} written={} \ + conflict_duplicate={} deferred={}", + measurement.source, + measurement.outcome, + measurement.already_matched, + measurement.written, + measurement.conflict_duplicate, + measurement.deferred, + ); +} + +fn reconcile_pull_sync_marker( + settings: &Settings, + registry: &PartnerRegistry, + ec_context: &mut EcContext, + response: &mut Response, +) { + let ec_id = ec_context.ec_value().map(str::to_owned); + let snapshot = ec_context.kv_snapshot().clone(); + reconcile_marker( + settings, + registry, + ec_id.as_deref(), + &snapshot, + ec_context.pull_sync_marker_mut(), + response, + ); } fn recover_orphaned_ec( @@ -217,8 +307,8 @@ fn confirm_then_recover_orphaned_ec( EcKvSnapshot::Present { .. } => { // The row became visible after the origin round trip: adopt it and // merge any pending updates rather than rotating a valid identity. - let merged = graph.upsert_partner_ids_from_snapshot(ec_id, updates, confirmed); - ec_context.set_kv_snapshot(merged); + ec_context.set_kv_snapshot(confirmed); + sync_eid_cookie_updates(graph, ec_context, ec_id, updates, EidSyncSource::Navigation); } EcKvSnapshot::Missing { .. } => match graph.key_exists_confirmed(ec_id) { Ok(false) => recover_orphaned_ec(settings, ec_context, graph, updates, response), @@ -318,49 +408,30 @@ fn finalize_unusable_consent( // the context — pull sync discloses the raw EC ID to partners — // sees the tombstone that was just written. Only the active ID has // a snapshot in the context to correct. - let outcome = graph.write_withdrawal_tombstone(ec_id, |snapshot| { - if ec_context.ec_value() == Some(ec_id) { - ec_context.set_kv_snapshot(snapshot); - } - }); - log_tombstone_outcome(ec_id, outcome); + let initial = if ec_context.kv_snapshot().belongs_to(ec_id) { + ec_context.kv_snapshot().clone() + } else { + EcKvSnapshot::NotRead + }; + let outcome = graph.tombstone_existing_from_snapshot(ec_id, initial); + // The browser cookie is already cleared, so a failed tombstone + // leaves a live row that server-side consumers still read as + // consented. Report every failure, including the non-active cookie + // ID whose outcome is not retained on the request context. + if matches!(outcome, EcKvSnapshot::Failed { .. }) { + log::warn!( + "EC withdrawal tombstone failed for '{}': the identity-graph row may \ + still be live with consent granted", + log_id(ec_id) + ); + } + if ec_context.ec_value() == Some(ec_id) { + ec_context.set_kv_snapshot(outcome); + } }); } } -/// Records what happened to one withdrawal tombstone. -/// -/// An unknown identity is expected traffic rather than a fault: the identifier -/// comes from a client-supplied cookie, so it may name something this -/// deployment never issued. An error is different: nothing was recorded, so a -/// real row may have gone unmarked, and that is logged as a fault. The browser -/// cookie is expired in every case, and that is the primary enforcement. -fn log_tombstone_outcome( - ec_id: &str, - outcome: Result>, -) { - match outcome { - Ok(TombstoneOutcome::Written) => {} - Ok(TombstoneOutcome::UnknownIdentity) => { - log::debug!( - "Skipping withdrawal tombstone for unknown EC ID '{}'", - log_id(ec_id), - ); - } - Err(err) => { - // Covers both a failed write and a check that could not determine - // whether the identity exists. Either way no marker was recorded, - // so a withdrawal may go unrecorded for the batch-sync window; the - // browser cookie is expired regardless. - log::error!( - "Could not record the withdrawal of EC ID '{}', so it may go unrecorded \ - for the batch-sync window; the browser cookie is still expired: {err:?}", - log_id(ec_id), - ); - } - } -} - fn withdrawal_ec_ids(ec_context: &EcContext) -> HashSet { let mut hashes = HashSet::new(); @@ -665,6 +736,7 @@ mod tests { ) .expect("should seed the identity"); ec_context.set_kv_snapshot(kv.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Auction); assert!(matches!( ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. } @@ -955,11 +1027,209 @@ mod tests { } #[test] - fn finalize_named_route_transient_miss_still_persists_eid_updates() { - // `/auction` and `/_ts/page-bids` save their first lookup into the - // context and are never recovery eligible, so a stale miss there has no - // later chance to retry. Finalization must revalidate before dropping - // the collected partner IDs. + fn finalize_returning_user_subresource_does_not_persist_eid_updates() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("subeid"); + let graph = KvIdentityGraph::in_memory("test_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), false); + let partners = vec![make_partner("sharedid.org")]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("shared-cookie-id"), + &mut response, + ); + + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert!( + !stored.ids.contains_key("sharedid.org"), + "a subresource response must not persist request EID cookies" + ); + } + + #[test] + fn finalize_navigation_persists_returning_user_eid_updates() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("naveid"); + let graph = KvIdentityGraph::in_memory("test_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + let partners = vec![make_partner("sharedid.org")]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("navigation-cookie-id"), + &mut response, + ); + + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert_eq!( + stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), + Some("navigation-cookie-id") + ); + } + + #[test] + fn finalize_generated_ec_persists_eid_updates() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("geneid"); + let graph = KvIdentityGraph::in_memory("test_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed generated row"); + let mut ec_context = + make_context(Some(&ec_id), None, false, true, Jurisdiction::NonRegulated); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let partners = vec![make_partner("sharedid.org")]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("generated-cookie-id"), + &mut response, + ); + + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert_eq!( + stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), + Some("generated-cookie-id") + ); + } + + #[test] + fn eid_sync_measurement_dimensions_are_bounded_and_identity_free() { + let sources = [ + EidSyncSource::Navigation, + EidSyncSource::Auction, + EidSyncSource::NewEc, + ]; + let outcomes = [ + EidCookieSyncOutcome::AlreadyMatched, + EidCookieSyncOutcome::Written, + EidCookieSyncOutcome::WrittenWithDeferredFreshness, + EidCookieSyncOutcome::ConflictMatched, + EidCookieSyncOutcome::DeferredConflict, + EidCookieSyncOutcome::DeferredFreshness, + EidCookieSyncOutcome::Missing, + EidCookieSyncOutcome::ConsentWithdrawn, + EidCookieSyncOutcome::Failed, + ]; + + assert_eq!( + sources.map(|source| source.to_string()), + ["navigation", "auction", "new_ec"] + ); + assert_eq!( + outcomes.map(|outcome| outcome.to_string()), + [ + "already_matched", + "written", + "written_with_deferred_freshness", + "conflict_matched", + "deferred_conflict", + "deferred_freshness", + "missing", + "consent_withdrawn", + "failed", + ] + ); + + assert_eq!( + EidSyncMeasurement::new( + EidSyncSource::Navigation, + EidCookieSyncOutcome::AlreadyMatched, + ), + EidSyncMeasurement { + source: EidSyncSource::Navigation, + outcome: EidCookieSyncOutcome::AlreadyMatched, + already_matched: 1, + written: 0, + conflict_duplicate: 0, + deferred: 0, + } + ); + assert_eq!( + EidSyncMeasurement::new( + EidSyncSource::Auction, + EidCookieSyncOutcome::WrittenWithDeferredFreshness, + ), + EidSyncMeasurement { + source: EidSyncSource::Auction, + outcome: EidCookieSyncOutcome::WrittenWithDeferredFreshness, + already_matched: 0, + written: 1, + conflict_duplicate: 0, + deferred: 1, + } + ); + assert_eq!( + EidSyncMeasurement::new(EidSyncSource::NewEc, EidCookieSyncOutcome::ConflictMatched,), + EidSyncMeasurement { + source: EidSyncSource::NewEc, + outcome: EidCookieSyncOutcome::ConflictMatched, + already_matched: 0, + written: 0, + conflict_duplicate: 1, + deferred: 0, + } + ); + } + + #[test] + fn finalize_auction_transient_miss_still_persists_eid_updates() { + // `/auction` saves its first lookup into the context and is never + // recovery eligible, so a stale miss there has no later chance to + // retry. Finalization must revalidate before dropping collected IDs. let settings = create_test_settings(); let ec_id = sample_ec_id("named1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -979,6 +1249,7 @@ mod tests { }, false, ); + ec_context.set_eid_sync_source(EidSyncSource::Auction); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1006,7 +1277,7 @@ mod tests { } #[test] - fn finalize_named_route_confirmed_miss_does_not_create_a_row() { + fn finalize_auction_confirmed_miss_does_not_create_a_row() { // The same path with a genuinely absent row must stay a no-op: a route // without orphan recovery must never mint an identity-graph entry. let settings = create_test_settings(); @@ -1019,6 +1290,7 @@ mod tests { }, false, ); + ec_context.set_eid_sync_source(EidSyncSource::Auction); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1523,6 +1795,280 @@ mod tests { ); } + #[test] + fn finalize_withdrawal_tombstones_both_present_ids_once() { + let settings = create_test_settings(); + let active_ec = sample_ec_id("activ3"); + let cookie_ec = sample_ec_id("cook3e"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&active_ec), Some(&cookie_ec), true, false, consent); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create( + &active_ec, + &KvEntry::minimal("active.example.com", "active-uid", 1_000), + ) + .expect("should seed active row"); + graph + .create( + &cookie_ec, + &KvEntry::minimal("cookie.example.com", "cookie-uid", 1_000), + ) + .expect("should seed cookie row"); + ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let (active_tombstone, active_generation) = graph + .get(&active_ec) + .expect("should read active row") + .expect("should retain active tombstone"); + let (cookie_tombstone, cookie_generation) = graph + .get(&cookie_ec) + .expect("should read cookie row") + .expect("should retain cookie tombstone"); + assert!( + !active_tombstone.consent.ok, + "active row should be withdrawn" + ); + assert!( + active_tombstone.ids.is_empty(), + "active IDs should be cleared" + ); + assert!( + !cookie_tombstone.consent.ok, + "cookie row should be withdrawn" + ); + assert!( + cookie_tombstone.ids.is_empty(), + "cookie IDs should be cleared" + ); + + let mut repeated_response = empty_response(); + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut repeated_response, + ); + + assert_eq!( + graph + .get(&active_ec) + .expect("should read active row") + .expect("should retain active tombstone") + .1, + active_generation, + "repeated finalization should not rewrite active tombstone" + ); + assert_eq!( + graph + .get(&cookie_ec) + .expect("should read cookie row") + .expect("should retain cookie tombstone") + .1, + cookie_generation, + "repeated finalization should not rewrite cookie tombstone" + ); + } + + #[test] + fn finalize_withdrawal_keeps_cookie_deletion_on_kv_failure() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("failw1"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + ec_context.set_pull_sync_marker_for_test( + crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, + ); + let graph = KvIdentityGraph::failing("unavailable-store"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let cookies = response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>(); + assert_eq!( + response.status(), + 200, + "KV failure should not change response status" + ); + assert!( + cookies + .iter() + .any(|cookie| { cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0") }), + "KV failure should not prevent EC cookie deletion" + ); + assert!( + cookies.iter().any(|cookie| { + cookie.starts_with("ts-ec-pull-complete=;") && cookie.contains("Max-Age=0") + }), + "KV failure should not prevent marker deletion" + ); + } + + #[test] + fn finalize_sets_marker_for_complete_pull_partner_snapshot() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("compl1"); + let mut partner = make_partner("ssp.example.com"); + partner.pull_sync_enabled = true; + partner.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); + partner.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; + partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); + let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); + let mut ec_context = make_context( + Some(&ec_id), + Some(&ec_id), + true, + false, + Jurisdiction::NonRegulated, + ); + let mut entry = live_entry(); + entry.ids.insert( + "ssp.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "partner-uid".to_owned(), + }, + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Present { + ec_id, + entry: Box::new(entry), + generation: Some(1), + }); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + ®istry, + None, + None, + &mut response, + ); + + let cookies = response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>(); + assert!( + cookies + .iter() + .any(|cookie| cookie.starts_with("ts-ec-pull-complete=v1.")), + "complete snapshot should issue the marker" + ); + } + + #[test] + fn explicit_withdrawal_without_marker_or_ec_cookie_does_not_set_cookie() { + let settings = create_test_settings(); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = make_context_with_consent(None, None, false, false, consent); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + response.headers().get(http::header::SET_COOKIE).is_none(), + "withdrawal without browser identity state should not add a cookie" + ); + } + + #[test] + fn explicit_withdrawal_expires_marker_without_ec_cookie() { + let settings = create_test_settings(); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = make_context_with_consent(None, None, false, false, consent); + ec_context.set_pull_sync_marker_for_test( + crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let cookies = response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>(); + assert!( + cookies.iter().any(|cookie| { + cookie.starts_with("ts-ec-pull-complete=;") && cookie.contains("Max-Age=0") + }), + "withdrawal should expire the marker independently of EC cookie state" + ); + assert!( + cookies.iter().all(|cookie| !cookie.starts_with("ts-ec=;")), + "missing EC cookie should not add an EC-cookie expiry" + ); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index fd9aa5773..1439db917 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -39,6 +39,9 @@ const ENTRY_TTL: Duration = Duration::from_secs(365 * 24 * 60 * 60); /// TTL for withdrawal tombstones (24 hours). const TOMBSTONE_TTL: Duration = Duration::from_secs(24 * 60 * 60); +/// Namespace for completion markers written after a withdrawal tombstone. +const WITHDRAWAL_MARKER_PREFIX: &str = "__ts_ec_withdrawal_complete__:"; + /// Outcome of an [`KvIdentityGraph::upsert_partner_id_if_exists`] call. /// /// Like [`KvIdentityGraph::upsert_partner_id`], this method fails closed when @@ -84,6 +87,96 @@ impl PartnerIdUpdate { } } +/// Terminal result of one browser EID-cookie persistence attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub(crate) enum EidCookieSyncOutcome { + /// The stored values already matched without a write. + #[display("already_matched")] + AlreadyMatched, + /// The one conditional write succeeded. + #[display("written")] + Written, + /// The write added missing IDs while deferring different values of unknown freshness. + #[display("written_with_deferred_freshness")] + WrittenWithDeferredFreshness, + /// A conflicting writer persisted every desired value. + #[display("conflict_matched")] + ConflictMatched, + /// A conflict left at least one desired value absent or different. + #[display("deferred_conflict")] + DeferredConflict, + /// A different stored value had unknown freshness. + #[display("deferred_freshness")] + DeferredFreshness, + /// The identity graph row was missing. + #[display("missing")] + Missing, + /// Consent had been withdrawn in the identity graph. + #[display("consent_withdrawn")] + ConsentWithdrawn, + /// KV or serialization failed. + #[display("failed")] + Failed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CookieUpdateApplication { + AlreadyMatched, + Changed { deferred: bool }, + DeferredFreshness, +} + +fn apply_cookie_partner_id_updates( + entry: &mut KvEntry, + updates: &[PartnerIdUpdate], +) -> CookieUpdateApplication { + let mut latest_updates = BTreeMap::new(); + for update in updates { + latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); + } + + let mut changed = false; + let mut deferred = false; + for (partner_id, uid) in latest_updates { + match entry.ids.get(partner_id) { + Some(existing) if existing.uid == uid => continue, + // Browser EID cookies carry no value-owned sequence or timestamp. + // A different value therefore has unknown freshness and must not + // replace the stored value. + Some(_) => { + deferred = true; + continue; + } + None => {} + } + + entry.ids.insert( + partner_id.to_owned(), + super::kv_types::KvPartnerId { + uid: uid.to_owned(), + }, + ); + changed = true; + } + + if changed { + CookieUpdateApplication::Changed { deferred } + } else if deferred { + CookieUpdateApplication::DeferredFreshness + } else { + CookieUpdateApplication::AlreadyMatched + } +} + +fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { + updates.iter().all(|update| { + entry + .ids + .get(&update.partner_id) + .is_some_and(|existing| existing.uid == update.uid) + }) +} + pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { @@ -380,6 +473,10 @@ impl KvIdentityGraph { // Serialize once and reuse across the fast path and CAS loop. let (body, meta_str) = Self::serialize_entry(entry, self.store_name())?; + // Completion markers belong to withdrawn generations. Remove any + // marker before this key can become live again. + self.clear_withdrawal_marker(ec_id)?; + // Try create first — fast path for new entries. if self.write_entry(ec_id, &body, &meta_str, ENTRY_TTL, EcKvWriteMode::Add)? == EcKvWriteOutcome::Written @@ -411,6 +508,11 @@ impl KvIdentityGraph { let mut current_gen = generation; for attempt in 0..MAX_CAS_RETRIES { + // A completion marker belongs to the tombstone generation. Remove + // it before making this key live so a later withdrawal cannot be + // suppressed by stale fallback state. + self.clear_withdrawal_marker(ec_id)?; + match self.write_entry( ec_id, &body, @@ -447,17 +549,16 @@ impl KvIdentityGraph { ))) } - /// Atomically merges multiple partner IDs into the existing entry. + /// Atomically merges browser partner IDs into an existing entry. /// - /// Uses one read-modify-write operation for all updates so request-local - /// EID cookie ingestion does not perform a KV read per matched partner. - /// Duplicate partner IDs are collapsed with the last value winning. + /// This compatibility entry point has no request-start observation, so it + /// can add missing IDs but cannot replace different values. It performs at + /// most one conditional write and one follow-up read on conflict. /// /// # Errors /// - /// Returns [`TrustedServerError::KvStore`] on store error, missing root - /// entry, withdrawn root entry, or CAS exhaustion after - /// [`MAX_CAS_RETRIES`] attempts. + /// Returns [`TrustedServerError::KvStore`] on store failure, a missing root, + /// or a withdrawn root. pub(crate) fn upsert_partner_ids( &self, ec_id: &str, @@ -467,68 +568,156 @@ impl KvIdentityGraph { return Ok(()); } - for attempt in 0..MAX_CAS_RETRIES { - let (mut entry, generation) = match self.get(ec_id)? { - Some(pair) => pair, - None => { - log::info!( - "upsert_partner_ids: no entry for '{}', rejecting {} partner updates", - log_id(ec_id), - updates.len(), - ); - return Err(self.kv_error(format!( - "Cannot upsert {} partner IDs for missing key '{}'", - updates.len(), - log_id(ec_id), - ))); - } - }; + let (_, outcome) = + self.sync_eid_cookie_updates_from_snapshot(ec_id, updates, EcKvSnapshot::NotRead); + match outcome { + EidCookieSyncOutcome::Missing => Err(self.kv_error(format!( + "Cannot upsert {} partner IDs for a missing key", + updates.len(), + ))), + EidCookieSyncOutcome::ConsentWithdrawn => Err(self.kv_error(format!( + "Cannot upsert {} partner IDs for a withdrawn key", + updates.len(), + ))), + EidCookieSyncOutcome::Failed => { + Err(self.kv_error(format!("Failed to upsert {} partner IDs", updates.len(),))) + } + EidCookieSyncOutcome::AlreadyMatched + | EidCookieSyncOutcome::Written + | EidCookieSyncOutcome::WrittenWithDeferredFreshness + | EidCookieSyncOutcome::ConflictMatched + | EidCookieSyncOutcome::DeferredConflict + | EidCookieSyncOutcome::DeferredFreshness => Ok(()), + } + } - // Reject upserts on withdrawn entries — a late sync must not - // repopulate partner IDs after consent withdrawal. - if !entry.consent.ok { - log::info!( - "upsert_partner_ids: entry for '{}' is a tombstone, rejecting {} partner updates", - log_id(ec_id), - updates.len(), + /// Persists browser EID cookies with one conditional write and one conflict read. + /// + /// Browser cookies do not carry a value-owned version, so a different + /// existing value has unknown freshness and is always deferred. After a CAS + /// conflict this method never writes again. A live follow-up becomes the + /// authoritative snapshot; a missing or failed follow-up retains the live + /// pre-write snapshot as proof and defers the update. + pub(crate) fn sync_eid_cookie_updates_from_snapshot( + &self, + ec_id: &str, + updates: &[PartnerIdUpdate], + snapshot: EcKvSnapshot, + ) -> (EcKvSnapshot, EidCookieSyncOutcome) { + if updates.is_empty() { + return (snapshot, EidCookieSyncOutcome::AlreadyMatched); + } + + let proven = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + .. + } if snapshot_id == ec_id => Some(snapshot.clone()), + _ => None, + }; + + let current = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(_), + .. + } if snapshot_id == ec_id => snapshot, + EcKvSnapshot::Failed { + ec_id: ref snapshot_id, + } if snapshot_id == ec_id => return (snapshot, EidCookieSyncOutcome::Failed), + _ => self.load_snapshot(ec_id), + }; + + let (mut entry, generation) = match current { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + generation: Some(generation), + } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), + EcKvSnapshot::Missing { .. } => { + let kept = Self::keep_proven(ec_id, current, proven.as_ref()); + return (kept, EidCookieSyncOutcome::Missing); + } + EcKvSnapshot::Failed { .. } => { + let kept = Self::keep_proven(ec_id, current, proven.as_ref()); + return (kept, EidCookieSyncOutcome::Failed); + } + EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { + return ( + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }, + EidCookieSyncOutcome::Failed, ); - return Err(self.kv_error(format!( - "Cannot upsert {} partner IDs for withdrawn key '{}'", - updates.len(), - log_id(ec_id), - ))); } + }; - if !apply_partner_id_updates(&mut entry, updates) { - return Ok(()); + if !entry.consent.ok { + return (current, EidCookieSyncOutcome::ConsentWithdrawn); + } + let deferred_freshness = match apply_cookie_partner_id_updates(&mut entry, updates) { + CookieUpdateApplication::AlreadyMatched => { + return (current, EidCookieSyncOutcome::AlreadyMatched); } + CookieUpdateApplication::DeferredFreshness => { + return (current, EidCookieSyncOutcome::DeferredFreshness); + } + CookieUpdateApplication::Changed { deferred } => deferred, + }; - let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; - - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - )? { - EcKvWriteOutcome::Written => return Ok(()), - EcKvWriteOutcome::PreconditionFailed => { - log::debug!( - "upsert_partner_ids: CAS conflict on attempt {}/{MAX_CAS_RETRIES} for '{}'", - attempt + 1, - log_id(ec_id), - ); - // Retry immediately; sleeping here blocks the edge worker. + let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { + return ( + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }, + EidCookieSyncOutcome::Failed, + ); + }; + match self.write_entry( + ec_id, + &body, + &meta_str, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(generation), + ) { + Ok(EcKvWriteOutcome::Written) => ( + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: None, + }, + if deferred_freshness { + EidCookieSyncOutcome::WrittenWithDeferredFreshness + } else { + EidCookieSyncOutcome::Written + }, + ), + Ok(EcKvWriteOutcome::PreconditionFailed) => { + let refreshed = self.load_snapshot(ec_id); + let Some(refreshed_entry) = refreshed.entry_for(ec_id) else { + let kept = Self::keep_proven(ec_id, refreshed, Some(¤t)); + return (kept, EidCookieSyncOutcome::DeferredConflict); + }; + if !refreshed_entry.consent.ok { + return (refreshed, EidCookieSyncOutcome::ConsentWithdrawn); } + let outcome = if partner_id_updates_match(refreshed_entry, updates) { + EidCookieSyncOutcome::ConflictMatched + } else { + EidCookieSyncOutcome::DeferredConflict + }; + (refreshed, outcome) + } + Err(_err) => { + log::warn!("EID cookie sync write failed"); + ( + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }, + EidCookieSyncOutcome::Failed, + ) } } - - Err(self.kv_error(format!( - "CAS conflict after {MAX_CAS_RETRIES} retries upserting {} partner IDs for '{}'", - updates.len(), - log_id(ec_id), - ))) } /// Merges partner IDs using request-scoped persisted state as the first CAS input. @@ -746,7 +935,6 @@ impl KvIdentityGraph { return Ok(()); } - // Merge the partner ID. entry.ids.insert( partner_id.to_owned(), super::kv_types::KvPartnerId { @@ -862,12 +1050,66 @@ impl KvIdentityGraph { self.store.key_exists(ec_id) } + fn withdrawal_marker_key(ec_id: &str) -> String { + format!("{WITHDRAWAL_MARKER_PREFIX}{ec_id}") + } + + fn withdrawal_marker_exists(&self, ec_id: &str) -> Result> { + let marker_key = Self::withdrawal_marker_key(ec_id); + Ok(self.store.count_keys_with_prefix(&marker_key, 1)? > 0) + } + + fn write_withdrawal_marker(&self, ec_id: &str) -> Result<(), Report> { + let marker_key = Self::withdrawal_marker_key(ec_id); + match self.store.insert( + &marker_key, + EcKvWrite { + body: "1", + metadata: "{}", + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + )? { + EcKvWriteOutcome::Written | EcKvWriteOutcome::PreconditionFailed => Ok(()), + } + } + + fn clear_withdrawal_marker(&self, ec_id: &str) -> Result<(), Report> { + if !self.withdrawal_marker_exists(ec_id)? { + return Ok(()); + } + + let marker_key = Self::withdrawal_marker_key(ec_id); + match self.store.delete(&marker_key) { + Ok(()) => Ok(()), + Err(delete_err) => match self.withdrawal_marker_exists(ec_id) { + // Another request removed the marker first. + Ok(false) => Ok(()), + Ok(true) | Err(_) => Err(delete_err), + }, + } + } + + fn record_withdrawal_completion(&self, ec_id: &str) { + if let Err(err) = self.write_withdrawal_marker(ec_id) { + // The root is already tombstoned. Preserve that successful privacy + // write even if the cost-control marker cannot be recorded. + log::warn!( + "withdrawal completion marker failed for '{}': {err:?}", + log_id(ec_id) + ); + } + } + /// Writes a withdrawal tombstone for consent enforcement. /// /// Overwrites the entry with `consent.ok = false`, empty partner IDs, /// and a 24-hour TTL. Uses unconditional overwrite (no CAS) since the /// entry is being withdrawn regardless of concurrent state. /// + /// A successful write records a same-TTL completion marker so repeated + /// stale misses do not overwrite the root or refresh its tombstone TTL. + /// /// The tombstone preserves consent enforcement for batch sync clients /// (`POST /_ts/api/v1/batch-sync`) during the 24-hour revocation window. /// @@ -931,13 +1173,17 @@ impl KvIdentityGraph { }, }); - written.map(|entry| { + let outcome = written.map(|entry| { if entry.is_some() { TombstoneOutcome::Written } else { TombstoneOutcome::UnknownIdentity } - }) + }); + if matches!(outcome, Ok(TombstoneOutcome::Written)) { + self.record_withdrawal_completion(ec_id); + } + outcome } /// Tombstones a held identity, returning the entry written. @@ -975,68 +1221,250 @@ impl KvIdentityGraph { }) } - /// Counts the number of keys sharing the same EC hash prefix. - /// - /// Uses the platform KV list API with a prefix filter, limited to - /// [`CLUSTER_LIST_LIMIT`] keys. If the limit is reached, the count - /// is capped — the exact number beyond the limit is not meaningful - /// for disambiguation. - /// - /// # Errors + /// Resolves a tombstone attempt whose point read reported the row absent. /// - /// Returns [`TrustedServerError::KvStore`] on store error. - pub fn count_hash_prefix_keys( - &self, - hash_prefix: &str, - ) -> Result> { - // The prefix ensures we only match EC IDs derived from the same - // IP+passphrase (i.e. same 64-hex hash). The backend already attaches - // store context to list failures, so propagate without re-wrapping. - self.store - .count_keys_with_prefix(hash_prefix, CLUSTER_LIST_LIMIT) + /// A proven-absent key is a no-op: there is nothing to withdraw, and a + /// forged cookie must not mint a row. A key that provably exists is + /// tombstoned unconditionally — no CAS generation is available after a + /// missed read, and a withdrawal must win over any concurrent write. An + /// existence check that itself fails leaves the withdrawal unresolved + /// rather than silently dropped. + fn tombstone_unproven_missing(&self, ec_id: &str, missing: EcKvSnapshot) -> EcKvSnapshot { + match self.key_exists_confirmed(ec_id) { + Ok(false) => missing, + Ok(true) => { + match self.withdrawal_marker_exists(ec_id) { + Ok(true) => { + log::debug!( + "withdrawal tombstone for '{}': completion marker already exists", + log_id(ec_id) + ); + return missing; + } + Ok(false) => {} + Err(err) => { + // Marker failure must not weaken withdrawal. Fall back + // to the existing unconditional privacy write. + log::warn!( + "withdrawal completion marker lookup failed for '{}': {err:?}", + log_id(ec_id) + ); + } + } + log::warn!( + "withdrawal tombstone for '{}': point read missed a row the store still \ + lists; writing an unconditional tombstone", + log_id(ec_id) + ); + let mut outcome_snapshot = EcKvSnapshot::NotRead; + match self.write_withdrawal_tombstone(ec_id, |snapshot| { + outcome_snapshot = snapshot; + }) { + Ok(TombstoneOutcome::Written) => outcome_snapshot, + Ok(TombstoneOutcome::UnknownIdentity) => EcKvSnapshot::Missing { + ec_id: ec_id.to_owned(), + }, + Err(err) => { + log::warn!( + "unconditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + Err(err) => { + log::warn!( + "withdrawal tombstone for '{}': existence check failed, cannot confirm \ + absence: {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } } - /// Evaluates the cluster size for an EC entry. + /// Writes a tombstone only when an existing row can be confirmed. /// - /// Returns the stored `cluster_size` when it has already been evaluated - /// for a live entry. Tombstone entries return `None` without store I/O so - /// their 24-hour withdrawal TTL is not extended. Otherwise, counts the - /// number of keys sharing the same hash prefix via - /// [`count_hash_prefix_keys`](Self::count_hash_prefix_keys) and writes the - /// result back to the entry. The CAS write is best-effort — on conflict - /// or write failure, the computed value is still returned. + /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` + /// cookie must not mint a row. But a *point read* cannot prove absence on + /// an eventually-consistent store, and dropping a withdrawal is worse than + /// a redundant read, so absence is established in two stages: /// - /// # Errors + /// 1. Any snapshot that is not a usable `Present` for this EC ID — a + /// publisher preload that read `Missing`, a read that `Failed`, or one + /// lacking a CAS generation — is re-read. On the publisher path that + /// re-read is separated from the preload by the full origin round trip, + /// which gives replication time to converge. + /// 2. A re-read that still reports the row absent is checked against + /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads + /// the primary data source. /// - /// Returns [`TrustedServerError::KvStore`] on store or list failure. - pub fn evaluate_cluster( + /// Resolving the initial snapshot happens outside the retry counter, so all + /// [`MAX_CAS_RETRIES`] iterations stay available for the tombstone write. + pub(crate) fn tombstone_existing_from_snapshot( &self, ec_id: &str, - entry: &KvEntry, - generation: u64, - ) -> Result, Report> { - if !entry.consent.ok { - log::trace!("evaluate_cluster: skipping tombstone entry"); - return Ok(None); - } - - if let Some(cluster_size) = entry - .network - .as_ref() - .and_then(|network| network.cluster_size) - { - log::trace!("evaluate_cluster: using stored cluster_size"); - return Ok(Some(cluster_size)); - } - - // Compute cluster size via prefix list. - let hash_prefix = ec_hash(ec_id); - let cluster_size = self.count_hash_prefix_keys(hash_prefix)?; - - log::debug!( - "evaluate_cluster: computed cluster_size={cluster_size} for '{}'", - log_id(ec_id) - ); + snapshot: EcKvSnapshot, + ) -> EcKvSnapshot { + let mut current = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + .. + } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(_), + .. + } if snapshot_id == ec_id => snapshot, + _ => self.load_snapshot(ec_id), + }; + + for _attempt in 0..MAX_CAS_RETRIES { + let generation = match current { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + .. + } if snapshot_id == ec_id && !entry.consent.ok => return current, + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(generation), + .. + } if snapshot_id == ec_id => generation, + // A missing row (including one that disappeared mid-retry) is + // only a no-op once absence is proven against the primary data + // source. + EcKvSnapshot::Missing { + ec_id: ref snapshot_id, + } if snapshot_id == ec_id => { + return self.tombstone_unproven_missing(ec_id, current); + } + // A refreshed read that failed (or any other unusable state) + // fails closed rather than silently dropping the withdrawal. + _ => { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + }; + + let tombstone = KvEntry::tombstone(current_timestamp()); + let Ok((body, meta_str)) = Self::serialize_entry(&tombstone, self.store_name()) else { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + }; + match self.write_entry( + ec_id, + &body, + &meta_str, + TOMBSTONE_TTL, + EcKvWriteMode::IfGenerationMatch(generation), + ) { + Ok(EcKvWriteOutcome::Written) => { + self.record_withdrawal_completion(ec_id); + return EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(tombstone), + generation: None, + }; + } + Ok(EcKvWriteOutcome::PreconditionFailed) => { + current = self.load_snapshot(ec_id); + } + Err(err) => { + log::warn!( + "conditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + } + } + + // Withdrawal enforcement lost every CAS race, so the row can still be + // live with consent granted while the browser cookie is cleared. That + // divergence is only visible to operators if it is logged here. + log::warn!( + "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the \ + identity-graph row may still be live with consent granted", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + + /// Counts the number of keys sharing the same EC hash prefix. + /// + /// Uses the platform KV list API with a prefix filter, limited to + /// [`CLUSTER_LIST_LIMIT`] keys. If the limit is reached, the count + /// is capped — the exact number beyond the limit is not meaningful + /// for disambiguation. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store error. + pub fn count_hash_prefix_keys( + &self, + hash_prefix: &str, + ) -> Result> { + // The prefix ensures we only match EC IDs derived from the same + // IP+passphrase (i.e. same 64-hex hash). The backend already attaches + // store context to list failures, so propagate without re-wrapping. + self.store + .count_keys_with_prefix(hash_prefix, CLUSTER_LIST_LIMIT) + } + + /// Evaluates the cluster size for an EC entry. + /// + /// Returns the stored `cluster_size` when it has already been evaluated + /// for a live entry. Tombstone entries return `None` without store I/O so + /// their 24-hour withdrawal TTL is not extended. Otherwise, counts the + /// number of keys sharing the same hash prefix via + /// [`count_hash_prefix_keys`](Self::count_hash_prefix_keys) and writes the + /// result back to the entry. The CAS write is best-effort — on conflict + /// or write failure, the computed value is still returned. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store or list failure. + pub fn evaluate_cluster( + &self, + ec_id: &str, + entry: &KvEntry, + generation: u64, + ) -> Result, Report> { + if !entry.consent.ok { + log::trace!("evaluate_cluster: skipping tombstone entry"); + return Ok(None); + } + + if let Some(cluster_size) = entry + .network + .as_ref() + .and_then(|network| network.cluster_size) + { + log::trace!("evaluate_cluster: using stored cluster_size"); + return Ok(Some(cluster_size)); + } + + // Compute cluster size via prefix list. + let hash_prefix = ec_hash(ec_id); + let cluster_size = self.count_hash_prefix_keys(hash_prefix)?; + + log::debug!( + "evaluate_cluster: computed cluster_size={cluster_size} for '{}'", + log_id(ec_id) + ); // Best-effort CAS write-back — update only the cluster size so any // future `network` fields are preserved across this lazy write. @@ -1076,7 +1504,7 @@ impl KvIdentityGraph { Ok(Some(cluster_size)) } - /// Hard-deletes the entry. + /// Hard-deletes the entry and any withdrawal completion marker. /// /// Reserved for the IAB data deletion framework (deferred). For consent /// withdrawal, use [`write_withdrawal_tombstone`](Self::write_withdrawal_tombstone). @@ -1087,7 +1515,8 @@ impl KvIdentityGraph { pub fn delete(&self, ec_id: &str) -> Result<(), Report> { // The backend's delete already attaches store context, so propagate // without re-wrapping the same message. - self.store.delete(ec_id) + self.store.delete(ec_id)?; + self.clear_withdrawal_marker(ec_id) } } @@ -1147,6 +1576,85 @@ mod tests { use super::*; use crate::ec::kv_backend::test_support::InMemoryEcKv; + /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition + /// and deletes the key, simulating a concurrent withdrawal that removes the + /// row between this writer's read and its write. + struct DisappearOnConflictEcKv { + inner: InMemoryEcKv, + conflicts_remaining: std::sync::Mutex, + } + + impl DisappearOnConflictEcKv { + fn new(conflicts: u32) -> Self { + Self { + inner: InMemoryEcKv::new("disappear-store"), + conflicts_remaining: std::sync::Mutex::new(conflicts), + } + } + + fn seed_live(&self, ec_id: &str) { + let (body, meta) = + KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) + .expect("should serialize seeded entry"); + self.inner + .insert( + ec_id, + EcKvWrite { + body: &body, + metadata: &meta, + ttl: ENTRY_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed live entry"); + } + } + + impl EcKvStore for DisappearOnConflictEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { + let mut remaining = self + .conflicts_remaining + .lock() + .expect("should lock conflict counter"); + if *remaining > 0 { + *remaining -= 1; + self.inner.delete(key).expect("should delete on conflict"); + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + } + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + fn snapshot_ec_id() -> String { format!("{}.ABC123", "a".repeat(64)) } @@ -1165,6 +1673,28 @@ mod tests { assert!(ts > 0, "should return a nonzero timestamp"); } + #[test] + fn kv_span_accumulates_across_graph_operations() { + let timings = crate::request_timing::RequestTimings::new(); + let graph = KvIdentityGraph::new(crate::platform::TimedKvStore::new( + crate::ec::kv_backend::test_support::InMemoryEcKv::new("test-store"), + timings.clone(), + )); + + graph + .create("ec-1", &live_entry()) + .expect("should create entry through the timed store"); + graph + .get("ec-1") + .expect("should read the entry back through the timed store"); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "should accumulate Phase::EcKv across both graph operations, not just the last write" + ); + } + #[test] fn serialize_entry_produces_valid_json() { let entry = KvEntry::tombstone(1000); @@ -1240,6 +1770,21 @@ mod tests { entry } + fn concurrent_live_entry() -> KvEntry { + let mut entry = live_entry(); + entry.ids.insert( + "concurrent.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "concurrent-uid".to_owned(), + }, + ); + entry + } + + // ----------------------------------------------------------------------- + // CAS-conflict injection tests + // ----------------------------------------------------------------------- + /// [`EcKvStore`] wrapper that injects generation conflicts: the first /// `conflicts_remaining` `IfGenerationMatch` inserts return /// [`EcKvWriteOutcome::PreconditionFailed`] without writing, optionally @@ -1248,6 +1793,7 @@ mod tests { inner: InMemoryEcKv, conflicts_remaining: std::sync::Mutex, revive_on_conflict: bool, + partner_update_on_conflict: bool, } impl ConflictInjectingEcKv { @@ -1256,6 +1802,16 @@ mod tests { inner: InMemoryEcKv::new("conflict-store"), conflicts_remaining: std::sync::Mutex::new(conflicts), revive_on_conflict, + partner_update_on_conflict: false, + } + } + + fn with_partner_update_on_conflict(conflicts: u32) -> Self { + Self { + inner: InMemoryEcKv::new("partner-conflict-store"), + conflicts_remaining: std::sync::Mutex::new(conflicts), + revive_on_conflict: true, + partner_update_on_conflict: true, } } @@ -1324,8 +1880,13 @@ mod tests { if self.revive_on_conflict { // Simulate a concurrent writer reviving the entry // between this writer's read and its CAS write. + let concurrent_entry = if self.partner_update_on_conflict { + concurrent_live_entry() + } else { + live_entry() + }; let (body, meta) = KvIdentityGraph::serialize_entry( - &live_entry(), + &concurrent_entry, self.inner.store_name(), ) .expect("should serialize concurrent live entry"); @@ -1360,37 +1921,154 @@ mod tests { } } - #[test] - fn create_or_revive_retries_cas_conflict_and_succeeds() { - let store = ConflictInjectingEcKv::new(2, false); - store.seed_tombstone("ec-1"); - let graph = KvIdentityGraph::new(store); - - graph - .create_or_revive("ec-1", &live_entry()) - .expect("should revive after re-reading a fresh generation"); - - let (entry, _) = graph - .get("ec-1") - .expect("should read entry") - .expect("entry should exist"); - assert!( - entry.consent.ok, - "tombstone should be revived after CAS retries" - ); + /// Store that replaces the row during the first EID CAS write and records + /// the request's reads and conditional writes. + struct EidConflictEcKv { + inner: InMemoryEcKv, + concurrent_entry: KvEntry, + lookups: std::sync::Arc, + conditional_writes: std::sync::Arc, + follow_up_miss: bool, + miss_next_lookup: std::sync::atomic::AtomicBool, } - #[test] - fn create_or_revive_short_circuits_on_concurrent_revive() { - // Inject more conflicts than MAX_CAS_RETRIES so the only way the call - // can succeed is the concurrent-revive short-circuit on re-read. - let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, true); - store.seed_tombstone("ec-2"); - let graph = KvIdentityGraph::new(store); + impl EidConflictEcKv { + fn new( + concurrent_entry: KvEntry, + lookups: std::sync::Arc, + conditional_writes: std::sync::Arc, + ) -> Self { + Self { + inner: InMemoryEcKv::new("eid-conflict-store"), + concurrent_entry, + lookups, + conditional_writes, + follow_up_miss: false, + miss_next_lookup: std::sync::atomic::AtomicBool::new(false), + } + } - graph - .create_or_revive("ec-2", &live_entry()) - .expect("should return Ok when a concurrent writer already revived the entry"); + fn with_follow_up_miss(mut self) -> Self { + self.follow_up_miss = true; + self + } + + fn seed_live(&self, ec_id: &str) { + let (body, meta) = + KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) + .expect("should serialize initial entry"); + self.inner + .insert( + ec_id, + EcKvWrite { + body: &body, + metadata: &meta, + ttl: ENTRY_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed initial entry"); + } + } + + impl EcKvStore for EidConflictEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.lookups + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if self + .miss_next_lookup + .swap(false, std::sync::atomic::Ordering::Relaxed) + { + return Ok(None); + } + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { + self.conditional_writes + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let (body, meta) = KvIdentityGraph::serialize_entry( + &self.concurrent_entry, + self.inner.store_name(), + ) + .expect("should serialize concurrent entry"); + self.inner + .insert( + key, + EcKvWrite { + body: &body, + metadata: &meta, + ttl: ENTRY_TTL, + mode: EcKvWriteMode::Overwrite, + }, + ) + .expect("should write concurrent entry"); + if self.follow_up_miss { + self.miss_next_lookup + .store(true, std::sync::atomic::Ordering::Relaxed); + } + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + #[test] + fn create_or_revive_retries_cas_conflict_and_succeeds() { + let store = ConflictInjectingEcKv::new(2, false); + store.seed_tombstone("ec-1"); + let graph = KvIdentityGraph::new(store); + + graph + .create_or_revive("ec-1", &live_entry()) + .expect("should revive after re-reading a fresh generation"); + + let (entry, _) = graph + .get("ec-1") + .expect("should read entry") + .expect("entry should exist"); + assert!( + entry.consent.ok, + "tombstone should be revived after CAS retries" + ); + } + + #[test] + fn create_or_revive_short_circuits_on_concurrent_revive() { + // Inject more conflicts than MAX_CAS_RETRIES so the only way the call + // can succeed is the concurrent-revive short-circuit on re-read. + let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, true); + store.seed_tombstone("ec-2"); + let graph = KvIdentityGraph::new(store); + + graph + .create_or_revive("ec-2", &live_entry()) + .expect("should return Ok when a concurrent writer already revived the entry"); } #[test] @@ -1501,6 +2179,235 @@ mod tests { assert_eq!(entry.ids["ssp_x"].uid, "original"); } + #[test] + fn eid_cookie_sync_conflict_matching_value_stops_after_one_write() { + let ec_id = snapshot_ec_id(); + let mut concurrent = live_entry(); + concurrent.ids.insert( + "ssp_x".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "desired-uid".to_owned(), + }, + ); + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let store = EidConflictEcKv::new( + concurrent, + std::sync::Arc::clone(&lookups), + std::sync::Arc::clone(&writes), + ); + store.seed_live(&ec_id); + let graph = KvIdentityGraph::new(store); + let snapshot = graph.load_snapshot(&ec_id); + + let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( + &ec_id, + &[PartnerIdUpdate::new("ssp_x", "desired-uid")], + snapshot, + ); + + assert_eq!(outcome, EidCookieSyncOutcome::ConflictMatched); + assert_eq!( + snapshot + .entry_for(&ec_id) + .and_then(|entry| entry.ids.get("ssp_x")) + .map(|id| id.uid.as_str()), + Some("desired-uid"), + "the authoritative follow-up snapshot should replace stale request state" + ); + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 2, + "a conflict should perform exactly one follow-up read" + ); + assert_eq!( + writes.load(std::sync::atomic::Ordering::Relaxed), + 1, + "a conflict must not trigger another conditional write" + ); + } + + #[test] + fn eid_cookie_sync_keeps_live_proof_when_conflict_follow_up_misses() { + let ec_id = snapshot_ec_id(); + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let store = EidConflictEcKv::new( + live_entry(), + std::sync::Arc::clone(&lookups), + std::sync::Arc::clone(&writes), + ) + .with_follow_up_miss(); + store.seed_live(&ec_id); + let graph = KvIdentityGraph::new(store); + + let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( + &ec_id, + &[PartnerIdUpdate::new("ssp_x", "desired-uid")], + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); + assert!( + snapshot.entry_for(&ec_id).is_some(), + "the live pre-write row should prevent conflict deferral from entering orphan recovery" + ); + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 2, + "the initial refresh and conflict follow-up should be the only reads" + ); + assert_eq!( + writes.load(std::sync::atomic::Ordering::Relaxed), + 1, + "the conflict must remain the request's only conditional write" + ); + } + + #[test] + fn eid_cookie_sync_conflicting_value_defers_after_one_write() { + let ec_id = snapshot_ec_id(); + let mut concurrent = live_entry(); + concurrent.ids.insert( + "ssp_x".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "concurrent-uid".to_owned(), + }, + ); + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let store = EidConflictEcKv::new( + concurrent, + std::sync::Arc::clone(&lookups), + std::sync::Arc::clone(&writes), + ); + store.seed_live(&ec_id); + let graph = KvIdentityGraph::new(store); + let snapshot = graph.load_snapshot(&ec_id); + + let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( + &ec_id, + &[PartnerIdUpdate::new("ssp_x", "stale-uid")], + snapshot, + ); + + assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 2, + "a conflict should perform exactly one follow-up read" + ); + assert_eq!( + writes.load(std::sync::atomic::Ordering::Relaxed), + 1, + "a conflict must not trigger another conditional write" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read concurrent row") + .expect("row should remain"); + assert_eq!(stored.ids["ssp_x"].uid, "concurrent-uid"); + } + + #[test] + fn eid_cookie_sync_preserves_concurrent_withdrawal_after_conflict() { + let ec_id = snapshot_ec_id(); + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let store = EidConflictEcKv::new( + KvEntry::tombstone(2_000), + std::sync::Arc::clone(&lookups), + std::sync::Arc::clone(&writes), + ); + store.seed_live(&ec_id); + let graph = KvIdentityGraph::new(store); + let snapshot = graph.load_snapshot(&ec_id); + + let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( + &ec_id, + &[PartnerIdUpdate::new("ssp_x", "desired-uid")], + snapshot, + ); + + assert_eq!(outcome, EidCookieSyncOutcome::ConsentWithdrawn); + assert!( + snapshot + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "the refreshed withdrawal tombstone must remain authoritative" + ); + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 2, + "a conflict should perform exactly one follow-up read" + ); + assert_eq!( + writes.load(std::sync::atomic::Ordering::Relaxed), + 1, + "a conflict must not trigger another conditional write" + ); + } + + #[test] + fn eid_cookie_sync_defers_different_values_without_value_owned_freshness() { + let ec_id = snapshot_ec_id(); + let graph = KvIdentityGraph::in_memory("freshness-store"); + let mut entry = live_entry(); + entry.ids.insert( + "ssp_x".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "protected-uid".to_owned(), + }, + ); + graph.create(&ec_id, &entry).expect("should seed entry"); + + let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( + &ec_id, + &[PartnerIdUpdate::new("ssp_x", "incoming-uid")], + graph.load_snapshot(&ec_id), + ); + + assert_eq!(outcome, EidCookieSyncOutcome::DeferredFreshness); + let (stored, _) = graph + .get(&ec_id) + .expect("should read protected row") + .expect("row should remain"); + assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); + } + + #[test] + fn eid_cookie_sync_adds_missing_ids_while_deferring_different_values() { + let ec_id = snapshot_ec_id(); + let graph = KvIdentityGraph::in_memory("mixed-freshness-store"); + let mut entry = live_entry(); + entry.ids.insert( + "ssp_x".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "protected-uid".to_owned(), + }, + ); + graph.create(&ec_id, &entry).expect("should seed entry"); + + let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( + &ec_id, + &[ + PartnerIdUpdate::new("ssp_x", "different-uid"), + PartnerIdUpdate::new("ssp_y", "new-uid"), + ], + graph.load_snapshot(&ec_id), + ); + + assert_eq!(outcome, EidCookieSyncOutcome::WrittenWithDeferredFreshness); + let (stored, _) = graph + .get(&ec_id) + .expect("should read updated row") + .expect("row should remain"); + assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); + assert_eq!(stored.ids["ssp_y"].uid, "new-uid"); + } + #[test] fn evaluate_cluster_returns_stored_value_without_store_io() { let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); @@ -1614,6 +2521,54 @@ mod tests { assert!(loaded.consent.ok, "should be live after revive"); } + #[test] + fn create_or_revive_clears_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + let snapshot = kv.load_snapshot(&ec_id); + kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + assert!( + kv.withdrawal_marker_exists(&ec_id) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); + + kv.create_or_revive(&ec_id, &live_entry()) + .expect("should revive tombstone"); + + let (loaded, _) = kv + .get(&ec_id) + .expect("should read revived entry") + .expect("should find revived entry"); + assert!(loaded.consent.ok, "should be live after revive"); + assert!( + !kv.withdrawal_marker_exists(&ec_id) + .expect("should read withdrawal marker"), + "revival should clear stale withdrawal completion" + ); + } + + #[test] + fn delete_removes_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + let snapshot = kv.load_snapshot(&ec_id); + kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + kv.delete(&ec_id).expect("should delete entry and marker"); + + assert!(kv.get(&ec_id).expect("should read store").is_none()); + assert!( + !kv.withdrawal_marker_exists(&ec_id) + .expect("should read withdrawal marker"), + "hard delete should remove withdrawal completion" + ); + } + #[test] fn upsert_partner_id_if_exists_reports_missing_key() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -1642,6 +2597,30 @@ mod tests { assert_eq!(second, UpsertResult::Unchanged); } + #[test] + fn upsert_partner_id_if_exists_retries_cas_conflict() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live entry"); + + let result = graph + .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") + .expect("should retry and write after one generation conflict"); + + assert_eq!(result, UpsertResult::Written); + let (entry, _) = graph + .get(&ec_id) + .expect("should read persisted entry") + .expect("should retain entry after conflict"); + assert_eq!( + entry.ids.get("ssp_x").map(|id| id.uid.as_str()), + Some("uid-1"), + "should persist requested UID after retry" + ); + } + #[test] fn upsert_partner_id_if_exists_rejects_tombstone() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -1698,15 +2677,54 @@ mod tests { } #[test] - fn write_withdrawal_tombstone_overwrites_live_entry() { + fn tombstone_existing_from_snapshot_never_creates_missing_key() { let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); + let snapshot = EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }; - assert_eq!( - kv.write_withdrawal_tombstone(&ec_id, drop) - .expect("should write tombstone"), - TombstoneOutcome::Written, + let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!(matches!(outcome, EcKvSnapshot::Missing { .. })); + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "withdrawal must not create a tombstone for an absent key" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_uses_existing_generation() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); + let snapshot = kv.load_snapshot(&ec_id); + + let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should return the persisted tombstone" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "should persist withdrawal state"); + } + + #[test] + fn write_withdrawal_tombstone_overwrites_live_entry() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); + + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should write tombstone"), + TombstoneOutcome::Written, "should tombstone an identity the store holds" ); @@ -1717,6 +2735,187 @@ mod tests { assert!(!loaded.consent.ok, "should be withdrawn after tombstone"); } + // ----------------------------------------------------------------------- + // Snapshot-aware mutation stores and tests + // ----------------------------------------------------------------------- + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + struct RecordedEcKvInsert { + mode: EcKvWriteMode, + ttl: Duration, + } + + #[derive(Default)] + struct RecordedEcKvOperations { + lookups: std::sync::atomic::AtomicUsize, + inserts: std::sync::Mutex>, + } + + impl RecordedEcKvOperations { + fn reset(&self) { + self.lookups.store(0, std::sync::atomic::Ordering::Relaxed); + self.inserts + .lock() + .expect("should lock recorded inserts") + .clear(); + } + + fn lookup_count(&self) -> usize { + self.lookups.load(std::sync::atomic::Ordering::Relaxed) + } + + fn inserts(&self) -> Vec { + self.inserts + .lock() + .expect("should lock recorded inserts") + .clone() + } + } + + /// In-memory store that records every backend operation before delegation. + struct RecordingEcKv { + inner: InMemoryEcKv, + operations: Arc, + stale_lookups_remaining: std::sync::Mutex, + } + + impl RecordingEcKv { + fn new(operations: Arc) -> Self { + Self::with_stale_lookups(operations, 0) + } + + fn with_stale_lookups(operations: Arc, stale_lookups: u32) -> Self { + Self { + inner: InMemoryEcKv::new("recording-store"), + operations, + stale_lookups_remaining: std::sync::Mutex::new(stale_lookups), + } + } + } + + impl EcKvStore for RecordingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.operations + .lookups + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut stale_lookups = self + .stale_lookups_remaining + .lock() + .expect("should lock stale lookup counter"); + if *stale_lookups > 0 { + *stale_lookups -= 1; + return Ok(None); + } + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.operations + .inserts + .lock() + .expect("should lock recorded inserts") + .push(RecordedEcKvInsert { + mode: write.mode, + ttl: write.ttl, + }); + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + /// Store whose completion-marker operations fail while root operations work. + struct MarkerFailingEcKv { + inner: InMemoryEcKv, + stale_lookups_remaining: std::sync::Mutex, + } + + impl MarkerFailingEcKv { + fn new(stale_lookups: u32) -> Self { + Self { + inner: InMemoryEcKv::new("marker-failing-store"), + stale_lookups_remaining: std::sync::Mutex::new(stale_lookups), + } + } + + fn marker_error(&self, operation: &str) -> Report { + Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: format!("completion marker {operation} failed"), + }) + } + } + + impl EcKvStore for MarkerFailingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + let mut stale_lookups = self + .stale_lookups_remaining + .lock() + .expect("should lock stale lookup counter"); + if *stale_lookups > 0 { + *stale_lookups -= 1; + return Ok(None); + } + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if key.starts_with(WITHDRAWAL_MARKER_PREFIX) { + return Err(self.marker_error("write")); + } + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + if prefix.starts_with(WITHDRAWAL_MARKER_PREFIX) { + return Err(self.marker_error("lookup")); + } + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + /// [`EcKvStore`] whose reads succeed but every write fails, simulating a /// store that becomes unwritable mid-request. struct WriteFailingEcKv { @@ -2023,6 +3222,475 @@ mod tests { ); } + #[test] + fn tombstone_existing_from_snapshot_skips_backend_for_authoritative_tombstone() { + for generation in [Some(7), None] { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation, + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); + + assert_eq!( + outcome, snapshot, + "should preserve authoritative tombstone state" + ); + assert_eq!( + operations.lookup_count(), + 0, + "should not reread a tombstone" + ); + assert!( + operations.inserts().is_empty(), + "should not attempt to rewrite a tombstone" + ); + } + } + + #[test] + fn tombstone_existing_from_snapshot_repeated_request_preserves_first_write() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let live_snapshot = graph.load_snapshot(&ec_id); + operations.reset(); + + graph.tombstone_existing_from_snapshot(&ec_id, live_snapshot); + + assert_eq!( + operations.lookup_count(), + 0, + "usable generation should avoid a read" + ); + assert_eq!( + operations.inserts(), + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, + ], + "first withdrawal should write the root and its completion marker" + ); + let first_snapshot = graph.load_snapshot(&ec_id); + let (first_entry, first_generation) = match &first_snapshot { + EcKvSnapshot::Present { + entry, generation, .. + } => (entry.as_ref().clone(), *generation), + other => panic!("should load first tombstone, got {other:?}"), + }; + operations.reset(); + + let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, first_snapshot); + + assert_eq!( + operations.lookup_count(), + 0, + "repeated withdrawal should not reread" + ); + assert!( + operations.inserts().is_empty(), + "repeated withdrawal should not refresh the tombstone TTL" + ); + assert_eq!( + second_outcome.generation_for(&ec_id), + first_generation, + "repeated withdrawal should preserve the stored generation" + ); + assert_eq!( + second_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + Some(first_entry.consent.updated), + "repeated withdrawal should preserve the first tombstone timestamp" + ); + } + + #[test] + fn tombstone_existing_from_repeated_stale_miss_preserves_first_write() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( + Arc::clone(&operations), + 2, + )); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + operations.reset(); + + let first_outcome = graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + let first_updated = first_outcome + .entry_for(&ec_id) + .expect("should return first tombstone") + .consent + .updated; + operations.reset(); + + graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + operations.inserts().is_empty(), + "a repeated stale miss should not rewrite the completed tombstone" + ); + let (stored, generation) = graph + .get(&ec_id) + .expect("should read stored tombstone") + .expect("should preserve tombstone"); + assert_eq!( + generation, 2, + "only the first withdrawal should advance the root generation" + ); + assert_eq!( + stored.consent.updated, first_updated, + "repeated withdrawal should preserve the first tombstone timestamp" + ); + } + + #[test] + fn tombstone_existing_from_stale_parallel_snapshot_stops_after_conflict() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let stale_snapshot = graph.load_snapshot(&ec_id); + operations.reset(); + + let first_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot.clone()); + let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot); + + assert_eq!( + operations.inserts(), + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + ], + "parallel loser should attempt stale CAS once and never replace the winner" + ); + assert_eq!( + operations.lookup_count(), + 1, + "parallel loser should reread exactly once after its conflict" + ); + assert_eq!( + second_outcome.generation_for(&ec_id), + Some(2), + "parallel loser should return the winner's stored generation" + ); + assert_eq!( + second_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + first_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + "parallel loser should preserve the winner's tombstone" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_succeeds_without_backend_for_tombstone() { + let graph = KvIdentityGraph::failing("unavailable-store"); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation: Some(3), + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); + + assert_eq!( + outcome, snapshot, + "authoritative tombstone should not touch unavailable backend" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_non_authoritative_states_reread_live_row() { + let ec_id = snapshot_ec_id(); + let states = [ + EcKvSnapshot::NotRead, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }, + EcKvSnapshot::Present { + ec_id: "different-ec-id".to_owned(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation: Some(9), + }, + ]; + + for state in states { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + operations.reset(); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, state); + + assert_eq!( + operations.lookup_count(), + 1, + "state should force one reread" + ); + assert_eq!( + operations.inserts().len(), + 2, + "live reread should write the root and completion marker" + ); + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "reread live row should be tombstoned" + ); + } + } + + #[test] + fn tombstone_stale_miss_still_writes_when_marker_operations_fail() { + let graph = KvIdentityGraph::new(MarkerFailingEcKv::new(1)); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + + let outcome = graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "marker failures must not suppress the withdrawal write" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read stored row") + .expect("should preserve the root"); + assert!(!stored.consent.ok, "root should remain tombstoned"); + } + + #[test] + fn tombstone_existing_from_snapshot_retries_cas_conflict() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should retry the conflict and persist the tombstone" + ); + } + + #[test] + fn tombstone_gen_unavailable_survives_four_conflicts_then_writes() { + // A generation-unavailable snapshot refreshes once before its CAS. That + // refresh must not spend a CAS attempt, so a withdrawal tombstone still + // persists after four conflicts and a successful fifth write. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "the fifth CAS attempt must persist the tombstone after a refresh and four conflicts" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_returns_failed_after_cas_exhaustion() { + // Every CAS attempt loses its race, so the row stays live with consent + // granted while the browser cookie is already cleared. The caller must + // see a failure it can report rather than a silent no-op. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "CAS exhaustion must report a failed withdrawal" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert!( + stored.consent.ok, + "the row is still live, which is exactly why the failure must be reported" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_overrides_concurrent_live_update() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::with_partner_update_on_conflict(1)); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + let entry = outcome + .entry_for(&ec_id) + .expect("should return persisted tombstone"); + assert!(!entry.consent.ok, "withdrawal should win after retry"); + assert!( + entry.ids.is_empty(), + "withdrawal should clear concurrent partner IDs" + ); + } + + #[test] + fn upsert_partner_id_rejects_tombstone() { + let graph = KvIdentityGraph::in_memory("test-store"); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &KvEntry::tombstone(1_000)) + .expect("should seed tombstone"); + + let result = graph.upsert_partner_id(&ec_id, "ssp.example.com", "uid-1"); + + assert!(result.is_err(), "public upsert should reject a tombstone"); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should preserve tombstone"); + assert!(!stored.consent.ok, "entry should remain withdrawn"); + assert!( + stored.ids.is_empty(), + "upsert should not repopulate partner IDs" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_store_failure_returns_failed() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!(matches!(outcome, EcKvSnapshot::Failed { .. })); + } + + #[test] + fn tombstone_existing_from_snapshot_noop_when_row_disappears_on_retry() { + let store = DisappearOnConflictEcKv::new(1); + store.seed_live(&snapshot_ec_id()); + let graph = KvIdentityGraph::new(store); + let ec_id = snapshot_ec_id(); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a row that disappears during retry becomes a no-op" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "must not recreate the disappeared key" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_reretries_failed_snapshot_read() { + // A prior request-scoped read failed, so the snapshot is `Failed`. A + // withdrawal must not silently drop consent removal: re-read the store + // and tombstone the row if it is authoritatively present. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + + let outcome = kv.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "a failed snapshot must re-read and persist the tombstone" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "withdrawal must reach the store"); + } + #[test] fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { let graph = KvIdentityGraph::stale_lookup("stale-store", 1); @@ -2170,8 +3838,8 @@ mod tests { .upsert_partner_ids(&ec_id, &[PartnerIdUpdate::new("partner", "uid")]) .expect_err("a batched upsert on a withdrawn key should be refused"); - // The CAS-exhaustion paths build their message the same way, and a - // store that never lets a write land is the only way to reach them. + // The remaining CAS-exhaustion paths build their message the same way, + // and a store that never lets a write land is the only way to reach them. let cas_revive = { let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); store.seed_tombstone(&ec_id); @@ -2186,13 +3854,6 @@ mod tests { .upsert_partner_id(&ec_id, "partner", "uid") .expect_err("should exhaust CAS retries") }; - let cas_batched = { - let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); - store.seed_live(&ec_id); - KvIdentityGraph::new(store) - .upsert_partner_ids(&ec_id, &[PartnerIdUpdate::new("partner", "uid")]) - .expect_err("should exhaust CAS retries") - }; let cas_if_exists = { let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); store.seed_live(&ec_id); @@ -2209,7 +3870,6 @@ mod tests { ("batched withdrawn key", batched_withdrawn), ("CAS exhaustion reviving", cas_revive), ("CAS exhaustion upserting", cas_upsert), - ("CAS exhaustion batch upserting", cas_batched), ("CAS exhaustion upserting if present", cas_if_exists), ] { let rendered = format!("{report:?}"); diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 3dc6e299e..6e0d07fbb 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -46,6 +46,7 @@ pub mod kv_types; pub mod partner; pub mod prebid_eids; pub mod pull_sync; +pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; @@ -72,7 +73,7 @@ use error_stack::Report; use http::Request; use crate::consent::{self as consent_mod, ConsentContext, ConsentPipelineInput}; -use crate::constants::COOKIE_TS_EC; +use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_PULL_COMPLETE}; use crate::cookies::handle_request_cookies; use crate::ec::cookies::ec_id_has_only_allowed_chars; use crate::error::TrustedServerError; @@ -83,6 +84,24 @@ use device::DeviceSignals; use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; use self::kv_types::KvEntry; +use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; + +/// Bounded request classifications that may persist browser EID cookies. +/// +/// Adapters assign a source only after pre-route filters allow dispatch. +/// Challenged or blocked requests remain unclassified and cannot persist EIDs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum EidSyncSource { + /// Publisher top-level document navigation. + #[display("navigation")] + Navigation, + /// `POST /auction` request. + #[display("auction")] + Auction, + /// Request that generated a new EC identity. + #[display("new_ec")] + NewEc, +} /// Request-scoped view of one EC identity-graph lookup. /// @@ -154,6 +173,8 @@ pub use generation::{ struct RequestEc { /// EC ID from the `ts-ec` cookie, if present. cookie_ec: Option, + /// Pull-sync completeness marker, if present. + pull_sync_marker: Option, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -170,8 +191,17 @@ fn parse_ec_from_request(req: &Request) -> Result Option { @@ -233,6 +263,10 @@ pub struct EcContext { kv_snapshot: EcKvSnapshot, /// Whether this request may rotate an orphaned EC identity. recovery_eligible: bool, + /// Browser-carried proof of recent pull-partner completeness. + pull_sync_marker: PullSyncMarkerState, + /// Allowed returning-user EID persistence source, assigned only after request filters pass. + eid_sync_source: Option, } impl EcContext { @@ -291,8 +325,6 @@ impl EcContext { req, config: &settings.consent, geo: geo_info, - ec_id: None, - kv_store: None, }); log::info!( @@ -314,6 +346,8 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, + pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), + eid_sync_source: None, }) } @@ -400,6 +434,7 @@ impl EcContext { self.ec_value = Some(ec_id); self.ec_generated = true; + self.pull_sync_marker.invalidate_for_replaced_ec(); return Ok(()); } @@ -496,17 +531,60 @@ impl EcContext { self.recovery_eligible = eligible; } + /// Allows returning-user EID cookie persistence for this request source. + pub fn set_eid_sync_source(&mut self, source: EidSyncSource) { + self.eid_sync_source = Some(source); + } + + /// Returns the allowed returning-user EID persistence source. + #[must_use] + pub fn eid_sync_source(&self) -> Option { + self.eid_sync_source + } + /// Returns whether orphan recovery is allowed for this request. #[must_use] pub fn recovery_eligible(&self) -> bool { self.recovery_eligible } + /// Validates a browser completeness marker against the active EC and partner set. + pub(crate) fn validate_pull_sync_marker( + &mut self, + settings: &Settings, + registry: ®istry::PartnerRegistry, + ) { + validate_marker_state( + &mut self.pull_sync_marker, + settings, + registry, + self.ec_value.as_deref(), + ); + } + + /// Returns the current pull-sync marker state. + #[must_use] + pub(crate) fn pull_sync_marker(&self) -> &PullSyncMarkerState { + &self.pull_sync_marker + } + + /// Returns mutable pull-sync marker state for response reconciliation. + pub(crate) fn pull_sync_marker_mut(&mut self) -> &mut PullSyncMarkerState { + &mut self.pull_sync_marker + } + + /// Sets pull-sync marker state in focused unit tests. + #[cfg(test)] + pub(crate) fn set_pull_sync_marker_for_test(&mut self, state: PullSyncMarkerState) { + self.pull_sync_marker = state; + } + /// Replaces an orphaned active ID after its new backing row is persisted. pub(crate) fn replace_with_generated(&mut self, ec_id: String, snapshot: EcKvSnapshot) { self.ec_value = Some(ec_id); self.ec_generated = true; self.kv_snapshot = snapshot; + self.pull_sync_marker.invalidate_for_replaced_ec(); } /// Returns whether EC creation is permitted by consent for this request. @@ -556,6 +634,8 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, + pull_sync_marker: PullSyncMarkerState::Absent, + eid_sync_source: None, } } @@ -578,6 +658,8 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, + pull_sync_marker: PullSyncMarkerState::Absent, + eid_sync_source: None, } } @@ -603,6 +685,8 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, + pull_sync_marker: PullSyncMarkerState::Absent, + eid_sync_source: None, } } } diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 3072473dd..1d70fa117 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -18,7 +18,6 @@ use crate::openrtb::{Eid, Uid}; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::MAX_UID_LENGTH; -use super::log_id; use super::registry::PartnerRegistry; /// Maximum raw `ts-eids` cookie size accepted before base64 decode. @@ -217,17 +216,12 @@ fn ingest_eid_cookies_with_writer( match writer.upsert_partner_ids(ec_id, &updates) { Ok(()) => { - log::debug!( - "EID cookies: synced {} partner IDs for EC ID '{}'", - updates.len(), - log_id(ec_id), - ); + log::debug!("EID cookies: processed {} partner IDs", updates.len()); } Err(err) => { log::warn!( - "EID cookies: failed to sync {} partner IDs for EC ID '{}': {err:?}", + "EID cookies: failed to process {} partner IDs: {err:?}", updates.len(), - log_id(ec_id), ); } } @@ -800,6 +794,29 @@ mod tests { ); } + #[test] + fn ingest_liveramp_eid_cookie_preserves_the_opaque_envelope() { + let registry = make_registry(vec![("liveramp", "liveramp.com")]); + let cookie = encode_json(&json!([ + { + "source": "liveramp.com", + "uids": [{"id": "opaque-test-envelope", "atype": 3}] + } + ])); + let writer = RecordingWriter::default(); + + ingest_eid_cookies_with_writer(Some(&cookie), None, "ec-id", &writer, ®istry); + + let calls = writer.calls.borrow(); + assert_eq!(calls.len(), 1, "should perform one bulk writer call"); + assert_eq!(calls[0].len(), 1, "should write one LiveRamp partner ID"); + assert_eq!( + calls[0][0], + PartnerIdUpdate::new("liveramp.com", "opaque-test-envelope"), + "should preserve the opaque envelope without decoding it" + ); + } + #[test] fn ingest_eid_cookies_sharedid_cookie_overrides_prebid_sharedid_update() { let registry = make_registry(vec![("sharedid", "sharedid.org")]); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 66b468d3c..b127d8c9b 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -59,8 +59,11 @@ struct PullSyncResponse { /// /// Returns `None` when consent denies EC or there is no active EC ID. #[must_use] -pub fn build_pull_sync_context(ec_context: &EcContext) -> Option { - if !ec_context.ec_allowed() { +pub fn build_pull_sync_context( + ec_context: &EcContext, + registry: &PartnerRegistry, +) -> Option { + if registry.pull_enabled_partners().is_empty() || !ec_context.ec_allowed() { return None; } @@ -70,6 +73,11 @@ pub fn build_pull_sync_context(ec_context: &EcContext) -> Option bool { + let pull_partners = registry.pull_enabled_partners(); + !pull_partners.is_empty() + && entry.consent.ok + && pull_partners + .iter() + .all(|partner| !is_partner_pull_eligible(partner, Some(entry))) +} + fn is_partner_pull_eligible(partner: &PartnerConfig, kv_entry: Option<&KvEntry>) -> bool { kv_entry .and_then(|entry| entry.ids.get(&partner.source_domain)) @@ -539,9 +558,13 @@ mod tests { ..ConsentContext::default() }; let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = EcContext::new_for_test(Some(ec_id), consent); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); + let graph = KvIdentityGraph::in_memory("pull_store"); + ec_context.set_kv_snapshot(seed_present_snapshot(&graph, &ec_id)); + let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) + .expect("should build registry"); - let context = build_pull_sync_context(&ec_context) + let context = build_pull_sync_context(&ec_context, ®istry) .expect("should build pull sync context for valid EC"); assert_eq!( context.ec_id(), @@ -557,14 +580,53 @@ mod tests { ..ConsentContext::default() }; let ec_context = EcContext::new_for_test(Some("invalid-ec".to_owned()), consent); + let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) + .expect("should build registry"); - let context = build_pull_sync_context(&ec_context); + let context = build_pull_sync_context(&ec_context, ®istry); assert!( context.is_none(), "should reject pull sync context when EC ID format is invalid" ); } + #[test] + fn build_pull_sync_context_skips_empty_registry_and_complete_snapshot() { + let consent = ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..ConsentContext::default() + }; + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); + let graph = KvIdentityGraph::in_memory("pull_store"); + let mut snapshot = seed_present_snapshot(&graph, &ec_id); + let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) + .expect("should build registry"); + + assert!( + build_pull_sync_context(&ec_context, &PartnerRegistry::empty()).is_none(), + "no pull partners should skip before graph construction" + ); + assert!( + build_pull_sync_context(&ec_context, ®istry).is_none(), + "an unread snapshot should skip before graph construction" + ); + + if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { + entry.ids.insert( + "ssp.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "uid".to_owned(), + }, + ); + } + ec_context.set_kv_snapshot(snapshot); + assert!( + build_pull_sync_context(&ec_context, ®istry).is_none(), + "a complete snapshot should skip post-send work" + ); + } + #[test] fn partner_is_eligible_when_missing_from_entry() { let partner = pull_partner(3600); @@ -587,6 +649,33 @@ mod tests { ); } + #[test] + fn completeness_requires_all_pull_partner_ids() { + let registry = PartnerRegistry::from_config(&[ + pull_enabled_ec_partner("a.example.com"), + pull_enabled_ec_partner("b.example.com"), + ]) + .expect("should build registry"); + let mut entry = KvEntry::minimal("a.example.com", "uid-a", 1_000); + + assert!( + !entry_is_pull_complete(&entry, ®istry), + "entry missing a pull partner ID should be incomplete" + ); + + entry.ids.insert( + "b.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "uid-b".to_owned(), + }, + ); + + assert!( + entry_is_pull_complete(&entry, ®istry), + "entry with every pull partner ID should be complete" + ); + } + #[test] fn validated_pull_sync_url_rejects_http_scheme() { let mut partner = pull_partner(3600); @@ -871,6 +960,69 @@ mod tests { ); } + #[test] + fn dispatch_pull_sync_calls_and_persists_only_missing_partner() { + let mut settings = create_test_settings(); + settings.ec.pull_sync_concurrency = 4; + let registry = PartnerRegistry::from_config(&[ + pull_enabled_ec_partner("alpha.example.com"), + pull_enabled_ec_partner("beta.example.com"), + ]) + .expect("should build pull registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + graph + .create( + &ec_id, + &KvEntry::minimal("alpha.example.com", "existing-alpha", 1_000), + ) + .expect("should seed partial entry"); + let snapshot = graph.load_snapshot(&ec_id); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, br#"{"uid":"new-beta"}"#.to_vec()); + let services = build_services_with_http_client(stub.clone()); + let context = PullSyncContext { + ec_id: ec_id.clone(), + snapshot, + }; + + dispatch_pull_sync( + &settings, + &graph, + ®istry, + &AllowAllRateLimiter, + &context, + &services, + ); + + assert_eq!( + stub.recorded_backend_names().len(), + 1, + "should call only the missing partner" + ); + let (entry, generation) = graph + .get(&ec_id) + .expect("should read store") + .expect("entry should exist"); + assert_eq!( + entry + .ids + .get("alpha.example.com") + .map(|partner_id| partner_id.uid.as_str()), + Some("existing-alpha"), + "existing UID should remain unchanged" + ); + assert_eq!( + entry + .ids + .get("beta.example.com") + .map(|partner_id| partner_id.uid.as_str()), + Some("new-beta"), + "missing UID should persist" + ); + assert_eq!(generation, 2, "missing UID should use one bulk CAS write"); + } + #[test] fn dispatch_pull_sync_skips_non_present_snapshots() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs new file mode 100644 index 000000000..ca24add85 --- /dev/null +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -0,0 +1,646 @@ +//! Short-lived browser marker for complete pull-sync partner state. + +use core::fmt; + +use edgezero_core::body::Body as EdgeBody; +use hmac::{Hmac, Mac as _}; +use http::{HeaderValue, Response, header}; +use sha2::{Digest as _, Sha256}; + +use crate::constants::COOKIE_TS_EC_PULL_COMPLETE; +use crate::redacted::Redacted; +use crate::settings::Settings; + +use super::pull_sync::entry_is_pull_complete; +use super::registry::PartnerRegistry; +use super::{EcKvSnapshot, current_timestamp}; + +type HmacSha256 = Hmac; + +const MARKER_VERSION: &str = "v1"; +const MARKER_KEY_LABEL: &[u8] = b"trusted-server/ec-pull-complete/key/v1"; +const MARKER_MAX_AGE_SECS: u64 = 60 * 60; +const MAX_MARKER_LENGTH: usize = 256; + +/// Request-local validation state for the pull-sync completeness marker. +#[derive(Clone, Default)] +pub(crate) enum PullSyncMarkerState { + /// No marker cookie was present. + #[default] + Absent, + /// A marker was present but has not been checked against the active EC and partner set. + Unvalidated(Redacted), + /// A present marker failed validation or was disproved by authoritative KV state. + Invalid, + /// The marker is valid until the given Unix timestamp. + Valid { expires_at: u64 }, +} + +impl fmt::Debug for PullSyncMarkerState { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Absent => formatter.write_str("Absent"), + Self::Unvalidated(_) => formatter.write_str("Unvalidated()"), + Self::Invalid => formatter.write_str("Invalid"), + Self::Valid { expires_at } => formatter + .debug_struct("Valid") + .field("expires_at", expires_at) + .finish(), + } + } +} + +impl PullSyncMarkerState { + /// Creates marker state from an optional request cookie value. + #[must_use] + pub(crate) fn from_cookie(value: Option) -> Self { + value + .map(Redacted::new) + .map_or(Self::Absent, Self::Unvalidated) + } + + /// Returns whether a marker cookie was present on the request. + #[must_use] + pub(crate) fn was_present(&self) -> bool { + !matches!(self, Self::Absent) + } + + /// Returns whether the marker is currently valid. + #[must_use] + pub(crate) fn is_valid(&self) -> bool { + matches!(self, Self::Valid { .. }) + } + + /// Invalidates state bound to a replaced active EC ID. + pub(crate) fn invalidate_for_replaced_ec(&mut self) { + if self.was_present() { + *self = Self::Invalid; + } + } +} + +/// Validates an unvalidated marker against the active EC and current pull-partner set. +pub(crate) fn validate_marker_state( + state: &mut PullSyncMarkerState, + settings: &Settings, + registry: &PartnerRegistry, + ec_id: Option<&str>, +) { + let PullSyncMarkerState::Unvalidated(value) = state else { + return; + }; + let valid_until = ec_id.and_then(|ec_id| { + validate_marker( + value.expose(), + settings, + registry, + ec_id, + current_timestamp(), + ) + }); + *state = valid_until.map_or(PullSyncMarkerState::Invalid, |expires_at| { + PullSyncMarkerState::Valid { expires_at } + }); +} + +/// Reconciles browser marker state against finalized authoritative KV state. +pub(crate) fn reconcile_marker( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: Option<&str>, + snapshot: &EcKvSnapshot, + state: &mut PullSyncMarkerState, + response: &mut Response, +) { + let pull_partners = sorted_pull_partner_domains(registry); + if pull_partners.is_empty() { + expire_if_present(state, response); + return; + } + + let Some(ec_id) = ec_id else { + expire_if_present(state, response); + return; + }; + + if !matches!(snapshot, EcKvSnapshot::NotRead) && !snapshot.belongs_to(ec_id) { + expire_if_present(state, response); + return; + } + + match snapshot { + EcKvSnapshot::Present { .. } => { + let entry = snapshot + .entry_for(ec_id) + .expect("snapshot binding should be checked before marker reconciliation"); + if !entry.consent.ok { + expire_if_present(state, response); + } else if entry_is_pull_complete(entry, registry) { + if !state.is_valid() { + set_marker(settings, registry, ec_id, state, response); + } + } else { + expire_if_present(state, response); + } + } + EcKvSnapshot::Missing { .. } => { + expire_if_present(state, response); + } + EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => { + if matches!(state, PullSyncMarkerState::Invalid) { + expire_if_present(state, response); + } + } + } +} + +/// Expires the completeness marker regardless of KV state. +pub(crate) fn expire_marker(state: &mut PullSyncMarkerState, response: &mut Response) { + append_cookie(response, &format_marker_cookie("", 0)); + *state = PullSyncMarkerState::Absent; +} + +fn set_marker( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, + state: &mut PullSyncMarkerState, + response: &mut Response, +) { + let now = current_timestamp(); + let expires_at = now.saturating_add(MARKER_MAX_AGE_SECS); + let Some(value) = create_marker(settings, registry, ec_id, expires_at) else { + return; + }; + append_cookie(response, &format_marker_cookie(&value, MARKER_MAX_AGE_SECS)); + *state = PullSyncMarkerState::Valid { expires_at }; +} + +fn expire_if_present(state: &mut PullSyncMarkerState, response: &mut Response) { + if state.was_present() { + expire_marker(state, response); + } +} + +fn append_cookie(response: &mut Response, value: &str) { + match HeaderValue::from_str(value) { + Ok(value) => { + response.headers_mut().append(header::SET_COOKIE, value); + } + Err(err) => { + log::warn!("Skipping pull-sync marker cookie: invalid header value: {err}"); + } + } +} + +fn format_marker_cookie(value: &str, max_age: u64) -> String { + format!( + "{COOKIE_TS_EC_PULL_COMPLETE}={value}; Path=/; Secure; SameSite=Lax; Max-Age={max_age}; HttpOnly" + ) +} + +#[cfg(test)] +pub(crate) fn create_marker_for_test( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, +) -> String { + create_marker_for_test_with_expiry( + settings, + registry, + ec_id, + current_timestamp().saturating_add(MARKER_MAX_AGE_SECS), + ) +} + +#[cfg(test)] +pub(crate) fn create_marker_for_test_with_expiry( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, + expires_at: u64, +) -> String { + create_marker(settings, registry, ec_id, expires_at) + .expect("should create marker for non-empty test registry") +} + +fn create_marker( + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, + expires_at: u64, +) -> Option { + let fingerprint = partner_set_fingerprint(registry)?; + let payload = marker_payload(ec_id, expires_at, &fingerprint); + let key = marker_key(settings); + let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); + mac.update(payload.as_bytes()); + let tag = hex::encode(mac.finalize().into_bytes()); + Some(format!("{MARKER_VERSION}.{expires_at}.{fingerprint}.{tag}")) +} + +fn validate_marker( + value: &str, + settings: &Settings, + registry: &PartnerRegistry, + ec_id: &str, + now: u64, +) -> Option { + if value.len() > MAX_MARKER_LENGTH { + return None; + } + + let mut segments = value.split('.'); + let version = segments.next()?; + let expires = segments.next()?; + let fingerprint = segments.next()?; + let tag = segments.next()?; + if segments.next().is_some() || version != MARKER_VERSION { + return None; + } + + let expires_at = expires.parse::().ok()?; + if expires_at <= now || expires_at > now.saturating_add(MARKER_MAX_AGE_SECS) { + return None; + } + + let expected_fingerprint = partner_set_fingerprint(registry)?; + if fingerprint != expected_fingerprint { + return None; + } + + let tag = hex::decode(tag).ok()?; + if tag.len() != 32 { + return None; + } + + let payload = marker_payload(ec_id, expires_at, fingerprint); + let key = marker_key(settings); + let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); + mac.update(payload.as_bytes()); + mac.verify_slice(&tag).ok()?; + Some(expires_at) +} + +fn marker_key(settings: &Settings) -> [u8; 32] { + let mut mac = HmacSha256::new_from_slice(settings.ec.passphrase.expose().as_bytes()) + .expect("should create marker key HMAC"); + mac.update(MARKER_KEY_LABEL); + mac.finalize().into_bytes().into() +} + +fn marker_payload(ec_id: &str, expires_at: u64, fingerprint: &str) -> String { + format!("{MARKER_VERSION}\0{ec_id}\0{expires_at}\0{fingerprint}") +} + +fn partner_set_fingerprint(registry: &PartnerRegistry) -> Option { + let domains = sorted_pull_partner_domains(registry); + if domains.is_empty() { + return None; + } + + let mut hasher = Sha256::new(); + hasher.update(b"trusted-server/ec-pull-partners/v1\0"); + for domain in domains { + hasher.update((domain.len() as u64).to_be_bytes()); + hasher.update(domain.as_bytes()); + } + Some(hex::encode(hasher.finalize())) +} + +fn sorted_pull_partner_domains(registry: &PartnerRegistry) -> Vec { + let mut domains = registry + .pull_enabled_partners() + .into_iter() + .map(|partner| partner.source_domain.clone()) + .collect::>(); + domains.sort(); + domains +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::kv_types::KvEntry; + use crate::redacted::Redacted; + use crate::settings::{EcPartner, Settings}; + use crate::test_support::tests::create_test_settings; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn pull_partner(source_domain: &str) -> EcPartner { + EcPartner { + name: format!("Partner {source_domain}"), + source_domain: source_domain.to_owned(), + openrtb_atype: EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Some(Redacted::new(format!( + "token-{source_domain}-32-bytes-minimum-value" + ))), + batch_rate_limit: EcPartner::default_batch_rate_limit(), + pull_sync_enabled: true, + pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), + pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], + pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), + pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + ts_pull_token: Some(Redacted::new("pull-token".to_owned())), + } + } + + fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { + let mut settings = create_test_settings(); + settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); + let registry = PartnerRegistry::from_config(&settings.ec.partners) + .expect("should build pull partner registry"); + (settings, registry) + } + + fn empty_response() -> Response { + Response::builder() + .status(200) + .body(EdgeBody::empty()) + .expect("should build response") + } + + fn live_snapshot(ec_id: &str, domains: &[&str]) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1_000); + entry.consent.ok = true; + for domain in domains { + entry.ids.insert( + (*domain).to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: format!("uid-{domain}"), + }, + ); + } + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: Some(1), + } + } + + fn marker_cookies(response: &Response) -> Vec<&str> { + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect() + } + + fn assert_expired(state: &PullSyncMarkerState, response: &Response) { + assert!(matches!(state, PullSyncMarkerState::Absent)); + assert_eq!( + marker_cookies(response), + vec!["ts-ec-pull-complete=; Path=/; Secure; SameSite=Lax; Max-Age=0; HttpOnly"], + "should emit the exact host-only expiration cookie" + ); + } + + #[test] + fn marker_round_trip_binds_ec_and_partner_set() { + let (settings, registry) = settings_and_registry(&["a.example.com", "b.example.com"]); + let now = 1_000; + let marker = + create_marker(&settings, ®istry, EC_ID, now + 3_600).expect("should create marker"); + + assert_eq!( + validate_marker(&marker, &settings, ®istry, EC_ID, now), + Some(now + 3_600), + "should validate the issued marker" + ); + assert!( + validate_marker(&marker, &settings, ®istry, "wrong", now).is_none(), + "should reject a marker for another EC" + ); + } + + #[test] + fn marker_fingerprint_is_order_independent_and_set_sensitive() { + let (settings, first) = settings_and_registry(&["a.example.com", "b.example.com"]); + let (_, reordered) = settings_and_registry(&["b.example.com", "a.example.com"]); + let (_, changed) = settings_and_registry(&["a.example.com", "c.example.com"]); + let marker = create_marker(&settings, &first, EC_ID, 4_600).expect("should create marker"); + + assert!( + validate_marker(&marker, &settings, &reordered, EC_ID, 1_000).is_some(), + "config ordering should not change the marker" + ); + assert!( + validate_marker(&marker, &settings, &changed, EC_ID, 1_000).is_none(), + "partner-set changes should invalidate the marker" + ); + } + + #[test] + fn fingerprint_changes_for_enable_disable_add_and_remove() { + let (_, enabled_one) = settings_and_registry(&["a.example.com"]); + let (_, enabled_two) = settings_and_registry(&["a.example.com", "b.example.com"]); + let mut disabled_config = pull_partner("a.example.com"); + disabled_config.pull_sync_enabled = false; + let disabled = PartnerRegistry::from_config(&[disabled_config]) + .expect("should build disabled registry"); + let removed = PartnerRegistry::empty(); + + let base = partner_set_fingerprint(&enabled_one); + assert_ne!( + base, + partner_set_fingerprint(&enabled_two), + "adding a pull partner should change the fingerprint" + ); + assert_ne!( + base, + partner_set_fingerprint(&disabled), + "disabling a pull partner should change the fingerprint" + ); + assert_ne!( + base, + partner_set_fingerprint(&removed), + "removing a pull partner should change the fingerprint" + ); + assert_ne!( + partner_set_fingerprint(&disabled), + partner_set_fingerprint(&enabled_one), + "enabling a partner should change the fingerprint" + ); + } + + #[test] + fn reconcile_rejects_snapshot_bound_to_different_ec() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let snapshot = live_snapshot("different-ec", &["a.example.com"]); + let mut valid_state = PullSyncMarkerState::Valid { expires_at: 4_600 }; + let mut valid_response = empty_response(); + + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &snapshot, + &mut valid_state, + &mut valid_response, + ); + assert_expired(&valid_state, &valid_response); + + let mut absent_state = PullSyncMarkerState::Absent; + let mut absent_response = empty_response(); + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &snapshot, + &mut absent_state, + &mut absent_response, + ); + assert!(marker_cookies(&absent_response).is_empty()); + assert!(matches!(absent_state, PullSyncMarkerState::Absent)); + } + + #[test] + fn authoritative_incomplete_states_expire_valid_marker() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let mut tombstone = KvEntry::tombstone(1_000); + tombstone.ids.insert( + "a.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "stale".to_owned(), + }, + ); + let snapshots = [ + live_snapshot(EC_ID, &[]), + EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }, + EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(tombstone), + generation: Some(1), + }, + ]; + + for snapshot in snapshots { + let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; + let mut response = empty_response(); + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &snapshot, + &mut state, + &mut response, + ); + assert_expired(&state, &response); + } + } + + #[test] + fn non_authoritative_states_preserve_valid_fixed_expiry_marker() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let snapshots = [ + EcKvSnapshot::Failed { + ec_id: EC_ID.to_owned(), + }, + EcKvSnapshot::NotRead, + ]; + + for snapshot in snapshots { + let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; + let mut response = empty_response(); + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &snapshot, + &mut state, + &mut response, + ); + assert!(matches!( + state, + PullSyncMarkerState::Valid { expires_at: 4_600 } + )); + assert!(marker_cookies(&response).is_empty()); + } + } + + #[test] + fn invalid_marker_is_cleared_without_authoritative_snapshot() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let mut state = PullSyncMarkerState::Invalid; + let mut response = empty_response(); + + reconcile_marker( + &settings, + ®istry, + Some(EC_ID), + &EcKvSnapshot::NotRead, + &mut state, + &mut response, + ); + + assert_expired(&state, &response); + } + + #[test] + fn marker_rejects_expired_overlong_and_tampered_values() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let marker = + create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); + let mut tampered = marker.clone(); + tampered.push('0'); + + assert!( + validate_marker(&marker, &settings, ®istry, EC_ID, 4_600).is_none(), + "expired marker should fail" + ); + assert!( + validate_marker(&marker, &settings, ®istry, EC_ID, 999).is_none(), + "marker more than one hour in the future should fail" + ); + assert!( + validate_marker(&tampered, &settings, ®istry, EC_ID, 1_000).is_none(), + "tampering should fail" + ); + assert!( + validate_marker( + &"x".repeat(MAX_MARKER_LENGTH + 1), + &settings, + ®istry, + EC_ID, + 1_000 + ) + .is_none(), + "overlong marker should fail" + ); + } + + #[test] + fn marker_rejects_wrong_passphrase_and_empty_partner_set() { + let (settings, registry) = settings_and_registry(&["a.example.com"]); + let marker = + create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); + let mut changed_settings = settings.clone(); + changed_settings.ec.passphrase = + Redacted::new("different-secret-key-32-bytes-minimum".to_owned()); + let empty = PartnerRegistry::empty(); + + assert!( + validate_marker(&marker, &changed_settings, ®istry, EC_ID, 1_000).is_none(), + "passphrase rotation should invalidate the marker" + ); + assert!( + create_marker(&settings, &empty, EC_ID, 4_600).is_none(), + "empty partner sets should not produce a marker" + ); + } + + #[test] + fn marker_cookie_is_host_only_and_secure() { + let cookie = format_marker_cookie("value", MARKER_MAX_AGE_SECS); + assert_eq!( + cookie, + "ts-ec-pull-complete=value; Path=/; Secure; SameSite=Lax; Max-Age=3600; HttpOnly" + ); + assert!(!cookie.contains("Domain="), "marker should be host-only"); + } +} diff --git a/crates/trusted-server-core/src/geo.rs b/crates/trusted-server-core/src/geo.rs index 63f7907f5..fe5785d26 100644 --- a/crates/trusted-server-core/src/geo.rs +++ b/crates/trusted-server-core/src/geo.rs @@ -48,6 +48,24 @@ impl GeoInfo { } } +/// Carries the outcome of a request-phase geo lookup across to +/// response-phase finalization, so a finalize consumer can reuse it instead +/// of performing a second lookup for the same request. +/// +/// Attached as a response extension on every exit path that attempted a +/// lookup, including the asset-route fallback (which does not carry an EC +/// finalize state). +#[derive(Debug, Clone)] +pub enum GeoLookupState { + /// No lookup has been attempted for this request. + NotAttempted, + /// A lookup ran and failed (or returned no result). This must not be + /// retried: finalize treats it the same as no geo info being available. + Attempted, + /// A lookup ran and resolved geo info. + Resolved(GeoInfo), +} + fn insert_geo_header(headers: &mut http::HeaderMap, name: http::header::HeaderName, value: &str) { match HeaderValue::from_str(value) { Ok(header_value) => { diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 4f45fbeb9..53b30ed6f 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -78,42 +78,54 @@ const APS_RENDERER_DOCUMENT: &str = r#" var match=/^#tsaps=([A-Za-z0-9_-]{22,128})$/.exec(location.hash); var expected=match&&match[1]; try{history.replaceState(null,'',location.pathname+location.search);}catch(_error){} -if(!expected)return; +var reported=false; +function report(reason,nonce){ + if(reported)return; + reported=true; + try{parent.postMessage({message:'trusted-server/aps/renderer-failed',nonce:nonce,reason:reason},'*');}catch(_error){} +} +if(!expected){report('bad_hash');return;} function keys(value,expectedKeys){ if(!value||typeof value!=='object'||Array.isArray(value))return false; var actual=Object.keys(value).sort(); return actual.length===expectedKeys.length&&actual.every(function(key,index){return key===expectedKeys[index];}); } -function validRenderer(renderer){ +function rendererProblem(renderer){ if(!keys(renderer,['aaxResponse','accountId','bidId','creativeId','creativeUrl','height','tagType','type','version','width'])&& - !keys(renderer,['aaxResponse','accountId','bidId','creativeUrl','height','tagType','type','version','width']))return false; - if(renderer.type!=='aps'||renderer.version!==1||typeof renderer.accountId!=='string'||!renderer.accountId||new TextEncoder().encode(renderer.accountId).length>1024)return false; - if(typeof renderer.bidId!=='string'||!renderer.bidId||!Number.isInteger(renderer.width)||renderer.width<=0||!Number.isInteger(renderer.height)||renderer.height<=0)return false; - if(Object.prototype.hasOwnProperty.call(renderer,'creativeId')&&(typeof renderer.creativeId!=='string'||!renderer.creativeId||new TextEncoder().encode(renderer.creativeId).length>1024))return false; - if(renderer.tagType!=='iframe'&&renderer.tagType!=='script')return false; - if(typeof renderer.creativeUrl!=='string'||new TextEncoder().encode(renderer.creativeUrl).length>4096)return false; - if(typeof renderer.aaxResponse!=='string'||!renderer.aaxResponse||renderer.aaxResponse.length>349528)return false; + !keys(renderer,['aaxResponse','accountId','bidId','creativeUrl','height','tagType','type','version','width']))return 'descriptor_keys'; + if(renderer.type!=='aps'||renderer.version!==1||typeof renderer.accountId!=='string'||!renderer.accountId||new TextEncoder().encode(renderer.accountId).length>1024)return 'descriptor_fields'; + if(typeof renderer.bidId!=='string'||!renderer.bidId||!Number.isInteger(renderer.width)||renderer.width<=0||!Number.isInteger(renderer.height)||renderer.height<=0)return 'descriptor_fields'; + if(Object.prototype.hasOwnProperty.call(renderer,'creativeId')&&(typeof renderer.creativeId!=='string'||!renderer.creativeId||new TextEncoder().encode(renderer.creativeId).length>1024))return 'descriptor_fields'; + if(renderer.tagType!=='iframe'&&renderer.tagType!=='script')return 'descriptor_fields'; + if(typeof renderer.creativeUrl!=='string'||new TextEncoder().encode(renderer.creativeUrl).length>4096)return 'descriptor_fields'; + if(typeof renderer.aaxResponse!=='string'||!renderer.aaxResponse||renderer.aaxResponse.length>349528)return 'descriptor_fields'; try{ var url=new URL(renderer.creativeUrl); - if(url.protocol!=='https:'||url.username||url.password)return false; + if(url.protocol!=='https:'||url.username||url.password)return 'descriptor_envelope'; var binary=atob(renderer.aaxResponse); - if(binary.length>262144||btoa(binary)!==renderer.aaxResponse)return false; + if(binary.length>262144||btoa(binary)!==renderer.aaxResponse)return 'descriptor_envelope'; var bytes=Uint8Array.from(binary,function(character){return character.charCodeAt(0);}); var decoded=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes)); - if(!keys(decoded,['seatbid'])||!Array.isArray(decoded.seatbid)||decoded.seatbid.length!==1)return false; + if(!keys(decoded,['seatbid'])||!Array.isArray(decoded.seatbid)||decoded.seatbid.length!==1)return 'descriptor_envelope'; var seat=decoded.seatbid[0]; - if(!keys(seat,['bid'])||!Array.isArray(seat.bid)||seat.bid.length!==1)return false; + if(!keys(seat,['bid'])||!Array.isArray(seat.bid)||seat.bid.length!==1)return 'descriptor_envelope'; var bid=seat.bid[0]; - if(!keys(bid,['ext','h','id','price','w'])||!keys(bid.ext,['creativeurl','tagtype']))return false; - return bid.id===renderer.bidId&&bid.w===renderer.width&&bid.h===renderer.height&& + if(!keys(bid,['ext','h','id','price','w'])||!keys(bid.ext,['creativeurl','tagtype']))return 'descriptor_envelope'; + if(bid.id===renderer.bidId&&bid.w===renderer.width&&bid.h===renderer.height&& bid.ext.creativeurl===renderer.creativeUrl&&bid.ext.tagtype===renderer.tagType&& - typeof bid.price==='number'&&Number.isFinite(bid.price)&&bid.price>=0; - }catch(_error){return false;} + typeof bid.price==='number'&&Number.isFinite(bid.price)&&bid.price>=0)return undefined; + return 'descriptor_envelope'; + }catch(_error){return 'descriptor_envelope';} } function receive(event){ - if(event.source!==parent)return; var message=event.data; - if(!keys(message,['nonce','renderer'])||message.nonce!==expected||!validRenderer(message.renderer))return; + // Stay silent for traffic that is not shaped like the render handshake, so an + // unrelated sender cannot consume this frame's single report. + if(!keys(message,['nonce','renderer']))return; + if(event.source!==parent){report('source_mismatch');return;} + if(message.nonce!==expected){report('nonce_mismatch');return;} + var problem=rendererProblem(message.renderer); + if(problem){report(problem,message.nonce);return;} removeEventListener('message',receive); var acceptedNonce=expected; expected=''; @@ -128,7 +140,7 @@ function receive(event){ var script=document.createElement('script'); script.src='https://client.aps.amazon-adsystem.com/prebid-creative.js'; script.onload=function(){parent.postMessage({message:'trusted-server/aps/renderer-ready',nonce:acceptedNonce},'*');}; - script.onerror=function(){parent.postMessage({message:'trusted-server/aps/renderer-failed',nonce:acceptedNonce},'*');}; + script.onerror=function(){report('amazon_script_error',acceptedNonce);}; document.head.appendChild(script); } addEventListener('message',receive); @@ -3308,4 +3320,41 @@ mod tests { assert!(APS_RENDERER_CSP.contains("sandbox allow-forms")); assert!(!APS_RENDERER_CSP.contains("allow-same-origin")); } + + #[test] + fn renderer_document_reports_a_reason_for_every_silent_guard() { + for reason in [ + "bad_hash", + "source_mismatch", + "nonce_mismatch", + "descriptor_keys", + "descriptor_fields", + "descriptor_envelope", + "amazon_script_error", + ] { + assert!( + APS_RENDERER_DOCUMENT.contains(reason), + "renderer document should report a `{reason}` reason instead of returning silently" + ); + } + + // Reasons travel on the existing failure message rather than a new channel. + assert!( + APS_RENDERER_DOCUMENT.contains("reason:reason"), + "should attach the reason to the failure message" + ); + + // A reason is a fixed category, never a copy of the rejected descriptor. + assert!(!APS_RENDERER_DOCUMENT.contains("JSON.stringify(renderer)")); + assert!(!APS_RENDERER_DOCUMENT.contains("reason:message")); + + // Reporting is one-shot so a hostile sender cannot flood the parent. + assert!( + APS_RENDERER_DOCUMENT.contains("if(reported)return"), + "should report at most one reason per frame" + ); + + // A foreign sender is answered through the parent, never the sender. + assert!(!APS_RENDERER_DOCUMENT.contains("event.source.postMessage")); + } } diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index c7cceaa80..425ff90f7 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -290,7 +290,7 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. - ts.scheduleInitialAdInit = function (initialBids, initialSlots) { + ts.scheduleInitialAdInit = function (initialBids, initialSlots, initialAuctionDiagnostics) { // The bundle may replace this scheduler after the fallback claims the initial // pass. Keep the latch on the shared document API so replacement cannot reset it. if ((ts.navGeneration || 0) !== 0 || ts.initialAdInitScheduled) return; @@ -300,6 +300,9 @@ // would overwrite a committed SPA navigation's slots. if (initialSlots !== undefined) ts.adSlots = initialSlots; if (initialBids !== undefined) ts.bids = initialBids; + if (initialAuctionDiagnostics !== undefined) { + ts.auctionDiagnostics = initialAuctionDiagnostics; + } var fire = function () { if ((ts.navGeneration || 0) !== 0) return; if (typeof ts.adInit === "function") ts.adInit(); diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 1447a8358..de1aedd36 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -62,6 +62,7 @@ pub enum GptDiagnosticsCookieAction { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct GptDiagnosticsRequestDecision { active: bool, + browser_session_active: bool, clean_browser_path_and_query: Option, cookie_action: GptDiagnosticsCookieAction, } @@ -73,6 +74,16 @@ impl GptDiagnosticsRequestDecision { self.active } + /// Whether this request came from an activated diagnostics browser session. + /// + /// Unlike [`Self::active`], this remains true for non-document requests such + /// as the SPA page-bids fetch. It is captured before the private activation + /// cookie is stripped from the request. + #[must_use] + pub(crate) fn browser_session_active(&self) -> bool { + self.browser_session_active + } + /// Whether the response must be private and non-storeable. #[must_use] pub fn requires_private_no_store(&self) -> bool { @@ -121,6 +132,7 @@ impl GptDiagnosticsRequestDecision { pub(crate) fn active_for_tests() -> Self { Self { active: true, + browser_session_active: true, clean_browser_path_and_query: None, cookie_action: GptDiagnosticsCookieAction::None, } @@ -143,6 +155,7 @@ mod head_seam_invariant_tests { ] { out.push(GptDiagnosticsRequestDecision { active, + browser_session_active: active, clean_browser_path_and_query: clean.clone(), cookie_action, }); @@ -279,12 +292,19 @@ pub fn prepare_request( replace_path_and_query(request, &clean_path)?; } - let mut decision = GptDiagnosticsRequestDecision::default(); + let mut decision = GptDiagnosticsRequestDecision { + browser_session_active: integration_enabled + && directive == QueryDirective::Absent + && cookie_state.occurrences == 1 + && cookie_state.canonical, + ..GptDiagnosticsRequestDecision::default() + }; if integration_enabled && eligible_navigation && had_reserved_query { decision.clean_browser_path_and_query = Some(clean_path); match directive { QueryDirective::Enable => { decision.active = true; + decision.browser_session_active = true; decision.cookie_action = GptDiagnosticsCookieAction::SetSession; } QueryDirective::Disable => { @@ -547,6 +567,22 @@ mod tests { assert_eq!(duplicate.headers()[header::COOKIE], "other=value"); } + #[test] + fn active_cookie_marks_non_document_requests_without_activating_document_behavior() { + let mut request = Request::builder() + .method(Method::GET) + .uri("https://publisher.example/_ts/page-bids?path=/article") + .header(header::COOKIE, "__Host-ts-console=1; other=value") + .body(EdgeBody::empty()) + .expect("should build page-bids request"); + + let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + + assert!(!decision.active()); + assert!(decision.browser_session_active()); + assert_eq!(request.headers()[header::COOKIE], "other=value"); + } + #[test] fn invalid_duplicate_and_disable_directives_fail_closed() { for query in [ diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 228cee1cd..5d21babe4 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1,6 +1,4 @@ -use std::collections::HashMap; -#[cfg(test)] -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, LazyLock}; #[cfg(test)] use std::time::Duration; @@ -222,6 +220,113 @@ fn extract_prebid_error_message( #[cfg(test)] const GPC_US_PRIVACY: &str = "1YYN"; +/// Rejects a Prebid User ID identifier that Prebid.js could not address. +/// +/// Applies only the constraints Prebid itself imposes on a `userSync.userIds` +/// entry name and on a storage key: a non-empty, untrimmed-free ASCII token. +/// Anything narrower would encode one vendor's rules into core. +fn validate_prebid_user_id_token(value: &str) -> Result<(), ValidationError> { + let is_valid = !value.is_empty() + && value.trim() == value + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.')); + if is_valid { + return Ok(()); + } + + let mut error = ValidationError::new("invalid_prebid_user_id_token"); + error.message = Some( + "must be a non-empty ASCII token of letters, digits, `_`, `-`, or `.` without surrounding whitespace" + .into(), + ); + Err(error) +} + +/// Browser storage mechanism for an operator-managed Prebid User ID module. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum PrebidUserIdStorageType { + /// Store the module's value in a browser cookie. + #[default] + Cookie, + /// Store the module's value in browser local storage. + Html5, +} + +/// Browser storage settings forwarded verbatim to a Prebid User ID module. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrebidManagedUserIdStorage { + /// Browser storage mechanism. + #[serde(default, rename = "type")] + pub storage_type: PrebidUserIdStorageType, + /// Cookie or local-storage key the module reads and writes. + #[validate(custom(function = "validate_prebid_user_id_token"))] + pub name: String, + /// Number of days the browser retains the stored value. + /// + /// Omitted leaves Prebid's own default in place. Core applies no upper + /// bound: the ceiling is a property of the selected module, not of Trusted + /// Server. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub expires: Option, + /// Number of seconds before the module may refresh the stored value. + /// + /// Omitted leaves Prebid's own default in place. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub refresh_in_seconds: Option, +} + +/// Rejects a managed User ID list that names the same module twice. +/// +/// Prebid keys `userSync.userIds` by entry name, so two entries sharing a name +/// give one submodule two conflicting configurations with no defined winner. +fn validate_unique_managed_user_id_names( + entries: &[PrebidManagedUserIdConfig], +) -> Result<(), ValidationError> { + let mut seen = HashSet::with_capacity(entries.len()); + let Some(duplicate) = entries + .iter() + .find(|entry| !seen.insert(entry.name.as_str())) + else { + return Ok(()); + }; + + let mut error = ValidationError::new("duplicate_managed_user_id_name"); + error.message = Some( + format!( + "managed Prebid User ID module `{}` is configured more than once", + duplicate.name + ) + .into(), + ); + Err(error) +} + +/// Operator-owned Prebid User ID module entry that Trusted Server manages. +/// +/// Core treats every entry as opaque: it validates only what Prebid.js needs to +/// address the module, then forwards the entry to the browser unchanged. Which +/// identity vendor an entry selects is an operator configuration choice, not a +/// property of core. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrebidManagedUserIdConfig { + /// Prebid `userSync.userIds` entry name, for example `sharedId`. + #[validate(custom(function = "validate_prebid_user_id_token"))] + pub name: String, + /// Module-specific parameters, forwarded to Prebid without inspection. + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + pub params: serde_json::Map, + /// Optional browser storage settings for the module. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(nested)] + pub storage: Option, +} + #[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] pub struct LegacyPrebidServerConfig { @@ -234,6 +339,13 @@ pub struct LegacyPrebidServerConfig { /// it in JavaScript. #[serde(default)] pub account_id: Option, + /// Prebid User ID modules that Trusted Server installs and keeps installed. + /// + /// Each entry is forwarded to Prebid.js verbatim; publisher-configured + /// entries with other names are preserved. Names must be unique. + #[serde(default)] + #[validate(nested, custom(function = "validate_unique_managed_user_id_names"))] + pub managed_user_ids: Vec, #[serde(default = "default_timeout_ms")] #[validate(range(min = 1, max = 60000))] pub timeout_ms: u32, @@ -393,6 +505,14 @@ pub struct PrebidIntegrationConfig { pub enabled: bool, #[serde(default)] pub account_id: Option, + /// Prebid User ID modules that Trusted Server installs and keeps installed. + /// + /// Each entry is forwarded to Prebid.js verbatim; publisher-configured + /// entries with other names are preserved. Names must be unique, and no two + /// names may resolve to the same Prebid User ID submodule. + #[serde(default)] + #[validate(nested, custom(function = "validate_unique_managed_user_id_names"))] + pub managed_user_ids: Vec, #[serde(default = "default_timeout_ms")] pub timeout_ms: u32, #[serde(default)] @@ -429,6 +549,7 @@ impl Default for PrebidIntegrationConfig { Self { enabled: default_enabled(), account_id: None, + managed_user_ids: Vec::new(), timeout_ms: default_timeout_ms(), debug: false, script_patterns: default_script_patterns(), @@ -454,6 +575,7 @@ impl From<&LegacyPrebidServerConfig> for PrebidIntegrationConfig { Self { enabled: config.enabled, account_id: config.account_id.clone(), + managed_user_ids: config.managed_user_ids.clone(), timeout_ms: config.timeout_ms, debug: config.debug, script_patterns: config.script_patterns.clone(), @@ -974,30 +1096,13 @@ impl PrebidIntegration { browser_config: &PrebidIntegrationConfig, plan: &AuctionPlan, ) -> Vec { - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct InjectedBrowserConfig<'a> { - account_id: &'a str, - timeout: u32, - debug: bool, - server_side_bidders: Vec<&'a str>, - #[serde(skip_serializing_if = "<[String]>::is_empty")] - client_side_bidders: &'a [String], - #[serde(skip_serializing_if = "<[String]>::is_empty")] - excluded_gam_ad_unit_path_suffixes: &'a [String], - } - - let payload = InjectedBrowserConfig { - account_id: browser_config.account_id.as_deref().unwrap_or_default(), - timeout: browser_config.timeout_ms, - debug: browser_config.debug, - server_side_bidders: if plan.enabled() { + let payload = InjectedPrebidClientConfig { + server_side_bidders: Some(if plan.enabled() { plan.browser_bidder_codes().collect() } else { Vec::new() - }, - client_side_bidders: &browser_config.client_side_bidders, - excluded_gam_ad_unit_path_suffixes: &browser_config.excluded_gam_ad_unit_path_suffixes, + }), + ..InjectedPrebidClientConfig::from(browser_config) }; let config_json = serialize_injected_prebid_config(&payload); @@ -1369,24 +1474,8 @@ impl IntegrationHeadInjector for PrebidIntegration { if let Some(inserts) = &self.planned_head_inserts { return inserts.clone(); } - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct InjectedPrebidClientConfig<'a> { - account_id: &'a str, - timeout: u32, - debug: bool, - bidders: &'a [String], - #[serde(skip_serializing_if = "<[String]>::is_empty")] - client_side_bidders: &'a [String], - #[serde(skip_serializing_if = "<[String]>::is_empty")] - excluded_gam_ad_unit_path_suffixes: &'a [String], - } - let payload = InjectedPrebidClientConfig { - account_id: self.config.account_id.as_deref().unwrap_or_default(), - timeout: self.config.timeout_ms, - debug: self.config.debug, - bidders: { + bidders: Some({ #[cfg(test)] { self.legacy_config @@ -1397,9 +1486,8 @@ impl IntegrationHeadInjector for PrebidIntegration { { &[] } - }, - client_side_bidders: &self.config.client_side_bidders, - excluded_gam_ad_unit_path_suffixes: &self.config.excluded_gam_ad_unit_path_suffixes, + }), + ..InjectedPrebidClientConfig::from(&self.config) }; let config_json = serialize_injected_prebid_config(&payload); @@ -1411,6 +1499,89 @@ impl IntegrationHeadInjector for PrebidIntegration { } } +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct InjectedManagedUserIdStorage<'a> { + #[serde(rename = "type")] + storage_type: PrebidUserIdStorageType, + name: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + expires: Option, + #[serde(skip_serializing_if = "Option::is_none")] + refresh_in_seconds: Option, +} + +impl<'a> From<&'a PrebidManagedUserIdStorage> for InjectedManagedUserIdStorage<'a> { + fn from(storage: &'a PrebidManagedUserIdStorage) -> Self { + Self { + storage_type: storage.storage_type, + name: &storage.name, + expires: storage.expires, + refresh_in_seconds: storage.refresh_in_seconds, + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct InjectedManagedUserId<'a> { + name: &'a str, + #[serde(skip_serializing_if = "serde_json::Map::is_empty")] + params: &'a serde_json::Map, + #[serde(skip_serializing_if = "Option::is_none")] + storage: Option>, +} + +impl<'a> From<&'a PrebidManagedUserIdConfig> for InjectedManagedUserId<'a> { + fn from(config: &'a PrebidManagedUserIdConfig) -> Self { + Self { + name: &config.name, + params: &config.params, + storage: config + .storage + .as_ref() + .map(InjectedManagedUserIdStorage::from), + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct InjectedPrebidClientConfig<'a> { + account_id: &'a str, + #[serde(skip_serializing_if = "Vec::is_empty")] + managed_user_ids: Vec>, + timeout: u32, + debug: bool, + #[serde(skip_serializing_if = "Option::is_none")] + bidders: Option<&'a [String]>, + #[serde(skip_serializing_if = "Option::is_none")] + server_side_bidders: Option>, + #[serde(skip_serializing_if = "<[String]>::is_empty")] + client_side_bidders: &'a [String], + #[serde(skip_serializing_if = "<[String]>::is_empty")] + excluded_gam_ad_unit_path_suffixes: &'a [String], +} + +impl<'a> From<&'a PrebidIntegrationConfig> for InjectedPrebidClientConfig<'a> { + fn from(config: &'a PrebidIntegrationConfig) -> Self { + Self { + account_id: config.account_id.as_deref().unwrap_or_default(), + managed_user_ids: config + .managed_user_ids + .iter() + .map(InjectedManagedUserId::from) + .collect(), + timeout: config.timeout_ms, + debug: config.debug, + bidders: None, + server_side_bidders: None, + client_side_bidders: &config.client_side_bidders, + excluded_gam_ad_unit_path_suffixes: &config.excluded_gam_ad_unit_path_suffixes, + } + } +} + /// Returns `true` when `params` is not a usable PBS bidder-params object — an /// empty object `{}` or any non-object value such as `null`. /// @@ -2515,8 +2686,8 @@ impl PrebidAuctionProvider { // Build user object — populate consent at both OpenRTB 2.6 top-level // and Prebid ext-based locations (dual placement). // In cookies_only mode, cookie-sourced consent travels through the - // forwarded Cookie header. KV/policy-sourced consent has no inbound - // cookie to forward, so carry it in the OpenRTB body instead. + // forwarded Cookie header. Policy-sourced consent has no inbound cookie + // to forward, so carry it in the OpenRTB body instead. let consent_ctx = request.user.consent.as_ref().filter(|ctx| { self.config.consent_forwarding.includes_body_consent() || !matches!(ctx.source, crate::consent::ConsentSource::Cookie) @@ -3487,6 +3658,7 @@ mod tests { enabled: true, server_url: "https://prebid.example".to_string(), account_id: Some("test-account".to_string()), + managed_user_ids: Vec::new(), timeout_ms: 1000, bidders: vec!["exampleBidder".to_string()], debug: false, @@ -3509,6 +3681,19 @@ mod tests { } } + fn valid_managed_user_id() -> PrebidManagedUserIdConfig { + PrebidManagedUserIdConfig { + name: "exampleId".to_string(), + params: serde_json::Map::from_iter([("pid".to_string(), json!("999"))]), + storage: Some(PrebidManagedUserIdStorage { + storage_type: PrebidUserIdStorageType::Cookie, + name: "example_env".to_string(), + expires: Some(15), + refresh_in_seconds: Some(1800), + }), + } + } + struct PredictOnlyBackend; impl PlatformBackend for PredictOnlyBackend { @@ -3879,6 +4064,219 @@ server_url = "https://prebid.example/openrtb2/auction" ); } #[test] + fn managed_user_ids_parse_with_opaque_params() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" +params = { pid = "999", notUse3P = false, nested = { depth = 2 } } + +[integrations.prebid.managed_user_ids.storage] +type = "html5" +name = "example_env" +expires = 30 +refresh_in_seconds = 3600 +"#, + ); + + let [entry] = config.managed_user_ids.as_slice() else { + panic!("should parse exactly one managed User ID entry"); + }; + assert_eq!(entry.name, "exampleId", "should preserve the module name"); + assert_eq!( + Json::Object(entry.params.clone()), + json!({"pid": "999", "notUse3P": false, "nested": {"depth": 2}}), + "should carry module parameters through without inspecting them" + ); + + let storage = entry.storage.as_ref().expect("should parse storage"); + assert_eq!( + storage.storage_type, + PrebidUserIdStorageType::Html5, + "should preserve the configured storage mechanism" + ); + assert_eq!(storage.name, "example_env", "should preserve storage key"); + assert_eq!(storage.expires, Some(30), "should preserve expiry"); + assert_eq!( + storage.refresh_in_seconds, + Some(3600), + "should preserve refresh interval" + ); + } + + #[test] + fn managed_user_ids_leave_prebid_defaults_in_place_when_unset() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" + +[integrations.prebid.managed_user_ids.storage] +name = "example_env" +"#, + ); + + let [entry] = config.managed_user_ids.as_slice() else { + panic!("should parse exactly one managed User ID entry"); + }; + assert!( + entry.params.is_empty(), + "should treat parameters as optional" + ); + + let storage = entry.storage.as_ref().expect("should parse storage"); + assert_eq!( + storage.storage_type, + PrebidUserIdStorageType::Cookie, + "should default to cookie storage" + ); + assert_eq!( + storage.expires, None, + "should leave Prebid's own expiry default in place" + ); + assert_eq!( + storage.refresh_in_seconds, None, + "should leave Prebid's own refresh default in place" + ); + } + + #[test] + fn managed_user_ids_allow_an_entry_without_storage() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" +"#, + ); + + let [entry] = config.managed_user_ids.as_slice() else { + panic!("should parse exactly one managed User ID entry"); + }; + assert!( + entry.storage.is_none(), + "should treat storage as optional for modules that need none" + ); + } + + #[test] + fn managed_user_ids_reject_invalid_values() { + for (name, entry_section) in [ + ("missing name", "params = { pid = \"999\" }"), + ("empty name", "name = \"\""), + ("padded name", "name = \" exampleId \""), + ("name with a space", "name = \"example id\""), + ( + "unknown entry field", + "name = \"exampleId\"\nunsupported = true", + ), + ( + "empty storage name", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"\"", + ), + ( + "missing storage name", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\ntype = \"cookie\"", + ), + ( + "zero expiry", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"example_env\"\nexpires = 0", + ), + ( + "zero refresh", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"example_env\"\nrefresh_in_seconds = 0", + ), + ( + "unknown storage mechanism", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"example_env\"\ntype = \"session\"", + ), + ( + "unknown storage field", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"example_env\"\nunsupported = true", + ), + ] { + let result = parse_prebid_toml_result(&format!( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +{entry_section} +"# + )); + + assert!(result.is_err(), "should reject {name}"); + } + } + + #[test] + fn managed_user_ids_reject_a_repeated_module_name() { + let result = parse_prebid_toml_result( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" +params = { pid = "1" } + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" +params = { pid = "2" } +"#, + ); + + assert!( + result.is_err(), + "should reject the same module configured twice" + ); + } + + #[test] + fn managed_user_ids_accept_distinct_module_names() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" + +[[integrations.prebid.managed_user_ids]] +name = "otherExampleId" +"#, + ); + + assert_eq!( + config.managed_user_ids.len(), + 2, + "should keep every distinctly named module" + ); + } + + #[test] + fn managed_user_ids_default_to_none_configured() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" +"#, + ); + + assert!( + config.managed_user_ids.is_empty(), + "should manage no User ID modules by default" + ); + } + #[test] fn excluded_gam_ad_unit_path_suffixes_reject_invalid_values() { for (suffix, expected_message) in [ ("", "must not be empty"), @@ -4903,6 +5301,173 @@ external_bundle_sri = "sha384-AAAA" assert!(!config.debug); } + #[test] + fn planned_registration_injects_managed_user_ids() { + let mut settings = make_settings(); + settings + .integrations + .insert_config( + "prebid", + &json!({ + "enabled": true, + "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", + "managed_user_ids": [{ + "name": "exampleId", + "params": {"nested": {"value": ""}}, + "storage": {"name": "example_env", "refresh_in_seconds": 3600} + }, {"name": "anotherId"}] + }), + ) + .expect("should configure prebid"); + let plan = + crate::auction::compile_auction_plan(&settings).expect("should compile auction plan"); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + for enabled in [true, false] { + let registration = register_for_plan(&settings, &plan.clone().with_enabled(enabled)) + .expect("should register prebid") + .expect("should enable prebid"); + let inserts = registration.head_injectors[0].head_inserts(&ctx); + let script = &inserts[0]; + assert!( + script.contains(r#""managedUserIds":[{"name":"exampleId""#), + "should inject managed IDs: {script}" + ); + assert!( + script.contains(r#""refreshInSeconds":3600"#), + "should use browser storage keys: {script}" + ); + assert!( + script.contains(r#"{"name":"anotherId"}"#), + "should omit unset fields: {script}" + ); + assert!( + !script.contains("\""), + "should escape script breakout: {script}" + ); + } + } + + #[test] + fn head_injector_includes_managed_user_ids() { + let mut config = base_config(); + config.managed_user_ids = vec![PrebidManagedUserIdConfig { + name: "exampleId".to_string(), + params: serde_json::Map::from_iter([ + ("pid".to_string(), json!("999")), + ("notUse3P".to_string(), json!(true)), + ]), + storage: Some(PrebidManagedUserIdStorage { + storage_type: PrebidUserIdStorageType::Html5, + name: "example_env".to_string(), + expires: Some(30), + refresh_in_seconds: Some(3600), + }), + }]; + let integration = PrebidIntegration::new(config); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&ctx); + let script = &inserts[0]; + + assert!( + script.contains( + r#""managedUserIds":[{"name":"exampleId","params":{"notUse3P":true,"pid":"999"},"storage":{"type":"html5","name":"example_env","expires":30,"refreshInSeconds":3600}}]"# + ), + "should inject the managed User ID entry verbatim: {script}" + ); + } + + #[test] + fn head_injector_omits_optional_managed_user_id_fields_when_unset() { + let mut config = base_config(); + config.managed_user_ids = vec![PrebidManagedUserIdConfig { + name: "exampleId".to_string(), + params: serde_json::Map::new(), + storage: None, + }]; + let integration = PrebidIntegration::new(config); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&ctx); + let script = &inserts[0]; + + assert!( + script.contains(r#""managedUserIds":[{"name":"exampleId"}]"#), + "should omit empty parameters and absent storage: {script}" + ); + } + + #[test] + fn head_injector_omits_managed_user_ids_when_none_configured() { + let integration = PrebidIntegration::new(base_config()); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&ctx); + let script = &inserts[0]; + + assert!( + !script.contains("managedUserIds"), + "should omit managed User IDs when none are configured: {script}" + ); + } + + #[test] + fn head_injector_escapes_script_breakout_in_managed_user_ids() { + let mut config = base_config(); + config.managed_user_ids = vec![PrebidManagedUserIdConfig { + params: serde_json::Map::from_iter([( + "pid".to_string(), + json!("1"), + )]), + ..valid_managed_user_id() + }]; + let integration = PrebidIntegration::new(config); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&ctx); + let script = &inserts[0]; + + assert!( + script.contains(r#""pid":"1\u003c/script>\u003cscript>alert(1)\u003c/script>""#), + "should retain the escaped module parameter: {script}" + ); + assert_eq!( + script.matches("").count(), + 1, + "should contain only the legitimate outer closing script tag" + ); + } + #[test] fn head_injector_includes_excluded_gam_ad_unit_path_suffixes() { let mut config = base_config(); @@ -5378,16 +5943,16 @@ external_bundle_sri = "sha384-AAAA" } #[test] - fn to_openrtb_includes_kv_consent_when_cookies_only_has_no_cookie_to_forward() { + fn to_openrtb_includes_policy_default_consent_when_cookies_only_has_no_cookie_to_forward() { let mut config = base_config(); config.consent_forwarding = ConsentForwardingMode::CookiesOnly; let provider = PrebidAuctionProvider::new(config); let mut auction_request = create_test_auction_request(); auction_request.user.consent = Some(ConsentContext { - raw_tc_string: Some("BOkv-backed-consent-string".to_string()), + raw_tc_string: Some("BOpolicy-consent-string".to_string()), raw_us_privacy: Some("1YNN".to_string()), gdpr_applies: true, - source: ConsentSource::KvStore, + source: ConsentSource::PolicyDefault, ..Default::default() }); @@ -5408,15 +5973,15 @@ external_bundle_sri = "sha384-AAAA" assert_eq!( openrtb.user.as_ref().and_then(|u| u.consent.as_deref()), - Some("BOkv-backed-consent-string"), - "cookies_only should fall back to body consent when consent came from KV" + Some("BOpolicy-consent-string"), + "cookies_only should carry policy-sourced consent in the body" ); let regs = openrtb.regs.as_ref().expect("should include consent regs"); assert_eq!(regs.gdpr, Some(true), "should carry GDPR applicability"); assert_eq!( regs.us_privacy.as_deref(), Some("1YNN"), - "should carry non-cookie consent strings from KV" + "should carry policy-sourced consent strings" ); } diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d858e5a12..996b20fd4 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -689,7 +689,17 @@ impl IntegrationRegistrationBuilder { } } -type RouteValue = (Arc, &'static str); +/// Proxy handler, integration id, and the registered route pattern (kept so +/// telemetry can label responses with the integration-defined literal, e.g. +/// `/integrations/prebid/*`, instead of deriving anything from the request +/// path). +type RouteValue = (Arc, &'static str, String); + +/// A test-constructor route entry: method, path pattern, and the proxy with +/// its integration id ([`IntegrationRegistry::from_routes`] fills the +/// pattern into [`RouteValue`] itself). +#[cfg(test)] +type RouteEntry<'a> = (Method, &'a str, (Arc, &'static str)); struct IntegrationRegistryInner { // Method-specific routers for O(log n) lookups @@ -835,7 +845,11 @@ impl IntegrationRegistry { for proxy in registration.proxies { for route in proxy.routes() { - let value = (proxy.clone(), registration.integration_id); + let value = ( + proxy.clone(), + registration.integration_id, + route.path.clone(), + ); // Convert /* wildcard to matchit's {*rest} syntax let matchit_path = if route.path.ends_with("/*") { @@ -932,6 +946,27 @@ impl IntegrationRegistry { self.find_route(method, path).is_some() } + /// The registered route pattern matched by `method` and `path`, if any. + /// + /// Patterns are integration-defined literals (for example + /// `/integrations/prebid/*`), so they are bounded and content-free and + /// safe to store as a telemetry dimension, unlike the request path. + #[must_use] + pub fn matched_route_pattern(&self, method: &Method, path: &str) -> Option<&str> { + self.find_route(method, path).map(|value| value.2.as_str()) + } + + /// Return true when at least one integration request filter is + /// registered. + /// + /// Adapters use this to decide whether to record a request-filter phase + /// timing span, so unconfigured deployments (no request filters) omit + /// that entry from observability output entirely. + #[must_use] + pub fn has_request_filters(&self) -> bool { + !self.inner.request_filters.is_empty() + } + /// Run pre-routing request filters. /// /// Request header mutations are applied immediately so later filters and @@ -1007,7 +1042,7 @@ impl IntegrationRegistry { services, mut req, } = input; - if let Some((proxy, _)) = self.find_route(method, path) { + if let Some((proxy, _, _)) = self.find_route(method, path) { // Organic proxy handler: generate if needed (best effort). // Only generate for document navigations — subresource requests // may lack consent signals such as the Sec-GPC header. @@ -1327,7 +1362,7 @@ impl IntegrationRegistry { /// # Panics /// /// Panics if route registration fails due to duplicate or invalid paths. - pub fn from_routes(routes: Vec<(Method, &str, RouteValue)>) -> Self { + pub fn from_routes(routes: Vec>) -> Self { let mut get_router = Router::new(); let mut post_router = Router::new(); let mut put_router = Router::new(); @@ -1336,7 +1371,8 @@ impl IntegrationRegistry { let mut head_router = Router::new(); let mut options_router = Router::new(); - for (method, path, value) in routes { + for (method, path, (proxy, integration_id)) in routes { + let value: RouteValue = (proxy, integration_id, path.to_owned()); // Convert /* wildcard to matchit's {*rest} syntax let matchit_path = if path.ends_with("/*") { format!( diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 76621baf7..b893a9647 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -31,6 +31,7 @@ ) )] +pub mod access_telemetry; pub(crate) mod asset_image_optimizer; pub mod auction; pub mod auction_config_types; @@ -61,13 +62,13 @@ pub mod proxy; pub mod publisher; pub mod redacted; pub mod request_signing; +pub mod request_timing; pub mod response_privacy; pub mod rsc_flight; pub(crate) mod s3_sigv4; pub mod secret_resolution; pub mod settings; pub mod settings_data; -pub mod storage; pub mod streaming_processor; pub mod streaming_replacer; pub mod test_support; diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index ad3e5350c..781089023 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -216,8 +216,6 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("s3_sigv4.rs", include_str!("s3_sigv4.rs")), ("settings.rs", include_str!("settings.rs")), ("settings_data.rs", include_str!("settings_data.rs")), - ("storage/kv_store.rs", include_str!("storage/kv_store.rs")), - ("storage/mod.rs", include_str!("storage/mod.rs")), ( "streaming_processor.rs", include_str!("streaming_processor.rs"), diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 2553229a4..4454d25cc 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -43,6 +43,7 @@ mod template_assembly; mod template_cache; #[cfg(test)] pub(crate) mod test_support; +mod timed_kv; mod traits; mod types; @@ -72,6 +73,7 @@ pub use template_cache::{ TemplateEntry, TemplateMetadata, TemplateMetadataEncodeError, UnavailableTemplateCache, VaryHeaderValues, VarySpec, }; +pub use timed_kv::TimedKvStore; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 70eb55a9a..0e1b9f8e7 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -593,6 +593,7 @@ impl PlatformHttpClient for StubHttpClient { .pop_front() .ok_or_else(|| Report::new(PlatformError::HttpClient))?; + let stream_response = stream_response || response.stream_body; let edge_response = build_stub_pending_response( StubPendingResponse { backend_name: request.backend_name, @@ -600,7 +601,7 @@ impl PlatformHttpClient for StubHttpClient { body: response.body, headers: response.headers, }, - stream_response || response.stream_body, + stream_response, request_is_head, )?; diff --git a/crates/trusted-server-core/src/platform/timed_kv.rs b/crates/trusted-server-core/src/platform/timed_kv.rs new file mode 100644 index 000000000..05554f242 --- /dev/null +++ b/crates/trusted-server-core/src/platform/timed_kv.rs @@ -0,0 +1,253 @@ +//! Latency-only timing decorator for KV store handles. +//! +//! [`TimedKvStore`] wraps an inner store plus a [`RequestTimings`] handle and +//! records [`Phase::EcKv`] around every call. It implements both +//! [`PlatformKvStore`] (for consent-store access obtained through +//! [`RuntimeServices`](super::RuntimeServices)) and [`EcKvStore`] (for +//! [`KvIdentityGraph`](crate::ec::kv::KvIdentityGraph) construction sites), +//! because no single existing abstraction covers the whole `ts-kv` taxonomy: +//! EC graph operations go through [`EcKvStore`] while consent persistence +//! uses [`PlatformKvStore`] directly. +//! +//! The decorator measures store-call latency only: it never reads, parses, +//! or logs any value passing through it. + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use edgezero_core::key_value_store::{KvError, KvPage, KvStore as PlatformKvStore}; +use error_stack::Report; + +use crate::ec::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteOutcome}; +use crate::error::TrustedServerError; +use crate::request_timing::{Phase, RequestTimings}; + +/// Wraps `inner` plus a [`RequestTimings`] handle, recording [`Phase::EcKv`] +/// around every store call made through it. +pub struct TimedKvStore { + /// The wrapped store handle. + inner: S, + /// The request's phase-timing collector. + timings: RequestTimings, +} + +impl TimedKvStore { + /// Creates a decorator around `inner` that records into `timings`. + #[must_use] + pub fn new(inner: S, timings: RequestTimings) -> Self { + Self { inner, timings } + } +} + +#[async_trait(?Send)] +impl PlatformKvStore for TimedKvStore> { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + let _span = self.timings.span(Phase::EcKv); + self.inner.get_bytes(key).await + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + let _span = self.timings.span(Phase::EcKv); + self.inner.put_bytes(key, value).await + } + + // Forwarded explicitly: the trait's default body falls back to + // `get_bytes`, which would silently downgrade a backend's cheap + // metadata-only existence probe (the Spin adapter has one) into a full + // value transfer just because the store was decorated. + async fn exists(&self, key: &str) -> Result { + let _span = self.timings.span(Phase::EcKv); + self.inner.exists(key).await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + let _span = self.timings.span(Phase::EcKv); + self.inner.put_bytes_with_ttl(key, value, ttl).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + let _span = self.timings.span(Phase::EcKv); + self.inner.delete(key).await + } + + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + let _span = self.timings.span(Phase::EcKv); + self.inner.list_keys_page(prefix, cursor, limit).await + } +} + +impl EcKvStore for TimedKvStore { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + let _span = self.timings.span(Phase::EcKv); + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + let _span = self.timings.span(Phase::EcKv); + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + let _span = self.timings.span(Phase::EcKv); + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + let _span = self.timings.span(Phase::EcKv); + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + let _span = self.timings.span(Phase::EcKv); + self.inner.delete(key) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration as StdDuration; + + use super::*; + use crate::ec::kv_backend::test_support::InMemoryEcKv; + + #[test] + fn ec_kv_store_operations_accumulate_into_ec_kv_phase() { + let timings = RequestTimings::new(); + let store = TimedKvStore::new(InMemoryEcKv::new("test-store"), timings.clone()); + + store + .insert( + "key-a", + EcKvWrite { + body: "{}", + metadata: "{}", + ttl: StdDuration::from_secs(60), + mode: crate::ec::kv_backend::EcKvWriteMode::Add, + }, + ) + .expect("should insert into the in-memory store"); + store.lookup("key-a").expect("should read back the entry"); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "should record Phase::EcKv across both store calls" + ); + } + + #[test] + fn exists_delegates_to_the_inner_store_not_get_bytes() { + // A stub whose `exists` answer contradicts its `get_bytes` answer: + // if the decorator fell back to the trait's get-and-discard default + // body, this would return `false`. + struct ExistsOnlyStore; + + #[async_trait::async_trait(?Send)] + impl PlatformKvStore for ExistsOnlyStore { + async fn get_bytes(&self, _key: &str) -> Result, KvError> { + Ok(None) + } + async fn put_bytes(&self, _key: &str, _value: Bytes) -> Result<(), KvError> { + Ok(()) + } + async fn put_bytes_with_ttl( + &self, + _key: &str, + _value: Bytes, + _ttl: StdDuration, + ) -> Result<(), KvError> { + Ok(()) + } + async fn delete(&self, _key: &str) -> Result<(), KvError> { + Ok(()) + } + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage { + keys: Vec::new(), + cursor: None, + }) + } + async fn exists(&self, _key: &str) -> Result { + Ok(true) + } + } + + let timings = RequestTimings::new(); + let inner: Arc = Arc::new(ExistsOnlyStore); + let store = TimedKvStore::new(inner, timings.clone()); + + let exists = futures::executor::block_on(store.exists("key")) + .expect("should forward the existence probe"); + assert!( + exists, + "should delegate to the inner exists, not the get_bytes default body" + ); + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "should time the existence probe like any other store operation" + ); + } + + #[test] + fn store_name_is_not_timed() { + let timings = RequestTimings::new(); + let store = TimedKvStore::new(InMemoryEcKv::new("test-store"), timings.clone()); + + assert_eq!(store.store_name(), "test-store"); + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_none(), + "store_name is a metadata accessor, not a store operation" + ); + } + + #[test] + fn platform_kv_store_operations_accumulate_into_ec_kv_phase() { + let timings = RequestTimings::new(); + let inner: Arc = Arc::new(crate::platform::UnavailableKvStore); + let store = TimedKvStore::new(inner, timings.clone()); + + // UnavailableKvStore errors on every call; the decorator still times + // the attempt regardless of outcome. + futures::executor::block_on(async { + let _ = store.get_bytes("key").await; + let _ = store.put_bytes("key", Bytes::from_static(b"value")).await; + }); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "should record Phase::EcKv even when the inner store errors" + ); + } +} diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 05daf0b4e..f5874576e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -55,9 +55,12 @@ use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, }; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; -use crate::creative_opportunities::{AssemblyMode, CreativeOpportunitiesConfig}; +use crate::creative_opportunities::{ + AdStackGateInput, AssemblyMode, CreativeOpportunitiesConfig, RuntimeAdStackExpected, + evaluate_ad_stack_gate, +}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; @@ -70,6 +73,7 @@ use crate::platform::{ contains_publisher_esi_directive, }; use crate::price_bucket::{PriceGranularity, price_bucket}; +use crate::request_timing::{AuctionWaitPlacement, Phase, RequestTimings}; use crate::response_privacy::{ apply_inactive_ad_stack_browser_cache_policy, cache_control_forbids_shared_storage, enforce_synthesized_html_cache_privacy, enforce_terminal_private_cache_privacy, @@ -90,21 +94,44 @@ const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); const HEADER_X_TS_TEMPLATE_CACHE: &str = "x-ts-template-cache"; const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; -#[derive(Clone, Copy, PartialEq, Eq)] -enum TemplateCacheResponseState { +/// Outcome of a template-cache lookup/store attempt for one response. +/// +/// Set on every response that passes through the assembly pipeline via +/// [`set_template_cache_response_state`], which writes both the +/// `x-ts-template-cache` response header and this same value as a typed +/// response extension, so the two can never drift. Access telemetry reads +/// the extension rather than the header, since operator-configured response +/// headers can override a managed header but cannot touch extensions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TemplateCacheResponseState { + /// The cached template was found and reused. Hit, + /// No cached template existed; the cache store is reserved for this + /// content type. MissReserved, + /// No cached template existed; one was stored after assembly. MissStored, + /// No cached template existed; storing the freshly assembled template + /// failed. MissStoreError, + /// The request bypassed the cache lookup. BypassRequest, + /// The response bypassed the cache store. BypassResponse, + /// The response's content type is not supported by the template cache. Unsupported, + /// The cached template entry was invalid and could not be reused. Invalid, + /// A backend error prevented the cache lookup or store. BackendError, } impl TemplateCacheResponseState { - const fn as_str(self) -> &'static str { + /// Renders this variant as the string written to the + /// `x-ts-template-cache` header and the `template_cache_state` access + /// telemetry column. + #[must_use] + pub const fn as_str(self) -> &'static str { match self { Self::Hit => "hit", Self::MissReserved => "miss-reserved", @@ -127,6 +154,7 @@ fn set_template_cache_response_state( HEADER_X_TS_TEMPLATE_CACHE, HeaderValue::from_static(state.as_str()), ); + response.extensions_mut().insert(state); } #[derive(Clone, Copy, PartialEq, Eq)] @@ -1616,6 +1644,12 @@ pub struct OwnedProcessResponseParams { /// rescanned from the output, which cannot tell a `nonce` attribute from the same /// word inside a script. pub(crate) csp_nonce_observed: Option>, + /// Per-request phase-timing handle, carried into the streaming/buffered + /// finalizers so the `` seam wait can be recorded with the right + /// [`AuctionWaitPlacement`]. Cheap to clone (an `Arc` handle); a request that + /// never attached one to its extensions gets a fresh, unattached collector + /// that nothing ever renders. + pub(crate) timings: RequestTimings, } /// Response-authorized template cache insert inputs. The key is built before origin lookup; the @@ -1859,6 +1893,8 @@ pub async fn buffer_publisher_response_async( ¶ms.request_scheme, ¶ms.request_host, ), + timings: params.timings.clone(), + placement: AuctionWaitPlacement::PreHeader, }, ) .await; @@ -2018,6 +2054,7 @@ fn build_template_assembly_params( request_scheme: &str, price_granularity: PriceGranularity, ad_bids_state: AdBidsState, + timings: RequestTimings, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { csp_nonce_observed: None, @@ -2040,6 +2077,7 @@ fn build_template_assembly_params( price_granularity, gpt_diagnostics: None, suppress_datadome_client_side_tag: false, + timings, } } @@ -2382,6 +2420,8 @@ pub async fn publisher_response_into_streaming_response( ¶ms.request_scheme, ¶ms.request_host, ), + timings: params.timings.clone(), + placement: AuctionWaitPlacement::InStream, }, ) .await; @@ -2500,6 +2540,7 @@ pub async fn publisher_response_into_streaming_response( &orchestrator, &services, &settings, + AuctionWaitPlacement::InStream, ) .await; // Collection reached a terminal result; disarm only now @@ -2521,6 +2562,8 @@ pub async fn publisher_response_into_streaming_response( ¶ms.request_scheme, ¶ms.request_host, ), + timings: params.timings.clone(), + placement: AuctionWaitPlacement::InStream, }; while let Some(step) = hold_step_next_chunk( @@ -2856,6 +2899,7 @@ pub async fn stream_publisher_body_async( orchestrator, services, settings, + AuctionWaitPlacement::PreHeader, ) .await; if body.is_stream() { @@ -2922,6 +2966,8 @@ pub async fn stream_publisher_body_async( services, settings, request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + timings: params.timings.clone(), + placement: AuctionWaitPlacement::PreHeader, }, }, ) @@ -2952,13 +2998,27 @@ fn request_head_snapshot(req: &Request) -> Request { snapshot } -fn should_preload_ec_snapshot( +#[derive(Clone, Copy)] +struct EcSnapshotPreloadInput { is_navigation: bool, is_get: bool, has_ec_id: bool, has_kv: bool, -) -> bool { - is_navigation && is_get && has_ec_id && has_kv + marker_valid: bool, + auction_needs_row: bool, + eid_cookie_may_need_persistence: bool, + privacy_needs_row: bool, + snapshot_already_read: bool, +} + +fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { + let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; + let marker_can_skip = input.marker_valid + && !input.auction_needs_row + && !input.eid_cookie_may_need_persistence + && !input.privacy_needs_row + && !input.snapshot_already_read; + eligible && !marker_can_skip } /// Rewrites a downstream request into an outbound publisher-origin request. @@ -3052,15 +3112,12 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -#[derive(Debug, Clone, Copy)] -struct ServerSideAdStackConfig { - /// Dedicated `[creative_opportunities].enabled` switch. - ad_templates_enabled: bool, - /// Global `[auction].enabled` gate used by publisher/page-bids flows. - auction_enabled: bool, -} - /// Returns whether request-scoped signals permit an ad-eligible navigation. +/// +/// This is the request half of the shared ad-stack gate: the configuration +/// halves (`matched_slots`, the kill switches) are deliberately absent, because +/// the cache policy for a structurally inactive template must distinguish a +/// page that no request could activate from one this particular request skipped. fn is_server_side_ad_eligible_navigation( is_get: bool, is_navigation: bool, @@ -3071,29 +3128,6 @@ fn is_server_side_ad_eligible_navigation( is_get && is_navigation && !is_prefetch && !is_bot && consent_allows_auction } -/// Returns true only when the publisher should inject and run server-side ad templates. -/// -/// This includes auction dispatch plus initial ad-slot injection. -fn should_run_server_side_ad_stack( - is_get: bool, - is_navigation: bool, - is_prefetch: bool, - is_bot: bool, - has_matched_slots: bool, - consent_allows_auction: bool, - config: ServerSideAdStackConfig, -) -> bool { - is_server_side_ad_eligible_navigation( - is_get, - is_navigation, - is_prefetch, - is_bot, - consent_allows_auction, - ) && config.ad_templates_enabled - && has_matched_slots - && config.auction_enabled -} - /// Write winning bids from an auction result into the shared `ad_bids_state` lock. /// Build the request origin (`scheme://host`, where `host` includes any port) /// used to emit absolute first-party URLs in inline creatives. Returns an empty @@ -3121,6 +3155,40 @@ fn request_origin(scheme: &str, host: &str) -> String { /// JSON for every non-empty map; `serde_json::from_str` failed and `unwrap_or_default()` /// turned the failure into `{}`. Shared modes therefore served **zero bids**, silently, /// on every request that had any. Every fixture had empty bids, so nothing caught it. +#[derive(Clone, Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct BrowserAuctionDiagnostics { + #[serde(skip_serializing_if = "Option::is_none")] + auction_dispatched_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auction_resolved_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auction_committed_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auction_wait_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auction_wait_placement: Option<&'static str>, +} + +impl BrowserAuctionDiagnostics { + fn from_request_timings(timings: &RequestTimings) -> Option { + let snapshot = timings.snapshot(); + snapshot.auction_dispatched_ms?; + Some(Self { + auction_dispatched_ms: snapshot.auction_dispatched_ms, + auction_resolved_ms: snapshot.auction_resolved_ms, + auction_committed_ms: snapshot.auction_committed_ms, + auction_wait_ms: snapshot.auction_wait_ms, + auction_wait_placement: snapshot.auction_wait_placement.map( + |placement| match placement { + AuctionWaitPlacement::PreHeader => "pre_header", + AuctionWaitPlacement::InStream => "in_stream", + }, + ), + }) + } +} + #[derive(Clone, Default)] pub(crate) struct AdBidsState { /// Rendered bids `` sequences inside the string. pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { + build_bids_script_with_diagnostics(bid_map, None) +} + +fn build_bids_script_with_diagnostics( + bid_map: &serde_json::Map, + auction_diagnostics: Option<&BrowserAuctionDiagnostics>, +) -> String { let json = serde_json::to_string(bid_map) .expect("serde_json::to_string of Map should be infallible"); let escaped = html_escape_for_script(&json); @@ -5512,6 +5720,23 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map(function(){{\ +var t=window.tsjs=window.tsjs||{{}};\ +var b=JSON.parse(\"{}\");\ +var d=JSON.parse(\"{}\");\ +var s=t.scheduleInitialAdInit;\ +if(typeof s===\"function\")s(b,void 0,d);\ +else{{t.bids=b;t.auctionDiagnostics=d;}}\ +}})();", + escaped, + html_escape_for_script(&diagnostics) + ); + } + format!( "", + html_escape_for_script(slots_json), + html_escape_for_script(&bids), + html_escape_for_script(&diagnostics) + ); + } + format!( "".to_string(), + ..valid_liveramp_config() + }); + + // Build the normal context. Assert the injected payload contains + // `1<\/script>")` + // has count 1: only the insert's legitimate outer closing tag remains. + // This test may build the invalid value directly because it exercises the + // serializer's defense in depth rather than TOML validation. +} +``` + +- [ ] **Step 6: Run both head-injection tests and verify they fail** + +Run: + +```bash +cargo test-fastly head_injector_includes_liveramp_config +cargo test-fastly head_injector_escapes_script_breakout_in_liveramp_config +``` + +Expected: both fail because the injected payload has no `liveRamp` property or +escaped LiveRamp Placement ID. + +- [ ] **Step 7: Inject a browser-specific serialization shape** + +Inside `IntegrationHeadInjector::head_inserts`, define a borrowed injected +shape so TOML remains snake_case while browser JSON is camelCase: + +```rust +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct InjectedPrebidLiveRampConfig<'a> { + placement_id: &'a str, + not_use_3p: bool, + storage_type: PrebidLiveRampStorageType, + expires_days: u16, + refresh_in_seconds: u32, +} +``` + +Add a skipped-when-absent `live_ramp` field to +`InjectedPrebidClientConfig`, map `self.config.liveramp.as_ref()` into the +borrowed shape, and retain the existing ` { + const spec = getAdapterSpec() + mockGetUserIdsAsEids.mockReturnValue([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }, + ]) + + const request = spec.buildRequests([ + { + adUnitCode: 'div-gpt-1', + bidId: 'bid-1', + sizes: [[300, 250]], + bidder: 'trustedServer', + params: {}, + }, + ]) + + expect(JSON.parse(request.data).eids).toEqual([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }, + ]) +}) +``` + +Add a logging assertion using a sentinel envelope and spies on `log.debug`, +`log.info`, `log.warn`, and `log.error`; no logged argument may contain the +sentinel. + +- [ ] **Step 4: Write the failing real-artifact test before implementation** + +Change the bundle built in `prebid-artifact-integration.test.mjs` to include +both `sharedIdSystem` and `identityLinkIdSystem`. Inject `liveRamp` before +evaluating the served shim, then assert after shim evaluation: + +```javascript +const configuredUserIds = pageWindow.pbjs.getConfig('userSync.userIds') +expect(configuredUserIds).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + storage: expect.objectContaining({ name: 'idl_env' }), + }), + ]) +) +``` + +Retain the current real `/auction` request assertion. Network remains stubbed; +the test must not contact LiveRamp. + +After the initial managed-entry assertion, call the real public merge API and +prove it cannot append a publisher-owned duplicate: + +```javascript +pageWindow.pbjs.mergeConfig({ + userSync: { + userIds: [ + { name: 'sharedId' }, + { name: 'identityLink', params: { pid: 'publisher-value' } }, + ], + }, +}) + +const mergedUserIds = pageWindow.pbjs.getConfig('userSync.userIds') +expect(mergedUserIds.filter(({ name }) => name === 'identityLink')).toEqual([ + expect.objectContaining({ + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + }), +]) +expect(mergedUserIds).toEqual( + expect.arrayContaining([expect.objectContaining({ name: 'sharedId' })]) +) +``` + +- [ ] **Step 5: Run the focused unit and artifact suites and verify failures** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/prebid/index.test.ts +npx vitest run test/prebid-artifact-integration.test.mjs +``` + +Expected: LiveRamp ownership tests fail because the injected shape and public +configuration normalizers do not exist, and the artifact test fails because no +managed entry is installed. The transport-only characterizations may already +pass; retain them as evidence of the pre-existing path. + +- [ ] **Step 6: Add the injected TypeScript types and constants** + +Add: + +```typescript +interface InjectedLiveRampConfig { + placementId: string + notUse3P: boolean + storageType: 'cookie' | 'html5' + expiresDays: number + refreshInSeconds: number +} + +interface InjectedPrebidConfig { + // Existing fields omitted. + liveRamp?: InjectedLiveRampConfig +} + +const IDENTITY_LINK_CONFIG_NAME = 'identityLink' +const IDENTITY_LINK_STORAGE_NAME = 'idl_env' +const LIVE_RAMP_SET_CONFIG_SENTINEL = '__tsLiveRampSetConfigInstalled' +``` + +Keep this internal to the Prebid module; do not add a new global API. + +- [ ] **Step 7: Extract reusable User ID list parsing** + +Refactor the shape handling currently embedded in +`configuredUserIdNamesFromConfig` into a helper that returns validated entry +objects from any of these inputs: + +- the direct array returned by `getConfig('userSync.userIds')`; +- `{ userSync: { userIds: [...] } }`; +- `{ userIds: [...] }`. + +Use explicit record and entry guards. A valid entry is a non-array object with +a non-empty string `name`; filter malformed members rather than forwarding +them. The parser returns an empty array for malformed containers. Keep a +separate `hasUserIdsPath(config)` predicate equivalent to: + +```typescript +function hasUserIdsPath(config: unknown): config is { + userSync: Record & { userIds: unknown } +} { + return ( + isRecord(config) && + isRecord(config.userSync) && + Object.prototype.hasOwnProperty.call(config.userSync, 'userIds') + ) +} +``` + +This distinction is required: an absent path passes through unchanged, while +an explicitly empty `userIds: []` is normalized to the managed entry. Update +`configuredUserIdNamesFromConfig` to derive names from that helper so +diagnostics and LiveRamp normalization agree on supported shapes. + +- [ ] **Step 8: Implement the managed entry and normalizer** + +Implement small focused helpers equivalent to: + +```typescript +function liveRampUserId( + config: InjectedLiveRampConfig +): Record { + return { + name: IDENTITY_LINK_CONFIG_NAME, + params: { pid: config.placementId, notUse3P: config.notUse3P }, + storage: { + type: config.storageType, + name: IDENTITY_LINK_STORAGE_NAME, + expires: config.expiresDays, + refreshInSeconds: config.refreshInSeconds, + }, + } +} + +function withManagedLiveRampUserId( + config: Record, + managedEntry: Record +): Record { + if (!hasUserIdsPath(config)) return config + + const retained = configuredUserIdEntries(config.userSync.userIds).filter( + (entry) => entry.name !== IDENTITY_LINK_CONFIG_NAME + ) + return { + ...config, + userSync: { + ...config.userSync, + userIds: [...retained, managedEntry], + }, + } +} +``` + +`configuredUserIdEntries` must support the three shapes from Step 7 and return +fresh arrays. The spread operations preserve top-level properties and sibling +`userSync` properties. Do not mutate publisher-owned arrays or objects in +place. + +- [ ] **Step 9: Install idempotent public configuration guards before queue processing** + +In `installPrebidNpm`, after confirming the real Prebid API and before the +existing base configuration and `processQueue()` call: + +1. If injected `liveRamp` is absent, do nothing. +2. Capture and bind the current `pbjs.setConfig` and optional + `pbjs.mergeConfig`. +3. Replace both public APIs with wrappers that share one normalizer for calls + containing `userSync.userIds`; pass all other calls through unchanged. +4. Mark the Prebid object with the sentinel so installation cannot stack. +5. Read effective User ID entries through `pbjs.getConfig`. +6. Call the wrapper synchronously with the effective list, producing one + managed entry before any queued auction. +7. Leave both wrappers installed across `processQueue()` and later calls. + +Use logic equivalent to: + +```typescript +const managedPbjs = pbjs as typeof pbjs & Record +if (managedPbjs[LIVE_RAMP_SET_CONFIG_SENTINEL] !== true) { + const originalSetConfig = pbjs.setConfig.bind(pbjs) + const originalMergeConfig = pbjs.mergeConfig?.bind(pbjs) + const managedEntry = liveRampUserId(config.liveRamp) + + const normalizePublisherConfig = (publisherConfig) => { + let nextConfig = publisherConfig + try { + if (hasUserIdsPath(publisherConfig)) { + nextConfig = withManagedLiveRampUserId(publisherConfig, managedEntry) + } + } catch { + log.error('Prebid LiveRamp configuration could not be normalized') + } + return nextConfig + } + + pbjs.setConfig = (publisherConfig) => + originalSetConfig(normalizePublisherConfig(publisherConfig)) + if (originalMergeConfig) { + pbjs.mergeConfig = (publisherConfig) => + originalMergeConfig(normalizePublisherConfig(publisherConfig)) + } + managedPbjs[LIVE_RAMP_SET_CONFIG_SENTINEL] = true + + const effective = configuredUserIdEntries(pbjs.getConfig('userSync.userIds')) + pbjs.setConfig({ userSync: { userIds: effective } }) +} +``` + +Adapt the callback and return types to the repository's actual `pbjs` typing. +Each original method is invoked exactly once, its return value is preserved, +and normalization errors never log values. The sentinel lives on `pbjs`, not +on the page-level shim state: a test must deliberately reset only +`__tsjsPrebidShimInstalled`, reinstall, and prove both wrapper references are +unchanged and publisher calls are normalized once. + +- [ ] **Step 10: Run focused unit and artifact tests and make them pass** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/prebid/index.test.ts +npx vitest run test/prebid-artifact-integration.test.mjs +``` + +Expected: unit tests pass; the real bundle advertises both User ID modules, the +shim configures one `identityLink` entry, a real `mergeConfig` call retains +exactly that managed entry, and the controlled auction still reaches +`/auction`. + +- [ ] **Step 11: Format, lint, and commit Task 2** + +Run: + +```bash +cd crates/trusted-server-js/lib +npm run format +npm run lint +git add src/integrations/prebid/index.ts test/integrations/prebid/index.test.ts test/prebid-artifact-integration.test.mjs +git commit -m "feat: manage LiveRamp identityLink configuration" +``` + +## Task 3: Lock bundle, transport, consent, and EC behavior with regression tests + +**Files:** + +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts` +- Test: `crates/trusted-server-js/lib/test/build-prebid-external.test.mjs` +- Test: `crates/trusted-server-core/src/auction/endpoints.rs` +- Test: `crates/trusted-server-core/src/consent/mod.rs` +- Test: `crates/trusted-server-core/src/ec/prebid_eids.rs` + +- [ ] **Step 1: Add the explicit registry mapping test** + +```typescript +it('maps LiveRamp EIDs to identityLinkIdSystem', () => { + expect( + resolvePrebidUserIdModulesFromEids([ + { source: 'liveramp.com', uids: [{ id: 'opaque-envelope', atype: 3 }] }, + ]) + ).toEqual({ + modules: ['userId', 'identityLinkIdSystem'], + missingSources: [], + }) +}) +``` + +Also load the checked-in registry JSON or expose a narrow helper and assert the +default preset contains `identityLinkIdSystem`. Do not duplicate the registry +as a second production constant. + +- [ ] **Step 2: Add a bundle manifest test for `identityLinkIdSystem`** + +Extend the existing `includes generated User ID metadata` case or add a focused +case that invokes: + +```javascript +await main([ + '--adapters', + 'rubicon', + '--user-id-modules', + 'identityLinkIdSystem', + '--out', + outputDirectory, +]) +``` + +Assert the manifest's `userIdModules` is exactly +`['identityLinkIdSystem']` and the generated bundle contains the module name. + +- [ ] **Step 3: Run the two characterization suites** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run \ + test/integrations/prebid/user_id_modules.test.ts \ + test/build-prebid-external.test.mjs +``` + +Expected: tests pass using the existing registry and generator. If they fail, +fix the single checked-in registry/generator source rather than introducing a +LiveRamp-only bundle path. + +- [ ] **Step 4: Add or rename LiveRamp-specific Rust regression fixtures** + +Add focused tests (or rename/extend an existing generic fixture while keeping +its broader assertions) proving: + +- in `auction/endpoints.rs`, a client `liveramp.com` UID equal to the resolved + KV UID is merged once, with server-resolved metadata winning on conflict; +- in `consent/mod.rs`, a `liveramp.com` EID is removed when consent denies + identity forwarding; +- in `ec/prebid_eids.rs`, a structured `ts-eids` cookie containing an opaque + `liveramp.com` envelope writes that exact opaque string once to a registry + partner whose source domain is `liveramp.com`. + +Use only synthetic values such as `opaque-test-envelope`. Assert that tests do +not decode or inspect an envelope's contents. + +- [ ] **Step 5: Run the LiveRamp Rust regression fixtures** + +Run from the repository root: + +```bash +cargo test-fastly liveramp +``` + +Expected: forwarding, merge/deduplication, consent removal, and later-request +EC ingestion fixtures all pass. + +- [ ] **Step 6: Commit Task 3** + +Run: + +```bash +git add crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts \ + crates/trusted-server-js/lib/test/build-prebid-external.test.mjs \ + crates/trusted-server-core/src/auction/endpoints.rs \ + crates/trusted-server-core/src/consent/mod.rs \ + crates/trusted-server-core/src/ec/prebid_eids.rs +git commit -m "test: cover LiveRamp Prebid bundle support" +``` + +## Task 4: Document configuration, lifecycle, and operational validation + +**Files:** + +- Modify: `trusted-server.example.toml:41` +- Modify: `docs/guide/integrations/prebid.md:50` +- Modify: `docs/guide/integrations/prebid.md:419` +- Modify: `docs/guide/configuration.md:1050` + +- [ ] **Step 1: Add the commented example configuration** + +Add beneath the Prebid bundle configuration in `trusted-server.example.toml`: + +```toml +# Optional managed LiveRamp RampID configuration. The external Prebid bundle +# must contain identityLinkIdSystem. Obtain the Placement ID and approve the +# publisher origin with LiveRamp before enabling. +# [integrations.prebid.liveramp] +# placement_id = "999" +# not_use_3p = false +# storage_type = "cookie" +# expires_days = 15 +# refresh_in_seconds = 1800 +``` + +Do not add a real Placement ID or credential. + +- [ ] **Step 2: Update the configuration reference** + +Add `[integrations.prebid.liveramp]` fields to both Prebid option tables: + +| Field | Type | Default | Description | +| ----------------------------- | ------------------- | ------------------------------- | ------------------------------------------------------------ | +| `liveramp.placement_id` | String | Required when subsection exists | Numeric LiveRamp Placement ID for `identityLink`. | +| `liveramp.not_use_3p` | Boolean | `false` | Disable cookie-recognized RampID envelopes when true. | +| `liveramp.storage_type` | `cookie` or `html5` | `cookie` | Browser storage used by the Prebid module. | +| `liveramp.expires_days` | Integer 1–30 | `15` | Envelope storage lifetime in days. | +| `liveramp.refresh_in_seconds` | Positive integer | `1800` | Interval before retrieving a potentially refreshed envelope. | + +State that storage name `idl_env` is fixed by the integration. + +- [ ] **Step 3: Add the LiveRamp guide section** + +In `docs/guide/integrations/prebid.md`, document: + +- prerequisites: Placement ID, approved origin, CMP/LiveRamp consent posture, + and an external bundle containing `identityLinkIdSystem`; +- the exact TOML example and `ts prebid bundle` selection; +- operator ownership of the single `identityLink` entry for calls through the + supported public `pbjs.setConfig` and `pbjs.mergeConfig` APIs while preserving + other User ID modules; explicitly state that this is not a security boundary + against retained pre-wrapper references or direct internal mutation; +- asynchronous resolution: a new browser's first auction may have no RampID; +- the existing flow through `getUserIdsAsEids()`, `/auction`, + `user.ext.eids`, `ts-eids`, and EC/KV; +- degraded behavior for no consent, no recognition, missing module, LiveRamp + network failure, and KV failure; +- privacy guidance: envelopes are opaque and must not be logged; +- the explicit product boundary: this forwards RampID EIDs, not ATS Direct + audience segments; +- a credential-based manual validation checklist matching Section 11.5 of the + design spec, recording only booleans, counts, source names, and status codes. + +- [ ] **Step 4: Format and verify docs** + +Run: + +```bash +cd docs +npm run format +``` + +Expected: all documentation and TOML examples satisfy Prettier checks. + +- [ ] **Step 5: Commit Task 4** + +Run: + +```bash +git add trusted-server.example.toml docs/guide/integrations/prebid.md docs/guide/configuration.md +git commit -m "docs: explain managed LiveRamp RampID setup" +``` + +## Task 5: Run full verification and prepare live validation handoff + +**Files:** + +- Verify: all files changed in Tasks 1–4 +- Reference: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + +- [ ] **Step 1: Run the complete TSJS test and build gates** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run +npm run format +npm run lint +node build-all.mjs +``` + +Expected: all commands exit 0. + +- [ ] **Step 2: Run Rust formatting and adapter test gates** + +From the repository root, run: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all commands exit 0. Do not substitute bare +`cargo test --workspace`. + +- [ ] **Step 3: Run all target-matched clippy gates** + +Run: + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: all commands exit 0 with warnings denied. + +- [ ] **Step 4: Re-run documentation formatting and inspect the final diff** + +Run: + +```bash +cd docs +npm run format +cd .. +git diff --check +git status --short +git diff main...HEAD --stat +``` + +Expected: formatting and diff checks pass; status contains only intentional +plan/implementation changes. + +- [ ] **Step 5: Record credential-gated validation status** + +If LiveRamp test configuration is available, execute the guide's manual +validation on an approved non-production origin and report only: + +- approved origin used (domain, not credentials); +- whether `idl_env` was created/refreshed; +- whether `getUserIdsAsEids()` exposed source `liveramp.com`; +- whether the controlled PBS request contained that source; +- whether a later request ingested the EID into the configured + `liveramp.com` EC partner; +- whether opt-out removed it; and +- whether an unapproved origin degraded to no LiveRamp EID without blocking + the auction; and +- status codes/counts without envelope values. + +If credentials remain unavailable, report exactly: “Code complete; live +LiveRamp validation pending IABTechLab/uid2-optout#385.” Do not block automated +verification or add fake live-success evidence. + +In either case, prepare the explicit parent-epic acceptance handoff: “RampID +identity envelopes traverse the existing Prebid auction path; ATS Direct +audience segments are not passed by this implementation.” + +- [ ] **Step 6: Commit any verification-only corrections** + +Only if verification required source changes, repeat the affected focused and +full gates, then commit the minimal correction: + +```bash +git add +git commit -m "fix: address LiveRamp verification findings" +``` + +Do not create an empty verification commit. + +## Correction tasks added after PR #1054 review (2026-08-24) + +These tasks implement the reviewed correction in the specification. Complete +them in order and preserve artifact-level evidence for configuration behavior. + +## Task 6: Characterize partial `userSync` updates + +**Files:** + +- Modify: `crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs:179` + +- [x] **Step 1: Test the review premise against the generated artifact** + +Preconfigure a publisher `sharedId`, install the shim, then call: + +```js +pageWindow.pbjs.setConfig({ userSync: { syncDelay: 50 } }) +``` + +Assert that `getConfig('userSync.userIds')` still contains `sharedId` and exactly +one managed `identityLink`, and that `getConfig('userSync.syncDelay')` is `50`. +This test uses the generated Prebid bundle and generated TSJS shim, not a mock +of `setConfig`. + +- [x] **Step 2: Verify actual pinned behavior** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/prebid/index.test.ts test/prebid-artifact-integration.test.mjs +``` + +Observed: all focused tests pass. The pinned Prebid artifact retains its +effective `userIds` list across the partial update. Mock-only tests that expected +the shim to inject `userIds` into the forwarded argument were discarded because +the mock does not model the shipped artifact's effective configuration behavior. +No production wrapper change is required. + +- [x] **Step 3: Commit the artifact characterization** + +```bash +git add \ + crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs +git commit -m "Characterize partial Prebid userSync updates" +``` + +## Task 7: Characterize exact default TCF enforcement + +**Files:** + +- Modify: `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs:103-225` +- Modify: `docs/guide/integrations/prebid.md:134-141` +- Modify: `docs/guide/integrations/prebid.md:518-524` +- Modify: `docs/guide/integrations/prebid.md:578-588` + +- [x] **Step 1: Replace the combined consent boolean with independent grants** + +Change the fixture API to accept explicit grants with safe defaults: + +```js +function tcData({ + purpose1 = true, + purpose3 = true, + purpose4 = true, + vendor97 = true, +} = {}) { + return { + // existing CMP fields + purpose: { + consents: { 1: purpose1, 3: purpose3, 4: purpose4 }, + legitimateInterests: {}, + }, + vendor: { + consents: { [LIVE_RAMP_GVL_VENDOR_ID]: vendor97 }, + legitimateInterests: {}, + }, + } +} +``` + +Pass this object through `runGdprPage` without combining the grants. + +- [x] **Step 2: Add four independent artifact cases plus the granted baseline** + +Assert the exact pinned defaults: + +1. Purpose 1 denied alone: no LiveRamp request, no `idl_env`, no retry cookie. +2. Vendor 97 denied alone: no LiveRamp request, no `idl_env`, no retry cookie. +3. Purpose 3 denied alone: one LiveRamp request and `idl_env` written. +4. Purpose 4 denied alone: one LiveRamp request and `idl_env` written. +5. All relevant grants present: one LiveRamp request and `idl_env` written. + +- [x] **Step 3: Run the consent artifact suite** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/prebid-consent-enforcement.test.mjs +``` + +Expected: all five behavioral cases and the module-presence assertion pass +against the real generated artifacts. If a case differs, inspect pinned Prebid +before changing the expected policy. + +- [x] **Step 4: Correct the operator-facing consent claims** + +Document that default client-side resolution/storage is blocked by Purpose 1 +and LiveRamp vendor consent. State that Purpose 3 has no standalone default +rule, Purpose 4 controls UFPD, and default EID transmission accepts qualifying +purpose/vendor basis from any Purpose 2–10 unless the publisher enables +`eidsRequireP4Consent`. Preserve the existing explicit GPP/US-state limitation. + +- [x] **Step 5: Format and verify the focused documentation** + +```bash +cd docs +npx prettier --write guide/integrations/prebid.md +npm run format +``` + +Expected: the guide is formatted and makes no broader enforcement claim than +the artifact matrix proves. + +- [x] **Step 6: Commit the consent characterization** + +```bash +git add \ + crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs \ + docs/guide/integrations/prebid.md +git commit -m "Clarify LiveRamp TCF enforcement defaults" +``` + +## Task 8: Verify and refresh PR #1054 + +**Files:** + +- Verify: all PR files +- Update externally: PR #1054 description + +- [x] **Step 1: Run TypeScript tests, build, and formatting** + +```bash +cd crates/trusted-server-js/lib +npx vitest run +node build-all.mjs +npm run lint +npm run format +``` + +Expected: every command exits 0. + +- [x] **Step 2: Run repository Rust and documentation gates** + +```bash +cd /Users/prk-jr/Desktop/opensource/rust/trusted-server +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy-cli +cargo clippy-codegen +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +cd docs && npm run format +``` + +Expected: all commands exit 0. Do not use bare `cargo test --workspace`. + +- [x] **Step 3: Inspect the final branch** + +```bash +git diff --check +git status --short +git diff origin/main...HEAD --stat +git log origin/main..HEAD --oneline +``` + +Expected: no uncommitted source changes and only intentional LiveRamp commits. + +- [x] **Step 4: Request final code review** + +Review the entire `origin/main...HEAD` diff with special attention to the +partial-`userSync` real-artifact case and the independent consent matrix. Fix +all Critical or Important findings and repeat affected gates. + +- [x] **Step 5: Push and update the draft PR description** + +Push without force. Create `/tmp/pr-1054-body.md` with the repository PR +template and these exact sections: + +- Summary: managed RampID configuration, opaque `liveramp.com` EID transport, + exact TCF default behavior, and ATS Direct exclusion. +- Closes: `Closes #355`. +- Status: `Code complete; live LiveRamp validation pending +IABTechLab/uid2-optout#385.` +- Changes table containing every one of these final diff paths and no removed + `crates/trusted-server-core/src/auction/endpoints.rs` row: + - `.cargo/config.toml` + - `CLAUDE.md` + - `crates/trusted-server-cli/src/prebid_bundle.rs` + - `crates/trusted-server-core/src/consent/mod.rs` + - `crates/trusted-server-core/src/ec/prebid_eids.rs` + - `crates/trusted-server-core/src/integrations/prebid.rs` + - `crates/trusted-server-js/lib/build-prebid-external.mjs` + - `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` + - `crates/trusted-server-js/lib/test/build-prebid-external.test.mjs` + - `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + - `crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts` + - `crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs` + - `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs` + - `docs/guide/configuration.md` + - `docs/guide/integrations/prebid.md` + - `docs/superpowers/plans/2026-08-21-liveramp-integration.md` + - `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + - `trusted-server.example.toml` +- Test plan: check every command completed in Steps 1–2; leave only live + credential validation unchecked. +- Hardening note: no config-derived regex or pattern compilation was added; + invalid enabled LiveRamp config fails typed validation. + +Verify the enumerated paths against +`git diff --name-only origin/main...HEAD` before writing the file. Apply the +body exactly with: + +```bash +git push origin issue-355-liveramp-integration +gh pr edit 1054 \ + --repo IABTechLab/trusted-server \ + --title "Add managed LiveRamp RampID integration" \ + --body-file /tmp/pr-1054-body.md +gh pr view 1054 \ + --repo IABTechLab/trusted-server \ + --json url,isDraft,headRefOid,body,statusCheckRollup +``` + +Do not mark the PR ready for review automatically; report the final readiness +assessment to the user first. diff --git a/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md b/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md new file mode 100644 index 000000000..9f3735d91 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md @@ -0,0 +1,307 @@ +# PR 823 Round-5 Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable finding in PR 823 review `4989897698` while preserving generation compatibility and enforcing root-less template safety. + +**Architecture:** Keep browser-option defaults and legacy clap compatibility at the CLI boundary, carry borrowed-root evidence through template inference, and reject unsafe overrides before rendering. Improve diagnostics and validation at their existing seams, then pin cross-language and documentation invariants with focused tests. + +**Tech Stack:** Rust 2024, clap 4 derive, `url`, `toml`, embedded JavaScript, mdBook/VitePress documentation. + +--- + +## File Map + +- `crates/trusted-server-cli/src/commands/audit/collector.rs`: generation browser default constants and option defaults. +- `crates/trusted-server-cli/src/commands/audit/mod.rs`: hidden legacy browser arguments, early TOML validation, conversion to generation arguments. +- `crates/trusted-server-cli/src/run.rs`: clap contract tests. +- `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs`: collector defaults and formatting. +- `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs`: borrowed-root inference metadata and root-gap refusal reasons. +- `crates/trusted-server-cli/src/commands/audit/generate/mod.rs`: redirect output, profile-scoped notes, merge-policy validation, explicit-pattern refusal. +- `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs`: timestamp-shaped volatile token recognition. +- `crates/trusted-server-cli/src/commands/audit/browser.rs`: Rust/JavaScript evidence-cap invariant test. +- `crates/trusted-server-cli/src/commands/audit/page.rs`: accurate final-URL/terminal-escaping test claims. +- `docs/guide/cli.md` and the volatile-collision design/plan: operator and historical documentation corrections. + +### Task 1: Restore generation browser defaults and legacy clap isolation + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add failing clap and default tests** + +Add parser coverage proving that `ts audit --help` does not advertise generation +browser flags, `ts audit --chrome /tmp/chrome generate ...` is rejected, and the +legacy `ts audit --chrome ... --settle-max-ms ...` form still parses and +reaches `GenerateArgs`. Add a generation-option default assertion for 750 ms and +12,000 ms. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin run::tests::audit_ -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::tests::legacy_ -- --nocapture +``` + +Expected: the hidden/help and 12-second assertions fail on the current branch. + +- [ ] **Step 3: Implement one generation-default source and legacy mirror** + +Define generation-specific constants in `collector.rs` and use them in clap +attributes and `GenerateBrowserOpts::default`: + +```rust +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; +``` + +Use those constants in `BrowserAuditCollector::default`. Replace the flattened +`GenerateBrowserOpts` under `LegacyGenerateArgs` with `LegacyBrowserOpts`, whose +seven fields each use `hide = true, requires = "legacy_url"`. Implement +`From<&LegacyBrowserOpts> for GenerateBrowserOpts` and use it in +`legacy_generate_args`. Add the missing blank line between collector methods. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run the Step 2 commands and the focused collector default test. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/collector.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/run.rs +git commit -m "Preserve generation browser option contracts" +``` + +### Task 2: Enforce borrowed-root and merge-policy safety + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing inference and end-to-end tests** + +Add tests proving: + +- `InferenceOutcome` identifies `ad-sidebar` as borrowing the root witnessed by + another slot; +- explicit `--page-pattern` values cause `run_update_slots` to fail before the + source config changes when any rendered template borrowed the root; +- no-policy inference gives affected multi-path slots the root-witness reason; +- a configured `section_segment = 1` with no `section_root` refuses inferred + segment 0 when preserved `{section}` slots exist; +- the same segment, or an unset segment, allows adopting the inferred root. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::unit_template::tests -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::tests::merge_ -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::tests::explicit_ -- --nocapture +``` + +Expected: borrowed stems are unavailable, explicit patterns are accepted, and +the configured-segment mismatch is accepted. + +- [ ] **Step 3: Carry borrowed stems and reject unsafe overrides** + +Add an ordered `borrowed_section_root: Vec` field to +`InferenceOutcome`. Populate it only when `RootUnwitnessed` successfully becomes +a template. Before building render slots, reject non-empty explicit patterns if +that vector is non-empty: + +```rust +return cli_error(format!( + "cannot apply --page-pattern to slot(s) {} because their {{section}} templates borrow section_root; remove --page-pattern so patterns can be derived from observed paths", + borrowed.join(", ") +)); +``` + +On the no-policy path, replace the generic multi-path refusal reason for +structurally valid root-unwitnessed slots with the specific missing-root-witness +reason. Preserve structural refusal reasons unchanged. + +Update `validate_merge_policy` so an explicit configured segment is compared +before the empty-root adoption return. Keep the guard limited to preserved +`{section}` slots and allow `--replace`. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Protect borrowed section templates during generation" +``` + +### Task 3: Make redirects, warnings, and config errors actionable + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` + +- [ ] **Step 1: Add failing diagnostic tests** + +Strengthen the HTTPS-upgrade assertion to require +`http://publisher.example/` and `https://publisher.example/`. Add a two-profile +warning test whose output names desktop and mobile separately. Add a malformed +whole-document TOML test while retaining tests for unknown valid settings and an +unreadable `[creative_opportunities]` section. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin update_slots_accepts_a_same_host_https_upgrade -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin profile_warning -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin creative_config -- --nocapture +``` + +- [ ] **Step 3: Implement scoped diagnostics and early parse failure** + +Render redirect endpoints as `origin.ascii_serialization() + path`. Thread the +profile label into `fold_collected`; keep the consent-stub warning global, label +page warnings/interstitials with path and profile, and retain the existing +site-wide discovery-warning dedupe. + +Replace `.ok()` in `creative_config` with an error mapping that identifies a +malformed existing TOML document and explains that generation did not start. +Continue parsing into `toml::Value`, not runtime `Settings`, so valid unknown +settings remain tolerated. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/mod.rs crates/trusted-server-cli/src/commands/audit/mod.rs +git commit -m "Clarify audit generation diagnostics" +``` + +### Task 4: Pin detector and embedded-collector invariants + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/page.rs` + +- [ ] **Step 1: Add failing invariant tests** + +Add a negative volatile-token test for `promo-20260820a-sidebar`, retain a +positive timestamp-shaped control with at least ten leading digits, and add the +embedded-JavaScript constant assertion: + +```rust +assert!( + AD_TEMPLATE_COLLECTOR_JS.contains(&format!( + "const __ts_max_entries = {MAX_EVIDENCE_ENTRIES}" + )), + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" +); +``` + +In the page summary test, assert the exact percent-encoded final URL line and +limit the raw-control assertion's comment to title and warning fields. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin per_render_token -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin evidence_entries -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin page_controlled_text -- --nocapture +``` + +- [ ] **Step 3: Tighten the token shape and correct the test claim** + +Require at least ten leading digits in `is_per_render_token`. Keep the rest of +the recognizer unchanged. Add the evidence-cap test and page assertion without +removing final-URL escaping. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs crates/trusted-server-cli/src/commands/audit/browser.rs crates/trusted-server-cli/src/commands/audit/page.rs +git commit -m "Pin audit evidence recognition invariants" +``` + +### Task 5: Align documentation and local style + +**Files:** + +- Modify: `docs/guide/cli.md` +- Modify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` +- Modify: `docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] In the volatile-collision design example, remove the section-varying + sidebar from the list of omitted/explained slots because root-less templating + now writes it with a borrowed-root diagnostic. +- [ ] In the volatile-collision implementation plan, state that a recognized + render token must have a non-empty family prefix before it and placement + content after it; remove the broader "in any position" claim. +- [ ] Update the guide to say that a configured segment without a root is + preserved for existing templates, and document the explicit-pattern refusal + for borrowed-root slots. +- [ ] Add the missing `GenerateArgs.browser` doc comment, change the `expect` + message to the required `"should ..."` form, and retain the method-separation + blank line from Task 1. +- [ ] Run `cd docs && npm run format` and `cargo fmt --all -- --check`. +- [ ] Commit: + +```bash +git add docs/guide/cli.md docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md crates/trusted-server-cli/src/commands/audit/generate/mod.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +git commit -m "Align ad-template generation documentation" +``` + +### Task 6: Verify the complete review resolution + +**Files:** + +- Verify all files above. + +- [ ] Run focused audit generation tests: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate -- --nocapture +``` + +- [ ] Run the complete host CLI suite: + +```bash +./scripts/test-cli.sh aarch64-apple-darwin +``` + +- [ ] Run lint and formatting gates: + +```bash +cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings +cargo fmt --all -- --check +cd docs && npm run format +git diff --check +``` + +- [ ] Inspect `git status --short`, `git log --oneline -6`, and the complete + diff from `073d5644` to ensure only the approved review resolution is present. +- [ ] Do not push or post GitHub replies without separate user authorization. diff --git a/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md b/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md new file mode 100644 index 000000000..2ec2f7979 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md @@ -0,0 +1,295 @@ +# Ad-template div-ID reconciliation implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Preserve observed numeric sibling creative opportunities during config merge and refuse singleton div IDs containing shorter high-entropy per-render tokens. + +**Architecture:** Reconciliation will use the normalized identities already retained by `EvidenceTable` to distinguish observed literals from intentional configured prefixes. GPT discovery will keep its vendor-neutral, position-aware volatile-family classifier and add a conservative eight-leading-digit/eight-character-suffix alternative without changing existing ten-digit behavior. + +**Tech Stack:** Rust 2024, `BTreeSet`, existing Trusted Server CLI evidence/merge pipeline, Cargo unit and browser integration tests. + +--- + +### Task 1: Preserve observed literal siblings during merge + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] **Step 1: Write failing numeric-sibling merge tests** + +Add focused tests beside the existing prefix tests: + +```rust +#[test] +fn observed_literal_does_not_claim_numeric_siblings() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10", "ad-sidebar-11"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-11")); + assert!(diagnostics.notes.is_empty()); +} +``` + +Add a second regression with an unrelated existing slot and discovered +`ad-sidebar-1` followed by `ad-sidebar-10`. It must prove a newly appended +observed literal cannot absorb a later sibling. Keep +`merge_reports_when_a_broad_prefix_claims_multiple_discovered_divs` unchanged as +the positive intentional-prefix control. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cargo test -p trusted-server-cli observed_literal_does_not_claim_numeric_siblings -- --nocapture +cargo test -p trusted-server-cli newly_appended_literal_does_not_claim_numeric_sibling -- --nocapture +``` + +Expected: both fail because `ad-sidebar-1` absorbs the longer discovered IDs. + +- [ ] **Step 3: Implement exact-first, evidence-aware prefix matching** + +In `merge_render_slots_with_observed_diagnostics`, build a borrowed set from +`observed_div_ids` once: + +```rust +let observed_literals = observed_div_ids + .iter() + .map(String::as_str) + .collect::>(); +``` + +Thread `&observed_literals` through discovered-slot reconciliation and +observed/unobserved classification. Refactor the matcher so it: + +1. searches all merged slots for an exact stable-key match; +2. returns that exact match immediately; +3. searches for the longest prefix only among prefixes absent from + `observed_literals`; and +4. retains configuration order for equal-length prefix ties. + +Use the same helper for seeding `observed_existing`, so merge behavior and stale +diagnostics cannot disagree. Keep exact matching available for configured slots +that omit `div_id` and therefore resolve through `id`. + +Update the `MergeDiagnostics` field comment from “raw crawl” to “normalized +evidence.” + +- [ ] **Step 4: Add and run the normalization-boundary regression** + +Use `discover_gpt_slots` plus `merge_slots` to show that a live +`ad-header-0-_R_3f_` identity normalizes to `ad-header-0`, and therefore makes +configured `ad-header-0` an observed literal rather than a prefix for a distinct +`ad-header-01` slot. Do not pass collector-level raw IDs into the merge. + +Run: + +```bash +cargo test -p trusted-server-cli normalized_stem_is_the_literal_merge_boundary -- --nocapture +``` + +Expected after implementation: PASS. + +- [ ] **Step 5: Run focused merge tests and verify GREEN** + +Run: + +```bash +cargo test -p trusted-server-cli slot_toml::tests -- --nocapture +``` + +Expected: all merge tests pass, including the existing intentional broad-prefix +test. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +git commit -m "Preserve observed literal ad slot siblings" +``` + +### Task 2: Refuse eight-digit, long-suffix volatile tokens + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` + +- [ ] **Step 1: Add failing shorter-token registry and request tests** + +Add singleton cases using a synthetic shape: + +```rust +const SHORT_VOLATILE_DIV: &str = + "vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1"; +``` + +Assert both registry and GAMPAD request discovery: + +- retain `had_slot_evidence`; +- produce no writable slots; and +- emit the existing volatile-family warning naming `vendor-tag`. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cargo test -p trusted-server-cli shorter_high_entropy_singleton -- --nocapture +``` + +Expected: FAIL because the current classifier requires ten leading digits and +accepts the eight-digit token literally. + +- [ ] **Step 3: Add failing classifier boundary tests** + +Extend the table-driven tests so these remain eligible: + +```text +vendor-tag_1234567AbCdEfGh_slot_inarticle_1 # seven leading digits +vendor-tag_12345678AbCdEfG_slot_inarticle_1 # seven-character suffix +promo-20260820a-sidebar # short calendar suffix +vendor-tag_1234567890123456_slot_inarticle_1 # bare numeric segment +``` + +Add `vendor-tag_12345678AbCdEfGh_slot_inarticle_1` to the volatile table. Run +the two boundary tests and confirm only the new 8+8 volatile assertion fails. + +- [ ] **Step 4: Implement the conservative alternative token shape** + +Keep the current all-ASCII-alphanumeric requirement and compute the suffix +length after the leading digit run. A segment is per-render when either: + +```rust +(leading_digits >= 10 && suffix_length >= 1) + || (leading_digits >= 8 && suffix_length >= 8) +``` + +Keep the existing requirement that the token occurs before another div-ID +segment. Do not add a vendor name or family-specific regular expression. + +- [ ] **Step 5: Run GPT discovery tests and verify GREEN** + +Run: + +```bash +cargo test -p trusted-server-cli gpt_slots::tests -- --nocapture +``` + +Expected: all discovery, normalization, collision, registry, request, and +boundary tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +git commit -m "Reject shorter high-entropy ad slot tokens" +``` + +### Task 3: Verify the complete change + +**Files:** + +- No source changes expected. + +- [ ] **Step 1: Run formatting and diff checks** + +```bash +cargo fmt --all -- --check +git diff --check +cd docs && npm run format +``` + +Expected: all exit zero and formatting makes no changes. + +- [ ] **Step 2: Run the complete CLI suite** + +```bash +./scripts/test-cli.sh +``` + +Expected: all unit, config overlay, proxy E2E, and ignored real-Chrome fixtures +pass. The browser portions require permission to bind loopback listeners. + +- [ ] **Step 3: Run host-target CLI clippy** + +```bash +cargo clippy \ + --manifest-path crates/trusted-server-cli/Cargo.toml \ + --target "$(rustc -vV | sed -n 's/^host: //p')" \ + --all-targets -- -D warnings +``` + +Expected: the changed CLI crate and all of its test targets lint without +warnings. The adapter-scoped aliases below do not include this crate. + +- [ ] **Step 4: Run repository target-specific Rust gates** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: every command exits zero with no warnings promoted to errors. + +- [ ] **Step 5: Run parity and JavaScript/docs gates** + +```bash +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +(cd crates/trusted-server-js/lib && npx vitest run) +(cd crates/trusted-server-js/lib && npm run format) +(cd docs && npm run format) +``` + +Expected: parity, Vitest, and formatting checks pass. + +- [ ] **Step 6: Review branch state** + +```bash +git status --short +git log --oneline --decorate -10 +``` + +Expected: clean feature worktree with the two implementation commits above the +approved design/plan commits. + +- [ ] **Step 7: Validate against the operator's dry-run output** + +Ask the operator to rerun the established desktop/mobile `--scroll --dry-run` +command with a current DataDome cookie. Confirm: + +- there is no `ad-sidebar-1` broad-prefix collision note; +- numeric sidebar siblings are emitted as distinct slots; +- the singleton mobile volatile-family slot is refused; and +- older configured volatile-family slots remain named as preserved but + unobserved until the operator deliberately prunes them. diff --git a/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md b/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md new file mode 100644 index 000000000..db6f22ce2 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md @@ -0,0 +1,358 @@ +# Ad-template Generate Scroll and Staleness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add opt-in scrolling to `ts audit ad-templates generate` and warn when a normal merge preserves configured slots that the current crawl did not observe. + +**Architecture:** Thread one parsed `--scroll` value through the generation browser session and reuse a shared deterministic scroll primitive before the generator's final evidence scrape. Extend merge reconciliation with structured diagnostics that record unmatched pre-existing slot IDs; format the warning at the command layer so it can account for whether scrolling was already enabled without changing merge behavior. + +**Tech Stack:** Rust 2024, clap, chromiumoxide/CDP, Tokio, existing CLI and Chrome-fixture test harnesses, rustfmt, clippy, Prettier. + +--- + +## File map + +- Create `crates/trusted-server-cli/src/commands/audit/browser_scroll.rs`: shared deterministic scroll primitive. +- Modify `crates/trusted-server-cli/src/commands/audit/mod.rs`: declare the shared module, parse `--scroll`, and wire it into generation. +- Modify `crates/trusted-server-cli/src/commands/audit/browser.rs`: reuse shared scrolling while retaining verifier-only phase marking. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs`: carry scroll state, scroll and re-settle, and test lazy GPT discovery. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/mod.rs`: carry scroll context and render contextual stale-slot notes. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs`: report unmatched preserved slots from the authoritative merge matcher. +- Modify `crates/trusted-server-cli/src/run.rs`: test parsing and defaults. +- Modify `scripts/test-cli.sh`: run the new ignored Chrome fixture. +- Modify `docs/guide/cli.md`: document both behaviors. + +### Task 1: Parse and wire generation scrolling + +**Files:** + +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing parsing tests** + +Extend `audit_generate_subcommands_use_generation_settle_defaults` with +`assert!(!generate.scroll)`. Add: + +```rust +#[test] +fn audit_ad_templates_generate_parses_scroll() { + let args = parse(&[ + "ts", "audit", "ad-templates", "generate", + "https://www.example.com/", "--scroll", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command else { + panic!("expected audit ad-templates generate command"); + }; + assert!(generate.scroll, "--scroll should enable generation scrolling"); +} +``` + +- [ ] **Step 2: Run the focused test and verify it fails** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_ad_templates_generate_parses_scroll +``` + +Expected: compilation fails because `AuditAdTemplatesGenerateArgs` has no +`scroll` field. + +- [ ] **Step 3: Add the flag and session wiring** + +Add to `AuditAdTemplatesGenerateArgs`: + +```rust +/// Perform a deterministic scroll pass after each page initially settles. +#[arg(long)] +pub scroll: bool, +``` + +Add `scroll: bool` to `BrowserAuditCollector` and `SessionSettings`, default it +to false, and add `with_scroll(bool)`. Thread it through `session()`, +`with_browser`, `collect_page_from_browser`, and `collect_open_page`; Task 2 +will use it. + +Add `scroll: bool` to `UpdateSlotsRequest`. In `run_audit`, set both the +collector option and request field from `gen_args.scroll`. Update every test +fixture constructing `UpdateSlotsRequest` with `scroll: false`, except the later +contextual-warning test. + +- [ ] **Step 4: Run parsing/default tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_generate_subcommands_use_generation_settle_defaults +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_ad_templates_generate_parses_scroll +``` + +Expected: both pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Add scroll option to ad-template generation" +``` + +### Task 2: Share and execute deterministic scrolling + +**Files:** + +- Create: `crates/trusted-server-cli/src/commands/audit/browser_scroll.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `scripts/test-cli.sh` + +- [ ] **Step 1: Add a failing Chrome fixture** + +Add a self-contained tall HTML page whose scroll listener installs a stub GPT +registry and defines `/123/lazy` in `ad-lazy-0` only after `window.scrollY > 0`. +Add this ignored test: + +```rust +#[test] +#[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] +fn collects_lazy_gpt_slot_only_when_scroll_is_enabled() { + if !browser_fixture_available() { + return; + } + let url = lazy_gpt_fixture_url(); + let without_scroll = BrowserAuditCollector::default() + .collect_page(&url, &[]) + .expect("should collect without scrolling"); + let with_scroll = BrowserAuditCollector::default() + .with_scroll(true) + .collect_page(&url, &[]) + .expect("should collect with scrolling"); + + assert!(without_scroll.gpt_slots.is_empty()); + assert!(with_scroll.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/lazy" && slot.div_id == "ad-lazy-0" + })); +} +``` + +Use loopback HTTP instead of `file://` if Chrome requires it for reliable scroll +events. Change `scripts/test-cli.sh` to run the ignored +`commands::audit::generate::browser_collector::tests::` prefix so lifecycle and +lazy-slot fixtures are both covered. + +- [ ] **Step 2: Run the fixture and verify it fails** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +TS_AUDIT_BROWSER_TESTS=1 cargo test --package trusted-server-cli --target "$HOST_TARGET" collects_lazy_gpt_slot_only_when_scroll_is_enabled -- --ignored --test-threads=1 +``` + +Expected: the scrolled result still lacks `/123/lazy`. + +- [ ] **Step 3: Implement the shared primitive** + +Create `browser_scroll.rs` with a `ScrollFailure` enum (evaluation failure and +timeout) and: + +```rust +pub(crate) async fn scroll_page(page: &chromiumoxide::Page) -> Vec { + let mut failures = Vec::new(); + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, document.documentElement.scrollHeight) * {fraction}))" + ); + evaluate(page, script, &mut failures).await; + tokio::time::sleep(Duration::from_millis(250)).await; + } + evaluate(page, "window.scrollTo(0, 0)".to_string(), &mut failures).await; + failures +} +``` + +Bound each evaluation at five seconds. Declare the module in `audit/mod.rs`. +In `browser.rs`, leave the pre-scroll evidence snapshot and +`window.__tsScrollPhase = true` marker in place, replace the local step loop with +the shared function, and map failures to existing `Warning` output. + +In the generation collector, after initial settle but before final HTML/GPT/ +network/link scraping, call the shared function when `scroll` is true, append +its failures as page warnings, and call `wait_for_page_settle` again. A second +settle timeout is a warning, not a discarded page. + +- [ ] **Step 4: Run browser tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser::tests:: +TS_AUDIT_BROWSER_TESTS=1 cargo test --package trusted-server-cli --target "$HOST_TARGET" collects_lazy_gpt_slot_only_when_scroll_is_enabled -- --ignored --test-threads=1 +``` + +Expected: all pass and `/123/lazy` appears only with scrolling. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/browser_scroll.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/audit/browser.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs scripts/test-cli.sh +git commit -m "Collect lazy ad slots during generation scroll" +``` + +### Task 3: Report unmatched slots preserved by merge + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing diagnostic tests** + +Next to `merge_keeps_existing_only_slots`, assert that diagnostic merging marks +preserved `sidebar` but not rediscovered `header`; multiple missing IDs retain +configuration order; and full rediscovery, empty existing slots, and +`--replace` produce no stale IDs. + +Add command tests with fake collectors and in-memory writers. Assert non-scroll +wording contains `or --scroll`, scroll wording omits that retry, stdout remains +only diff/summary content, and preserved slots remain in candidate TOML. + +- [ ] **Step 2: Run focused tests and verify they fail** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" merge_reports_preserved_unobserved_slots +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots_reports_preserved_unobserved_slots +``` + +Expected: failures because unmatched existing slots are not exposed. + +- [ ] **Step 3: Add structured merge diagnostics** + +Define: + +```rust +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct MergeDiagnostics { + pub(super) notes: Vec, + pub(super) unobserved_existing_slot_ids: Vec, +} +``` + +Change `merge_render_slots_with_diagnostics` to return this structure with the +merged slots. Record every matched existing index in a `BTreeSet`, then +collect unmatched existing IDs by enumerating configuration order. Preserve the +current broad-prefix messages in `notes`. The `replace || existing.is_empty()` +early path returns default diagnostics. Keep `merge_render_slots` returning only +the slot vector. + +- [ ] **Step 4: Format the contextual note in `run_update_slots`** + +Extend pending notes with `merge_diagnostics.notes`. If unmatched IDs exist, +append their count and comma-separated IDs. End with: + +```rust +let follow_up = if request.scroll { + "Re-run with broader page/profile coverage; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." +} else { + "Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." +}; +``` + +Do not change the merged configuration. `emit_notes` remains the only terminal +sanitization/output boundary. + +- [ ] **Step 5: Run merge and command tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::slot_toml::tests::merge_ +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots_reports_preserved_unobserved_slots +``` + +Expected: all pass, with unchanged merged TOML and warnings only on stderr. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Warn about preserved unobserved ad slots" +``` + +### Task 4: Document and verify + +**Files:** + +- Modify: `docs/guide/cli.md` + +- [ ] **Step 1: Document both behaviors** + +Add a generation `--scroll` example under “Bounding and steering the crawl.” +Explain that every page/profile scrolls after initial settle and settles again, +and that it is opt-in because it adds time, requests, and publisher side effects. + +Update merge documentation: missing existing slots are preserved and named on +stderr; absence may reflect coverage, targeting, or lazy loading; only +`--replace` intentionally prunes them. + +- [ ] **Step 2: Format docs and inspect scope** + +```bash +cd docs && npm run format +git diff --check +git diff -- docs/guide/cli.md +``` + +Expected: formatting passes and only intended docs change. + +- [ ] **Step 3: Run the full CLI harness, including Chrome fixtures** + +```bash +./scripts/test-cli.sh +``` + +Expected: all host CLI and configured ignored browser tests pass. + +- [ ] **Step 4: Run formatting and lint gates** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: all exit zero without warnings. + +- [ ] **Step 5: Run adapter regression suites** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all pass. Do not use bare `cargo test --workspace`. + +- [ ] **Step 6: Review scope and commit docs** + +```bash +git status --short +git diff --check +git diff HEAD -- crates/trusted-server-cli scripts/test-cli.sh docs/guide/cli.md +``` + +Confirm `fastly.toml` remains untouched and issue #1059 produced no code changes. +Then: + +```bash +git add docs/guide/cli.md +git commit -m "Document generation scroll and stale-slot warnings" +``` diff --git a/docs/superpowers/plans/2026-08-24-request-phase-timing.md b/docs/superpowers/plans/2026-08-24-request-phase-timing.md new file mode 100644 index 000000000..1b25b70d1 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-request-phase-timing.md @@ -0,0 +1,1040 @@ +# Request Phase Timing Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Every application response attributes its own server time by phase via a +Server-Timing header and a sampled Tinybird access-telemetry row. + +**Architecture:** A core `RequestTimings` handle (Arc-shared, infallible recording) +collects phase spans always-on; the Fastly adapter freezes and emits at +`send_edgezero_response` immediately before `into_parts()`; a post-send emitter ships +one NDJSON row to the Tinybird Events API with a bounded, 2xx-validated await. + +**Tech Stack:** Rust 2024, `edgezero` HTTP types, Fastly Compute (wasm32-wasip1, +Viceroy tests), Axum (native tests), Tinybird Events API. + +**Spec:** `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md`: the +plan argues from the spec; executors read both. Spec section numbers are cited per +task. + +## Global Constraints + +- Errors use `error-stack` (`Report`); errors defined with + `derive_more::Display`; never thiserror, never anyhow (except the Spin entry point). +- No `unwrap()` in production code; `expect("should ...")` only. Assertion messages + `"should ..."`. Tests use Arrange-Act-Assert. +- No inline comments; comments on their own line above the code. +- Functions never exceed 7 arguments; use a struct instead (this bit + `ec_finalize_response` in review; the timings handle travels inside existing state). +- No local imports inside functions; `use super::*` only in `#[cfg(test)]`. +- Only example/fictional data in tests and docs (`example.com` domains). +- Recording is infallible: saturating math, lock failure drops the sample, no panics + (spec 5, 13). +- Vendor identity never appears in emitted surfaces: the filter span is `ts-filter` + (spec 3). +- Test commands: `cargo test-axum` (native, fast inner loop), `cargo test-fastly` + (Viceroy) for adapter tasks. Before PR handoff: the full CI gate list in + `CLAUDE.md`. +- Commit style: sentence case, imperative, no prefixes, no trailers. + +## File Structure + +| File | Responsibility | +| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/request_timing.rs` (new) | `Phase`, `AuctionWaitPlacement`, `RequestTimings`, `PhaseSpan`, `TimingSnapshot`, header rendering | +| `crates/trusted-server-core/src/access_telemetry.rs` (new) | `RouteClass`, `publisher_route_template`, `AccessTelemetrySnapshot`, `AccessEventRow` NDJSON | +| `crates/trusted-server-core/src/geo.rs` (modify) | `GeoLookupState` response-extension type | +| `crates/trusted-server-core/src/settings.rs` (modify) | `ObservabilitySettings`, tinybird flag decoupling, access validation | +| `crates/trusted-server-core/src/publisher.rs` (modify) | `ts-origin`, `ts-template-cache`, auction-wait spans | +| `crates/trusted-server-core/src/ec/kv.rs` (modify) | `ts-kv` at the graph abstraction | +| `crates/trusted-server-adapter-fastly/src/main.rs` (modify) | T0, appbuild span, freeze point, `DeliveryOutcome`, post-send emission ordering | +| `crates/trusted-server-adapter-fastly/src/app.rs` (modify) | filter span, geo span + `GeoLookupState` attach, route class assignment | +| `crates/trusted-server-adapter-fastly/src/middleware.rs` (modify) | finalize consumes `GeoLookupState` | +| `crates/trusted-server-adapter-fastly/src/tinybird.rs` (modify) | access sink with confirmed delivery | +| `crates/trusted-server-adapter-axum/src/` (modify) | terminal freeze layer, header emission | +| `tinybird/datasources/access_logs_raw.datasource` (modify) | phase-column schema, non-null sorting key | +| `trusted-server.example.toml` (modify) | `[observability]`, tinybird keys | + +Out of scope for this plan: the Grafana dashboard JSON (separate telemetry repo, +spec 11) and Cloudflare/Spin emission wiring (spec non-goal). + +--- + +### Task 1: Core `RequestTimings` + +**Files:** + +- Create: `crates/trusted-server-core/src/request_timing.rs` +- Modify: `crates/trusted-server-core/src/lib.rs` (add `pub mod request_timing;`) +- Test: same file, `#[cfg(test)]` + +**Interfaces:** + +- Consumes: nothing (leaf module; `std::time`, `std::sync`). +- Produces (later tasks rely on these exact names): + - `pub enum Phase { AppBuild, Filter, Geo, EcKv, Origin, TemplateCacheLookup, AuctionWait, Stream }` + - `pub enum AuctionWaitPlacement { PreHeader, InStream }` + - `#[derive(Clone)] pub struct RequestTimings` with: + - `pub fn new() -> Self` + - `pub fn record(&self, phase: Phase, dur: Duration)` (saturating accumulate) + - `pub fn record_auction_wait(&self, placement: AuctionWaitPlacement, dur: Duration)` + - `pub fn span(&self, phase: Phase) -> PhaseSpan` (records on drop) + - `pub fn mark_headers_ready(&self)` (first call wins) + - `pub fn mark_request_elapsed(&self)` (first call wins) + - `pub fn set_resp_bytes(&self, bytes: u64)` + - `pub fn server_timing_value(&self) -> Option` + - `pub fn snapshot(&self) -> TimingSnapshot` + - `pub struct TimingSnapshot { pub time_elapsed_ms: Option, pub request_elapsed_ms: Option, pub appbuild_ms: Option, pub filter_ms: Option, pub geo_ms: Option, pub kv_ms: Option, pub origin_ms: Option, pub template_cache_ms: Option, pub auction_wait_ms: Option, pub stream_ms: Option, pub auction_wait_placement: Option, pub resp_bytes: Option }` + +- [ ] **Step 1: Write the failing tests** + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn render_omits_unrecorded_phases_and_orders_total_first() { + let timings = RequestTimings::new(); + timings.record(Phase::Filter, Duration::from_micros(9_100)); + timings.mark_headers_ready(); + let value = timings + .server_timing_value() + .expect("should render after mark_headers_ready"); + assert!( + value.starts_with("ts-total;dur="), + "should lead with ts-total: {value}" + ); + assert!(value.contains("ts-filter;dur=9.1"), "should render one decimal: {value}"); + assert!(!value.contains("ts-geo"), "should omit unrecorded phases: {value}"); + } + + #[test] + fn render_returns_none_before_headers_ready() { + let timings = RequestTimings::new(); + timings.record(Phase::Geo, Duration::from_millis(1)); + assert!(timings.server_timing_value().is_none(), "should require the snapshot"); + } + + #[test] + fn repeated_phases_accumulate_saturating() { + let timings = RequestTimings::new(); + timings.record(Phase::Geo, Duration::from_millis(2)); + timings.record(Phase::Geo, Duration::from_millis(3)); + timings.mark_headers_ready(); + let snapshot = timings.snapshot(); + assert_eq!(snapshot.geo_ms, Some(5), "should accumulate repeats"); + } + + #[test] + fn mark_headers_ready_is_first_call_wins() { + let timings = RequestTimings::new(); + timings.mark_headers_ready(); + let first = timings.snapshot().time_elapsed_ms; + std::thread::sleep(Duration::from_millis(5)); + timings.mark_headers_ready(); + assert_eq!(timings.snapshot().time_elapsed_ms, first, "should not restamp"); + } + + #[test] + fn span_guard_records_on_drop() { + let timings = RequestTimings::new(); + { + let _span = timings.span(Phase::Origin); + std::thread::sleep(Duration::from_millis(2)); + } + timings.mark_headers_ready(); + assert!( + timings.snapshot().origin_ms.expect("should record on drop") >= 1, + "should measure elapsed span time" + ); + } + + #[test] + fn auction_wait_records_placement() { + let timings = RequestTimings::new(); + timings.record_auction_wait(AuctionWaitPlacement::PreHeader, Duration::from_millis(40)); + let snapshot = timings.snapshot(); + assert_eq!(snapshot.auction_wait_ms, Some(40), "should record wait"); + assert_eq!( + snapshot.auction_wait_placement, + Some(AuctionWaitPlacement::PreHeader), + "should record placement" + ); + } + + #[test] + fn rendered_names_never_include_vendor_terms() { + let timings = RequestTimings::new(); + for phase in [Phase::AppBuild, Phase::Filter, Phase::Geo, Phase::EcKv, Phase::Origin, Phase::TemplateCacheLookup] { + timings.record(phase, Duration::from_millis(1)); + } + timings.mark_headers_ready(); + let value = timings.server_timing_value().expect("should render"); + assert!(!value.to_ascii_lowercase().contains("datadome"), "should mask vendors"); + } +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum -p trusted-server-core request_timing` +Expected: compile FAIL, module does not exist. + +- [ ] **Step 3: Implement** + +```rust +//! Per-request phase timing collection and Server-Timing rendering. +//! +//! Collection is always-on and infallible: saturating math, lock failure +//! drops the sample, no panics. See the design spec +//! `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md`. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +const PHASE_COUNT: usize = 8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Phase { + AppBuild, + Filter, + Geo, + EcKv, + Origin, + TemplateCacheLookup, + AuctionWait, + Stream, +} + +impl Phase { + fn index(self) -> usize { /* match self -> 0..=7 */ } + + /// Header entry name; row-only phases return None. + fn header_name(self) -> Option<&'static str> { + match self { + Self::AppBuild => Some("ts-appbuild"), + Self::Filter => Some("ts-filter"), + Self::Geo => Some("ts-geo"), + Self::EcKv => Some("ts-kv"), + Self::Origin => Some("ts-origin"), + Self::TemplateCacheLookup => Some("ts-template-cache"), + Self::AuctionWait | Self::Stream => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuctionWaitPlacement { + PreHeader, + InStream, +} + +struct Inner { + t0: Instant, + phases: [Option; PHASE_COUNT], + headers_ready_total: Option, + request_elapsed: Option, + auction_wait_placement: Option, + resp_bytes: Option, +} + +#[derive(Clone)] +pub struct RequestTimings(Arc>); +``` + +Implementation notes (all bodies in this task, none deferred): + +- Every method takes `if let Ok(mut inner) = self.0.try_lock()` and silently + returns otherwise: contention and poison both drop the sample instead of waiting, + per the infallibility constraint. +- `record` accumulates with `saturating_add` semantics + (`Some(existing.saturating_add(dur))`). +- `mark_headers_ready` and `mark_request_elapsed` write `t0.elapsed()` only when the + slot is `None`. +- `server_timing_value` returns `None` unless `headers_ready_total` is set; renders + `ts-total` first from the stored snapshot, then the six header phases in enum order + with `{:.1}` millisecond formatting (`dur.as_secs_f64() * 1000.0`). +- `PhaseSpan { timings: RequestTimings, phase: Phase, started: Instant }`; `Drop` + calls `record(self.phase, self.started.elapsed())`. +- `TimingSnapshot` converts each `Duration` with + `u32::try_from(dur.as_millis()).unwrap_or(u32::MAX)`. +- `impl Default for RequestTimings` delegates to `new()`. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum -p trusted-server-core request_timing` +Expected: all 7 tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/request_timing.rs crates/trusted-server-core/src/lib.rs +git commit -m "Add RequestTimings phase collection and Server-Timing rendering" +``` + +--- + +### Task 2: Settings: `[observability]`, tinybird decoupling, access validation + +**Files:** + +- Modify: `crates/trusted-server-core/src/settings.rs` +- Modify: `trusted-server.example.toml` + +**Interfaces:** + +- Produces: + - `pub struct ObservabilitySettings { pub server_timing_enabled: bool }` as + `settings.observability`, `#[serde(default)]` on the field and + `#[serde(skip_serializing_if = "ObservabilitySettings::is_default")]`. + - `TinybirdSettings.auction_enabled: bool` (`#[serde(default = "default_true")]`). + - `prepare_runtime` validation: `access_enabled` requires `enabled`, non-empty + `api_host`, `secret_store`, `access_dataset`, `access_token_secret`, + `max_body_bytes > 0`, and `access_sample_rate > 0.0`. + +- [ ] **Step 1: Write the failing tests** (in `settings.rs` tests module) + +```rust +#[test] +fn observability_defaults_off_and_serializes_away() { + let settings = create_test_settings(); + assert!(!settings.observability.server_timing_enabled, "should default off"); + let toml = toml::to_string(&settings).expect("should serialize settings"); + assert!( + !toml.contains("[observability]"), + "should omit the default table so a prior binary can parse the config" + ); +} + +#[test] +fn access_enabled_requires_positive_sample_rate() { + // access_enabled = true with access_sample_rate = 0 is armed-but-silent: an error. + let err = settings_from_toml_with( + "[tinybird]\nenabled = true\napi_host = \"api.example.com\"\naccess_enabled = true\naccess_sample_rate = 0.0\n", + ) + .expect_err("should reject armed-but-silent access telemetry"); + assert!(format!("{err:?}").contains("access_sample_rate"), "should name the field"); +} + +#[test] +fn access_and_auction_emission_are_independent() { + let settings = settings_from_toml_with( + "[tinybird]\nenabled = true\napi_host = \"api.example.com\"\nauction_enabled = false\naccess_enabled = true\naccess_sample_rate = 1.0\n", + ) + .expect("should accept access without auction"); + assert!(!settings.tinybird.auction_enabled, "should disable auction emission"); + assert!(settings.tinybird.access_enabled, "should enable access emission"); +} + +#[test] +fn auction_enabled_defaults_true_for_existing_configs() { + let settings = settings_from_toml_with("[tinybird]\nenabled = true\napi_host = \"api.example.com\"\n") + .expect("should parse a pre-decoupling config"); + assert!(settings.tinybird.auction_enabled, "should preserve current behavior"); +} +``` + +Also REPLACE the existing rejection test +(`tinybird_access_enabled_is_rejected_until_emitter_is_wired`, `settings.rs:4123`) +with a wiring test asserting a fully-specified access config is accepted. + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum -p trusted-server-core observability access_enabled auction_enabled` +Expected: compile FAIL (`observability` field missing). + +- [ ] **Step 3: Implement** + +- Add `ObservabilitySettings` (derive `Debug, Clone, Default, PartialEq, Deserialize, +Serialize`, `#[serde(deny_unknown_fields)]`), with + `fn is_default(&self) -> bool { *self == Self::default() }`. +- Add the `observability` field to `Settings` with the serde attributes above. +- Add `auction_enabled` to `TinybirdSettings` with `default_true()`; update + `Default for TinybirdSettings`. +- Extend `TinybirdSettings::prepare_runtime` with the access validation matrix; error + messages name the failing field (`"tinybird.access_sample_rate must be > 0 when +access_enabled"` and so on). +- `trusted-server.example.toml`: add a commented `[observability]` block with + `server_timing_enabled = false` present-but-false and the env-override note (the + overlay cannot create a missing leaf), plus `auction_enabled`/access keys in the + tinybird section comments. +- Gate the auction sink: in `crates/trusted-server-adapter-fastly/src/app.rs`, + `auction_sink_from_settings` condition becomes + `settings.tinybird.enabled && settings.tinybird.auction_enabled`. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum -p trusted-server-core` then `cargo test-fastly` (the sink gate +touches the Fastly adapter). +Expected: PASS, including the replaced wiring test. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/settings.rs trusted-server.example.toml crates/trusted-server-adapter-fastly/src/tinybird.rs crates/trusted-server-adapter-fastly/src/app.rs +git commit -m "Add observability settings and decouple tinybird access and auction emission" +``` + +--- + +### Task 3: Fastly freeze point, header emission, `DeliveryOutcome` + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (entry T0, appbuild span, + `send_edgezero_response`) +- Test: `crates/trusted-server-adapter-fastly/src/app.rs` tests module (route-level + tests run under Viceroy) + +**Interfaces:** + +- Consumes: `RequestTimings`, `Phase` (Task 1); + `trusted_server_core::cache_policy::cache_control_headers_are_private_or_no_store`. +- Produces: + - `RequestTimings` inserted into request extensions at dispatch + (`core_req.extensions_mut().insert(timings.clone())`), alongside the existing + `config_store`/`device_signals`/`client_info` inserts. + - `send_edgezero_response(response, effects, timings) -> DeliveryOutcome` where + `pub(crate) struct DeliveryOutcome { pub bytes: u64, pub result: DeliveryResult }` + and `pub(crate) enum DeliveryResult { Complete, Error }` (streaming partial + detection lands in Task 6). + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn server_timing_emitted_on_private_response_when_enabled() { + // Arrange: settings with observability.server_timing_enabled = true; publisher + // route fixture whose response is Cache-Control: private, no-store. + // Act: dispatch through the full adapter path. + // Assert: + let header = response_header(&response, "server-timing").expect("should emit header"); + assert!(header.contains("ts-total;dur="), "should carry the stored total"); + assert_eq!( + header.matches("ts-total").count(), 1, + "should emit exactly one TS-owned metric set" + ); +} + +#[test] +fn server_timing_absent_when_flag_off() { /* same fixture, flag false: no ts-total */ } + +#[test] +fn server_timing_absent_on_cacheable_responses() { + // tsjs route (public, max-age=31536000, immutable) and a bare max-age=60 response: + // both must carry no ts-total even with the flag on. +} + +#[test] +fn preexisting_server_timing_values_survive() { + // Fixture response already carrying Server-Timing: upstream;dur=1 stays present + // alongside the appended TS set. +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-fastly server_timing` +Expected: FAIL, no header emitted. + +- [ ] **Step 3: Implement** + +In `edgezero_main` (`main.rs`): + +```rust +let timings = RequestTimings::new(); +{ + let _appbuild = timings.span(Phase::AppBuild); + // existing: open_trusted_server_config_store() + build_app_with_state() +} +``` + +Move the config-store open inside the span scope. Insert `timings.clone()` into +request extensions before dispatch. Thread the handle into both send sites and the +error paths by value (it is a cheap clone). + +In `send_edgezero_response`, immediately before `response.into_parts()`: + +```rust +timings.mark_headers_ready(); +let conclusively_private = + cache_control_headers_are_private_or_no_store(response.headers()); +if settings_enabled_server_timing && conclusively_private { + if let Some(value) = timings.server_timing_value() { + match HeaderValue::from_str(&value) { + Ok(header_value) => { + response.headers_mut().append(header::SERVER_TIMING, header_value); + } + Err(error) => log::warn!("skipping server-timing header: {error}"), + } + } +} +``` + +`settings_enabled_server_timing` arrives inside a small +`SendContext { timings: RequestTimings, server_timing_enabled: bool }` so the +function stays at or under seven parameters. Return `DeliveryOutcome` with per-mode +semantics: buffered bodies capture the byte count from the body length before +`send_to_client()` (which returns no delivery result) and report complete-on-return; +the streaming branch gains a counting writer in Task 6. Existing callers ignore the +outcome in this task (Task 8 consumes it). + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-fastly` and `cargo clippy-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/main.rs crates/trusted-server-adapter-fastly/src/app.rs +git commit -m "Emit Server-Timing at the send freeze point on conclusively private responses" +``` + +--- + +### Task 4: Filter span and geo span with `GeoLookupState` dedupe + +**Files:** + +- Modify: `crates/trusted-server-core/src/geo.rs` (add `GeoLookupState`) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` + (`run_pre_route_filters` wrapper, `build_ec_request_state` geo span + state attach) +- Modify: `crates/trusted-server-adapter-fastly/src/middleware.rs` and `main.rs` + (`resolve_geo_for_response` consumes carried state) + +**Interfaces:** + +- Consumes: `RequestTimings` from request extensions (Task 3). +- Produces: + - `pub enum GeoLookupState { NotAttempted, Attempted, Resolved(GeoInfo) }` in + `trusted_server_core::geo`, attached as a response extension on every exit path + that attempted a lookup (including the asset fallback). + - `resolve_geo_for_response` gains the carried state as input: live lookup only on + `NotAttempted`; `Attempted` is never retried; fallback lookups are wrapped in + `timings.span(Phase::Geo)`. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn finalize_reuses_request_phase_geo_without_second_lookup() { + // Counting geo stub: dispatch a publisher route; assert lookup count == 1 and + // x-geo-country still set on the response. +} + +#[test] +fn failed_lookup_is_not_retried() { + // Stub returns None once; assert GeoLookupState::Attempted carried and the + // finalize path performs zero further lookups. +} + +#[test] +fn asset_fallback_carries_geo_state_without_ec_finalize_state() { + // Asset route: response extension holds GeoLookupState, EcFinalizeState absent. +} + +#[test] +fn filter_span_recorded_when_request_filter_runs() { + // Registry fixture with a test request filter; assert snapshot().filter_ms is Some. +} + +#[test] +fn geo_lookup_skipped_for_unauthorized_responses() { + // Existing 401 rule preserved: no lookup, state NotAttempted. +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-fastly geo_ filter_span` +Expected: FAIL (lookup count 2; no `GeoLookupState`). + +- [ ] **Step 3: Implement** + +- `GeoLookupState` derives `Debug, Clone`; store in response extensions from the + dispatch layer right after `build_ec_request_state` resolves (or fails) its lookup. +- Wrap the `build_ec_request_state` lookup and any finalize fallback lookup in + `timings.span(Phase::Geo)` (accumulating slot handles the repeat case). +- Wrap `run_pre_route_filters` (`app.rs:751`) in `timings.span(Phase::Filter)`, + recording only when at least one filter is registered (skip the span when the + registry has no request filters, so the header omits `ts-filter` on unconfigured + deployments). +- `resolve_geo_for_response(response, carried: &GeoLookupState, client_ip, lookup)` + keeps the 401 short-circuit first, then matches the carried state. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-fastly` and `cargo test-axum` +Expected: PASS including untouched existing geo header tests. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/geo.rs crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-adapter-fastly/src/middleware.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Record filter and geo spans and dedupe the per-request geo lookup" +``` + +--- + +### Task 5: Core spans: origin, template cache, KV abstraction + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` (origin send ~4496, template + cache lookup ~4391 on current main) +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (graph-level `ts-kv`) +- Test: `publisher.rs` and `ec/kv.rs` test modules + +**Interfaces:** + +- Consumes: `RequestTimings` read from request extensions inside + `handle_publisher_request`; `KvIdentityGraph` gains + `pub fn with_timings(self, timings: RequestTimings) -> Self` (builder-style, + optional field), set where the graph is constructed in `main.rs`. +- Produces: `origin_ms`, `template_cache_ms`, `kv_ms` populated in snapshots. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn origin_span_covers_the_publisher_fetch() { + // Stubbed origin with a small injected delay; assert snapshot().origin_ms is Some. +} + +#[test] +fn template_cache_span_recorded_only_when_lookup_runs() { + // Inline mode fixture: template_cache_ms None. Shared-mode eligible fixture: + // template_cache_ms Some. +} + +#[test] +fn kv_span_accumulates_across_graph_operations() { + // Stub KV recording two operations through a TimedKvStore-wrapped graph; assert + // kv_ms Some and covers both (accumulated, not last-write). +} + +#[test] +fn consent_store_reads_are_timed_and_pull_sync_is_not() { + // Consent read through the decorated RuntimeServices store: kv_ms Some. + // Pull-sync graph built from the untimed store: records nothing. +} + +#[test] +fn ec_finalize_kv_lands_before_freeze() { + // Adapter-level (test-fastly): EC-enabled fixture with eids cookies; assert the + // emitted header contains ts-kv, proving the freeze point sits after finalize. +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum -p trusted-server-core origin_span template_cache_span kv_span` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +- `handle_publisher_request` reads the handle: + `let timings = req.extensions().get::().cloned().unwrap_or_default();` + (a defaulted handle records into nothing that ever renders, keeping non-adapter + tests unchanged). +- Origin: `let origin_span = timings.span(Phase::Origin);` immediately before + `services.http_client().send(platform_request).await`; `drop(origin_span)` when the + response headers are available (directly after the `match` arm binds the response). +- Template cache: same guard pattern around + `services.template_cache().lookup_or_reserve(key).await`. +- KV: add `TimedKvStore` (new type in `crates/trusted-server-core/src/platform/`), + a decorator implementing `PlatformKvStore` that wraps `Arc` + plus a `RequestTimings` handle and records `Phase::EcKv` around every trait + method. Every request-path `KvIdentityGraph` construction site (request setup, + identify, admin lookup, batch sync, finalization) receives the timed store; + consent-store access through `RuntimeServices` uses the same decorator; pull-sync + constructs its graph from the untimed store explicitly (add a test asserting the + pull-sync store records nothing). `ec_finalize_response` keeps seven arguments: + the handle rides inside the store the graph already receives. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum` then `cargo test-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs crates/trusted-server-core/src/ec/kv.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Record origin, template cache, and KV phase spans in core" +``` + +--- + +### Task 6: Body-phase capture: stream, auction wait placement, bytes + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` (seam wait + buffered wait) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (stream drive timing, + `DeliveryOutcome.bytes`) +- Test: `publisher.rs` tests + adapter tests + +**Interfaces:** + +- Consumes: `record_auction_wait` (Task 1), `DeliveryOutcome` (Task 3). +- Produces: `stream_ms`, `auction_wait_ms` + placement, `resp_bytes`, + `mark_request_elapsed()` called by the adapter immediately after the stream drive + returns. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn streaming_seam_wait_records_in_stream_placement() { + // Streaming fixture with a delayed auction: placement InStream, and + // stream_ms >= auction_wait_ms. +} + +#[test] +fn buffered_template_miss_records_pre_header_placement() { + // Shared-template authorized miss (buffered finalizer): placement PreHeader; the + // wait is recorded even though headers had not committed. +} + +#[test] +fn delivery_outcome_reports_bytes_and_request_elapsed_set() { + // Adapter: after send, snapshot has resp_bytes Some(body_len) and + // request_elapsed_ms Some; request_elapsed excludes post-send emitter time by + // construction (asserted by ordering test in Task 8). +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-fastly seam_wait buffered_template delivery_outcome` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +- Streaming path: around the `collect_stream_auction(...)` await inside the body + stream, measure with `Instant::now()` and call + `timings.record_auction_wait(AuctionWaitPlacement::InStream, waited)`. The handle + reaches the stream closure through `OwnedProcessResponseParams`/assembly params (it + is `Clone`; add a field). +- Buffered path (`buffer_publisher_response_async` and the shared-template miss + finalizer): same measurement with `AuctionWaitPlacement::PreHeader`. +- Adapter stream drive: wrap the `block_on(stream_asset_body(...))` region with a + counting writer that tallies bytes and observes truncation/error, record + `Phase::Stream` with the elapsed drive time, populate `DeliveryOutcome` with + bytes and Complete/Partial/Error, call `timings.set_resp_bytes(bytes)` and + `timings.mark_request_elapsed()` immediately after the drive returns, before + anything else post-send. Buffered responses keep the Task 3 complete-on-return + semantics; `body_mode` distinguishes the regimes in the row. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-fastly` and `cargo test-axum` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Capture stream duration, auction wait placement, and response bytes" +``` + +--- + +### Task 7: `AccessTelemetrySnapshot`, route class, route template + +**Files:** + +- Create: `crates/trusted-server-core/src/access_telemetry.rs` +- Modify: `crates/trusted-server-core/src/lib.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (RouteMetadata attach at + handler wrappers), `main.rs` (snapshot build at freeze point), + `crates/trusted-server-core/src/publisher.rs` (typed template-cache state + extension) + +**Interfaces:** + +- Consumes: `TimingSnapshot` (Task 1), `GeoLookupState` (Task 4). +- Produces: + - `pub enum RouteClass { PublisherHtml, Tsjs, IntegrationProxy, Ec, AuctionApi, Other }` + with `pub fn as_str(&self) -> &'static str` (snake_case values from the spec). + - `pub fn publisher_route_template(path: &str) -> String`: `/` plus first segment + filtered to `[a-z0-9_-]`, truncated to 32 chars, plus `/*` when deeper; empty or + disallowed first segments render `/other/*`. + - `pub struct AccessTelemetrySnapshot { pub method: String, pub status: u16, pub route_class: RouteClass, pub route_template: String, pub publisher_domain: String, pub env: String, pub service_id: String, pub pop: String, pub ts_version: String, pub country: String, pub template_cache_state: String, pub body_mode: &'static str, pub sample_rate: f64 }` + - `pub fn access_event_row(snapshot: &AccessTelemetrySnapshot, timings: &TimingSnapshot, event_ts_epoch_ms: u64) -> String` (one NDJSON line). + +- [ ] **Step 1: Write the failing tests** (adversarial, per spec 9) + +```rust +#[test] +fn admin_ec_route_template_never_contains_the_identifier() { + // Named-route template comes from the route table: "/_ts/admin/ec/{id}". + // Assert a row built for that route never contains a 64-hex EC id fixture. +} + +#[test] +fn publisher_paths_normalize_to_coarse_templates() { + assert_eq!(publisher_route_template("/news/some-article-slug"), "/news/*"); + assert_eq!(publisher_route_template("/"), "/"); + assert_eq!( + publisher_route_template("/user@example.com/profile"), + "/other/*", + "should reject non-allowlisted characters" + ); + assert_eq!( + publisher_route_template(&format!("/{}", "a".repeat(500))), + format!("/{}", "a".repeat(32)), + "should bound segment length" + ); + assert_eq!(publisher_route_template("/search terms here"), "/other/*"); +} + +#[test] +fn row_serializes_nulls_for_missing_phases() { + // Sparse TimingSnapshot: absent phases serialize as JSON null, dimension fields + // never null (unknown sentinel). +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum -p trusted-server-core access_telemetry route_template` +Expected: compile FAIL. + +- [ ] **Step 3: Implement** + +Row serialization via `serde_json::json!` mapping spec section 9 column names exactly +(`time_elapsed_ms`, `appbuild_ms`, ..., `auction_wait_placement` as +`pre_header|in_stream|none`). `pop`/`service_id` read from Fastly env +(`FASTLY_SERVICE_ID`, `FASTLY_POP`), defaulting `"unknown"`; `env` derived by the +adapter from `FASTLY_IS_STAGING` (the `x-ts-env` input), never from `Settings`. +Route identity travels as a typed `RouteMetadata` response extension +(`pub struct RouteMetadata { pub route_class: RouteClass, pub route_template: String }` +in `access_telemetry.rs`): each named-route handler wrapper attaches its matched +route-table pattern verbatim, and the fallback and tsjs handlers attach their class +plus the coarse template; the freeze point consumes the extension (no `RouteClass` +column in `NAMED_ROUTES`, no reconstruction from a handler enum). Also in this task: +make `TemplateCacheResponseState` a typed response extension in `publisher.rs`, set +at every point that writes `x-ts-template-cache` so header and extension cannot +drift; the row reads the extension. The snapshot is built (when access +telemetry is enabled) in +`send_edgezero_response` right after `mark_headers_ready()` and returned inside +`DeliveryOutcome` (add field `pub snapshot: AccessTelemetrySnapshot`). + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum` and `cargo test-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/access_telemetry.rs crates/trusted-server-core/src/lib.rs crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Add access telemetry snapshot, route classes, and coarse route templates" +``` + +--- + +### Task 8: Access sink with confirmed delivery + post-send ordering + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/tinybird.rs` (access sink) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (post-send ordering) + +**Interfaces:** + +- Consumes: `AccessTelemetrySnapshot` + `access_event_row` (Task 7), settings flags + (Task 2), `DeliveryOutcome` (Tasks 3/6). +- Produces: `pub(crate) async fn emit_access_event(client: &FastlyPlatformHttpClient, target: &TinybirdEventsTarget, row: String) -> Result<(), Report>`, + sending via the adapter's stateless platform client (the blocking variant, + post-delivery), checking `response.status().is_success()`, warning with status + otherwise. The transport context is adapter-owned and route-independent (target + derived from settings once at entry), so asset, admin, and error responses emit + without `RuntimeServices` or `EcFinalizeState`. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn access_emitter_posts_ndjson_and_validates_2xx() { + // RecordingHttpClient returning 202: assert URI is /v0/events?name=access_logs_raw, + // body is the row, Authorization bearer from the secret stub. +} + +#[test] +fn access_emitter_warns_and_drops_on_non_2xx() { + // RecordingHttpClient returning 422: emit returns Err naming the status; no retry + // request recorded (exactly one request seen). +} + +#[test] +fn sampled_out_requests_emit_nothing() { + // access_sample_rate stub decision false: RecordingHttpClient sees zero requests. +} + +#[test] +fn post_send_order_is_elapsed_then_pull_sync_then_telemetry() { + // Instrumented stubs record call order; assert request_elapsed snapshot precedes + // pull-sync dispatch which precedes the telemetry send. +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-fastly access_emitter post_send_order` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +- Reuse `TinybirdEventsTarget` with a second constructor + `from_access_config(config: TinybirdSettings)` using `access_dataset` and + `access_token_secret`. +- Sampling decision: `fn sampled_in(rate: f64, entropy: u64) -> bool` where entropy is + derived from the event timestamp nanos XOR a per-request counter (no `rand` + dependency; document that uniformity is approximate and sufficient). +- `main.rs` post-send, in order: `timings.mark_request_elapsed()` (already placed in + Task 6), existing pull-sync dispatch unchanged, then when + `settings.tinybird.enabled && settings.tinybird.access_enabled` and sampled in: + build the row from `outcome.snapshot` + `timings.snapshot()`, call + `emit_access_event`, log one warning on `Err`. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-fastly` and `cargo clippy-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/tinybird.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Emit confirmed access telemetry rows after pull-sync post-send" +``` + +--- + +### Task 9: Tinybird datasource schema + +**Files:** + +- Modify: `tinybird/datasources/access_logs_raw.datasource` + +**Interfaces:** + +- Consumes: column names exactly as serialized by `access_event_row` (Task 7). +- Produces: the deployed schema contract for the dashboard (separate repo). + +- [ ] **Step 1: Rewrite the schema** per spec section 9: keep + `event_ts DateTime64(3)`, `method`, `status UInt16`, `time_elapsed_ms UInt32`, + `sample_rate Float64` + 30-day TTL; add the columns from spec 9 with + dimension columns non-nullable `LowCardinality(String)` and phase columns + `Nullable(UInt32)`; drop `path` and `cache_state`; set + `ENGINE_SORTING_KEY "toDate(event_ts), service_id, publisher_domain, env, route_class, pop, status"` + (`event_date` was dropped for the sorting-key expression; see spec section 9). + +- [ ] **Step 2: Validate** with the tinybird toolchain if available locally + (`tb check` / project tests under `tinybird/tests`); otherwise assert the file + parses by review and rely on rollout step 4's remote verification. Add a fixture row + in `tinybird/fixtures` matching `access_event_row` output. + +- [ ] **Step 3: Commit** + +```bash +git add tinybird/datasources/access_logs_raw.datasource tinybird/fixtures +git commit -m "Extend access_logs_raw with phase columns and a non-null sorting key" +``` + +--- + +### Task 10: Axum adapter emission + +**Files:** + +- Modify: `crates/trusted-server-adapter-axum/src/` (terminal layer at the response + serialization boundary; locate the equivalent of the Fastly send path) +- Test: axum adapter tests (`cargo test-axum`) + +**Interfaces:** + +- Consumes: `RequestTimings`, header emission helper. Extract the emission block from + Task 3 into a shared core helper so both adapters call one function: + `pub fn append_server_timing_if_private(response: &mut Response, timings: &RequestTimings, enabled: bool)` + in `request_timing.rs` (move the Fastly inline logic here and re-point Task 3's call + site). +- Produces: Axum responses carry the header under the same conservative predicate; + `ts-appbuild` absent by construction (state built at startup); router-generated + 404/405 covered by the terminal layer; `/health` excluded by route match. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn axum_emits_header_on_private_response() { /* flag on, private response: ts-total present, ts-appbuild absent */ } + +#[test] +fn axum_404_carries_header_when_private() { /* router-generated 404 passes through the terminal layer */ } + +#[test] +fn axum_health_is_excluded() { /* /health: no ts-total */ } +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum axum_emits axum_404 axum_health` +Expected: FAIL. + +- [ ] **Step 3: Implement** an outer service wrapper around the `RouterService` + inside `AxumDevServer` (not router middleware, which router-generated 404/405 + responses bypass and which returns before body serialization): create + `RequestTimings::new()` per request in the wrapper, insert into request + extensions, and on the wrapper's response side call `mark_headers_ready()` + + `append_server_timing_if_private(...)`, skipping the `/health` path by match. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-adapter-axum/src crates/trusted-server-core/src/request_timing.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Emit Server-Timing from the Axum terminal layer with adapter-specific semantics" +``` + +--- + +### Task 11: Full gate, docs, and PR + +- [ ] **Step 1: Docs.** Add a short operator section to `docs/guide/configuration.md`: + the `[observability]` flag, the tinybird access keys, the deploy/rollback ordering + from spec section 12 (binary first, config second; config first on rollback), and + the conservative emission rule. Run `cd docs && npm run format`. + +- [ ] **Step 2: Full CI gate list** from `CLAUDE.md`: + `cargo fmt --all -- --check`; all six clippy aliases; `test-fastly`, `test-axum`, + `test-cloudflare`, `test-spin`; the integration-tests parity suite; JS build/test + and formats. Cloudflare/Spin compile the new core modules (collection only), which + is exactly what the non-goal requires. + +- [ ] **Step 3: Commit docs, push the branch, open the implementation PR** referencing + the spec PR #1069 and issue #1068, with the rollout section of the spec quoted as + the deployment checklist (staging pass-through + MISS/HIT replay before production + flag-on). + +--- + +## Self-Review + +- Spec coverage: sections 5 (Task 1), 12 (Task 2), 7 (Tasks 3, 10), 8/8a (Tasks 4, + 10), 6 (Tasks 4-6), 9 (Tasks 7, 9), 10 (Task 8), 13 (Tasks 1, 3, 8), 14 (test + steps throughout), 15 steps 1-4 (Task 11 + deployment checklist). Section 11 + (dashboard) is explicitly out of scope for this repo's plan. +- Type consistency: `RequestTimings`/`TimingSnapshot`/`RouteClass`/ + `AccessTelemetrySnapshot`/`DeliveryOutcome` names and signatures match across + Tasks 1, 3, 6, 7, 8, 10. +- Known intentional deferral: `DeliveryResult::Partial` detection is named in Task 3 + and wired when the stream drive reports bytes in Task 6; no other deferrals. diff --git a/docs/superpowers/plans/2026-08-26-auction-timeline-offsets.md b/docs/superpowers/plans/2026-08-26-auction-timeline-offsets.md new file mode 100644 index 000000000..6dfff6c8c --- /dev/null +++ b/docs/superpowers/plans/2026-08-26-auction-timeline-offsets.md @@ -0,0 +1,64 @@ +# Auction Timeline Offsets Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Record three T0-anchored auction milestones (dispatched, resolved, committed) plus the auction id on `RequestTimings`, and emit them as four additive columns on the `access_logs_raw` row. + +**Architecture:** Follows spec section 18 exactly. All state lives in the existing `RequestTimings` inner (same `try_lock`/first-call-wins/saturating model as `mark_headers_ready`); the row builder reads the values from `TimingSnapshot`, so no new emission path and no adapter changes. + +**Tech Stack:** Rust (core crate only), Tinybird datasource file. + +**Spec:** `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md` section 18. + +## Global Constraints + +- Marks are first-call-wins; `try_lock` only; a contended lock drops the sample. +- Null offsets mean "no auction ran", never zero. `auction_id` sentinel is `none`. +- Column names: `auction_dispatched_ms`, `auction_resolved_ms`, `auction_committed_ms`, `auction_id`; JSONPaths `json:$.`; FORWARD_QUERY extended in the same order. +- Dispatch mark records only on `DispatchAuctionOutcome::Dispatched`; a failed dispatch leaves all three offsets null (the auction dataset still records the failure). +- No header emission, no config surface, no changes outside `trusted-server-core` and `tinybird/`. + +--- + +### Task 1: RequestTimings marks and snapshot fields + +**Files:** + +- Modify: `crates/trusted-server-core/src/request_timing.rs` + +**Interfaces:** + +- Produces: `mark_auction_dispatched(&self, auction_id: String)`, `mark_auction_resolved(&self)`, `mark_auction_committed(&self)`; `TimingSnapshot { auction_dispatched_ms, auction_resolved_ms, auction_committed_ms: Option, auction_id: Option, .. }` + +- [ ] Add `auction_dispatched`, `auction_resolved`, `auction_committed: Option` and `auction_id: Option` to `Inner`; initialize `None`. +- [ ] Add the three mark methods, first-call-wins on their own field, storing `inner.t0.elapsed()`; dispatched also stores the id first-call-wins. +- [ ] Map all four into `TimingSnapshot` via `duration_ms` / clone. +- [ ] Tests: first-call-wins per mark; snapshot maps offsets and id; unmarked snapshot yields all `None`. +- [ ] `cargo test-fastly request_timing`, commit. + +### Task 2: Publisher call sites + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +**Interfaces:** + +- Consumes: Task 1 methods; `observation.auction_id` (`AuctionObservationContext`), in scope at the dispatch site. + +- [ ] In the `DispatchAuctionOutcome::Dispatched` arm (~line 4341): `timings.mark_auction_dispatched(observation.auction_id.to_string());` +- [ ] After both `record_auction_wait` calls (collect sites ~3952 and ~4012): `mark_auction_resolved()`. +- [ ] After both `write_bids_to_state` calls (~3954 and ~4017): `mark_auction_committed()`. +- [ ] `cargo test-fastly`, commit. + +### Task 3: Row columns and datasource + +**Files:** + +- Modify: `crates/trusted-server-core/src/access_telemetry.rs` +- Modify: `tinybird/datasources/access_logs_raw.datasource` + +- [ ] `access_event_row`: add the three offset keys (nullable) and `auction_id` with `none` sentinel, after the existing phase keys. +- [ ] Extend `row_serializes_nulls_for_missing_phases` and `row_serializes_recorded_phases_as_numbers` for the new keys. +- [ ] Datasource: four schema columns with JSONPaths (`Nullable(UInt32)` ×3, `String`), appended at the end of SCHEMA and FORWARD_QUERY so existing column order stays stable. +- [ ] Full gates: fmt, clippy (all six), test-fastly/axum/cloudflare/spin, parity. Commit. diff --git a/docs/superpowers/plans/2026-08-28-managed-user-id-bundle-validation.md b/docs/superpowers/plans/2026-08-28-managed-user-id-bundle-validation.md new file mode 100644 index 000000000..2265e49b2 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-managed-user-id-bundle-validation.md @@ -0,0 +1,680 @@ +# Managed User ID Bundle Validation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `ts prebid bundle` fail before updating deployable metadata when a configured managed User ID name is unknown, ambiguous, or missing its required module from the freshly generated bundle manifest. + +**Architecture:** Extend the CLI's focused TOML reader with managed User ID names, resolve them through the same checked-in JSON registry used by the JavaScript generator, invalidate any stale output manifest, and validate the newly generated manifest before patching hash/SRI metadata. Keep core vendor-neutral and retain the existing browser diagnostic as defense in depth. + +**Tech Stack:** Rust 2024, Serde/serde_json, TOML/toml_edit, host-target CLI tests, Prettier Markdown formatting. + +**Specification:** `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + +**Working tree:** Use the existing `issue-355-liveramp-integration` branch as explicitly requested by the user. Do not create a worktree and do not push without separate authorization. + +--- + +## File structure + +| File | Responsibility | +| ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-cli/src/prebid_bundle.rs` | Parse managed names, load/resolve the registry, invalidate stale manifests, validate the generated manifest, and contain focused unit/command tests. | +| `docs/guide/integrations/prebid.md` | Explain the registry-backed bundle failure and runtime fallback diagnostic. | +| `docs/guide/configuration.md` | Replace the obsolete “not validated” configuration warning. | +| `trusted-server.example.toml` | Tell operators that the bundle command validates managed-name/module pairing. | + +No core, TypeScript runtime, JavaScript generator, registry schema, manifest producer, or public configuration shape changes are required. + +## Task 1: Strictly parse managed User ID names + +**Files:** + +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:32-37` +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:182-248` +- Test: `crates/trusted-server-cli/src/prebid_bundle.rs:558-683` + +- [ ] **Step 1: Write failing parser tests** + +Add tests proving an absent list becomes empty, valid entries preserve order, and malformed values fail instead of being skipped: + +```rust +#[test] +fn bundle_config_loader_reads_managed_user_id_names_in_order() { + let (_temp, path) = write_config( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" + +[[integrations.prebid.managed_user_ids]] +name = "pubCommonId" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"#, + ); + + let config = load_bundle_config(&path).expect("should load managed names"); + + assert_eq!( + config.managed_user_id_names, + ["identityLink", "pubCommonId"], + "should preserve managed entry order" + ); +} + +#[test] +fn bundle_config_loader_rejects_managed_entry_without_string_name() { + let (_temp, path) = write_config( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +managed_user_ids = [{ params = { pid = "999" } }] + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"#, + ); + + let error = load_bundle_config(&path).expect_err("should reject missing managed name"); + + assert!( + error.contains("integrations.prebid.managed_user_ids[0].name"), + "should identify the malformed managed entry: {error}" + ); +} +``` + +Add separate cases for: + +- `managed_user_ids` being a string/table rather than an array; +- an array element being a string rather than a table; +- a missing `name`; +- a non-string `name`; +- an empty or whitespace-only `name`. + +Also extend the existing missing-list test to assert `managed_user_id_names.is_empty()`. + +- [ ] **Step 2: Run the CLI suite and verify the new tests fail** + +Run: + +```bash +./scripts/test-cli.sh +``` + +Expected: FAIL because `PrebidBundleConfig` has no `managed_user_id_names` field and no strict reader exists. + +- [ ] **Step 3: Implement the focused TOML reader** + +Extend the config structure: + +```rust +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PrebidBundleConfig { + pub adapters: Vec, + pub user_id_modules: Option>, + pub managed_user_id_names: Vec, + pub external_bundle_url: Option, +} +``` + +Add a narrow helper; do not deserialize or validate vendor parameters: + +```rust +fn read_managed_user_id_names( + prebid: &toml::Value, + config_path: &Path, +) -> CliResult> { + let Some(value) = prebid.get("managed_user_ids") else { + return Ok(Vec::new()); + }; + let entries = value.as_array().ok_or_else(|| { + report_error(format!( + "{} integrations.prebid.managed_user_ids must be an array of tables", + config_path.display() + )) + })?; + + entries + .iter() + .enumerate() + .map(|(index, entry)| { + let table = entry.as_table().ok_or_else(|| { + report_error(format!( + "{} integrations.prebid.managed_user_ids[{index}] must be a table", + config_path.display() + )) + })?; + let field = format!("integrations.prebid.managed_user_ids[{index}].name"); + let name = table.get("name").and_then(toml::Value::as_str).ok_or_else(|| { + report_error(format!( + "{} {field} must be a non-empty string", + config_path.display() + )) + })?; + if name.trim().is_empty() { + return cli_error(format!( + "{} {field} must be a non-empty string", + config_path.display() + )); + } + Ok(name.to_string()) + }) + .collect() +} +``` + +Call it from `load_bundle_config` and store the result. Keep full token, duplicate-name, params, and storage validation in core; the CLI validates only fields required for bundle consistency. + +- [ ] **Step 4: Run the CLI suite and verify it passes** + +Run: `./scripts/test-cli.sh` + +Expected: all `trusted-server-cli` tests PASS. + +- [ ] **Step 5: Commit locally** + +```bash +git add crates/trusted-server-cli/src/prebid_bundle.rs +git commit -m "Parse managed User ID bundle inputs" +``` + +Do not push. + +## Task 2: Resolve managed names through the shared registry + +**Files:** + +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:10-12` +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:121-128` +- Test: `crates/trusted-server-cli/src/prebid_bundle.rs:547-924` +- Read-only contract: `crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json` + +- [ ] **Step 1: Write failing registry-resolution tests** + +Define tests around an in-memory registry: + +```rust +#[test] +fn managed_names_resolve_aliases_to_registered_modules() { + let registry = PrebidUserIdModuleRegistry { + modules: vec![PrebidUserIdModuleRegistryEntry { + module_name: "sharedIdSystem".to_string(), + config_names: vec!["sharedId".to_string(), "pubCommonId".to_string()], + }], + }; + let registry_path = Path::new("user_id_modules.json"); + + let required = resolve_managed_user_id_modules( + &["pubCommonId".to_string(), "sharedId".to_string()], + ®istry, + registry_path, + ) + .expect("should resolve aliases"); + + assert_eq!( + required, + [ + RequiredPrebidUserIdModule { + config_name: "pubCommonId".to_string(), + module_name: "sharedIdSystem".to_string(), + }, + RequiredPrebidUserIdModule { + config_name: "sharedId".to_string(), + module_name: "sharedIdSystem".to_string(), + }, + ], + "should retain each managed name while allowing a shared module" + ); +} +``` + +Add cases proving: + +- `identityLink` resolves to `identityLinkIdSystem` from the actual checked-in registry; +- an unknown name fails and identifies the name plus registry path; +- a synthetic name mapped to two distinct modules fails and lists both candidates deterministically; +- an empty managed-name list returns an empty requirement list. + +- [ ] **Step 2: Run the CLI suite and verify the tests fail** + +Run: `./scripts/test-cli.sh` + +Expected: FAIL because the registry types, loader, and resolver do not exist. + +- [ ] **Step 3: Implement registry loading and deterministic resolution** + +Add vendor-neutral types: + +```rust +const USER_ID_REGISTRY_RELATIVE_PATH: &str = + "src/integrations/prebid/user_id_modules.json"; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PrebidUserIdModuleRegistry { + modules: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PrebidUserIdModuleRegistryEntry { + module_name: String, + config_names: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RequiredPrebidUserIdModule { + config_name: String, + module_name: String, +} +``` + +Load the exact file beneath the already-resolved JS library directory: + +```rust +fn load_user_id_registry( + js_lib_dir: &Path, +) -> CliResult<(PathBuf, PrebidUserIdModuleRegistry)> { + let path = js_lib_dir.join(USER_ID_REGISTRY_RELATIVE_PATH); + let contents = fs::read_to_string(&path).map_err(|error| { + report_error(format!( + "failed to read Prebid User ID registry {}: {error}", + path.display() + )) + })?; + let registry = serde_json::from_str(&contents).map_err(|error| { + report_error(format!( + "failed to parse Prebid User ID registry {}: {error}", + path.display() + )) + })?; + Ok((path, registry)) +} +``` + +Implement `resolve_managed_user_id_modules` with these rules: + +1. Collect matching `module_name` values for every exact `config_names` match. +2. Sort and deduplicate candidate modules for deterministic diagnostics. +3. Zero candidates: fail with managed name and registry path. +4. One candidate: return a requirement retaining both config and module names. +5. More than one candidate: fail with the managed name, registry path, and candidates. + +Do not hardcode `identityLink`, `identityLinkIdSystem`, `liveramp.com`, or any other vendor/module name in production code. + +- [ ] **Step 4: Run the CLI suite and verify it passes** + +Run: `./scripts/test-cli.sh` + +Expected: all CLI tests PASS, including the checked-in registry contract. + +- [ ] **Step 5: Commit locally** + +```bash +git add crates/trusted-server-cli/src/prebid_bundle.rs +git commit -m "Resolve managed User IDs through the bundle registry" +``` + +Do not push. + +## Task 3: Require a fresh manifest containing every managed module + +**Files:** + +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:121-180` +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:419-454` +- Test: `crates/trusted-server-cli/src/prebid_bundle.rs:797-907` + +- [ ] **Step 1: Make the fake generator express exact manifest behavior** + +Replace `write_manifest: bool` with an optional complete JSON document: + +```rust +struct FakeGenerator { + generate_error: Option, + generate_calls: Vec, + manifest: Option, +} +``` + +When the option is `Some`, write that exact JSON value to `manifest.json`. When +it is `None`, return according to `generate_error` without writing a manifest. +Add a `fake_manifest(user_id_modules: serde_json::Value)` helper that returns the +otherwise-valid manifest object. This lets tests emit a valid array, a non-array +value, or a complete object with `userIdModules` removed. Update existing tests +without changing their intent. + +- [ ] **Step 2: Write failing command-level consistency tests** + +Add command-level tests proving: + +```rust +#[test] +fn run_bundle_rejects_managed_name_when_manifest_omits_required_module() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(serde_json::json!(["sharedIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject missing managed module"); + + assert!(error.contains("identityLink"), "should name managed config: {error}"); + assert!( + error.contains("identityLinkIdSystem"), + "should name required module: {error}" + ); + assert!( + error.contains("integrations.prebid.bundle.user_id_modules"), + "should identify corrective field: {error}" + ); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original, + "should not patch metadata after consistency failure" + ); +} +``` + +Add cases proving: + +- the same managed config passes when the manifest contains `identityLinkIdSystem`; +- two managed names require both modules; +- two aliases backed by `sharedIdSystem` both pass with one manifest module; +- omission of `bundle.user_id_modules` passes when the fake generated manifest contains the default module; +- unknown and malformed names fail through `run_bundle` before `generate_calls` + receives an entry, with the entire original config (including existing + hash/SRI metadata) unchanged; +- an ambiguous name fails through the private registry-injected orchestration + seam described in Step 6 before `generate_calls` receives an entry, with the + entire original config (including existing hash/SRI metadata) unchanged; +- fake manifests with a missing or non-array `userIdModules` field fail + manifest parsing; +- a missing required module never changes existing hash/SRI metadata. + +- [ ] **Step 3: Write the failing stale-manifest regression test** + +Prepopulate `/manifest.json` with valid old metadata, make the fake generator return success without writing, then assert: + +- `run_bundle` fails to read the generated manifest; +- the old manifest no longer exists; +- config metadata is unchanged. + +Run: `./scripts/test-cli.sh` + +Expected: FAIL because the current CLI accepts an old manifest and does not validate `userIdModules`. + +- [ ] **Step 4: Extend manifest deserialization and validation** + +```rust +#[derive(Debug, Deserialize)] +struct PrebidBundleManifest { + #[serde(rename = "userIdModules")] + user_id_modules: Vec, + sha256: String, + sri: String, + filename: String, +} + +fn validate_managed_user_id_modules( + requirements: &[RequiredPrebidUserIdModule], + manifest: &PrebidBundleManifest, + config_path: &Path, +) -> CliResult<()> { + for requirement in requirements { + if !manifest + .user_id_modules + .iter() + .any(|module| module == &requirement.module_name) + { + return cli_error(format!( + "{} configures managed User ID {:?}, which requires Prebid module {:?}, but the generated manifest omits it; add {:?} to integrations.prebid.bundle.user_id_modules and rerun `ts prebid bundle`", + config_path.display(), + requirement.config_name, + requirement.module_name, + requirement.module_name, + )); + } + } + Ok(()) +} +``` + +Serde must reject a missing or non-array `userIdModules` field. Do not default it to an empty list. + +- [ ] **Step 5: Invalidate only the exact old manifest before generation** + +```rust +fn invalidate_manifest(path: &Path) -> CliResult<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => cli_error(format!( + "failed to remove stale Prebid manifest {}: {error}", + path.display() + )), + } +} +``` + +Do not delete the output directory or any generated bundle files. + +- [ ] **Step 6: Add a private registry-injected orchestration seam** + +Keep public command behavior in `run_bundle`, but move the post-registry workflow +into a private helper so command ordering can be tested with a synthetic +ambiguous registry: + +```rust +struct PrebidBundleRunContext<'a> { + current_dir: &'a Path, + js_lib_dir: PathBuf, + registry_path: &'a Path, + registry: &'a PrebidUserIdModuleRegistry, +} + +fn run_bundle_with_context( + args: &PrebidBundleArgs, + config: PrebidBundleConfig, + context: PrebidBundleRunContext<'_>, + generator: &mut dyn PrebidBundleGenerator, + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()> { + // Resolve requirements before output mutation or generator invocation, + // then perform generation, manifest validation, and metadata patching. +} +``` + +`run_bundle` must load config, determine the current/JS directories, load the +real registry, then delegate. Tests may pass a synthetic registry only to this +private helper. + +- [ ] **Step 7: Wire the command in the specified order** + +Implement the helper workflow in this order: + +1. Load focused config. +2. Locate the JS directory. +3. Load the shared registry. +4. Resolve all managed names; return before generator invocation on failure. +5. Ensure the output directory is writable. +6. Invalidate only `/manifest.json`. +7. Invoke the generator. +8. Load the newly created manifest. +9. Validate all resolved requirements. +10. Patch hash/SRI metadata. + +Keep `external_bundle_url` output behavior unchanged. Add the synthetic +ambiguous-registry command test now and assert the fake generator has zero +calls and the original config remains byte-for-byte unchanged. + +- [ ] **Step 8: Run the CLI suite and verify it passes** + +Run: `./scripts/test-cli.sh` + +Expected: all CLI tests PASS. + +- [ ] **Step 9: Run formatting and CLI lint** + +Run: + +```bash +cargo fmt --all -- --check +cargo clippy-cli +``` + +Expected: both commands exit 0 with no warnings. + +- [ ] **Step 10: Commit locally** + +```bash +git add crates/trusted-server-cli/src/prebid_bundle.rs +git commit -m "Reject incomplete managed User ID bundles" +``` + +Do not push. + +## Task 4: Document the build-time guard + +**Files:** + +- Modify: `docs/guide/integrations/prebid.md:487-527` +- Modify: `docs/guide/configuration.md:1270-1293` +- Modify: `trusted-server.example.toml:420-445` + +- [ ] **Step 1: Replace obsolete unvalidated-pairing guidance** + +Document these exact semantics in both guides: + +- `ts prebid bundle` resolves each managed name through `user_id_modules.json`; +- unknown or ambiguous config names fail; +- the command confirms required modules in the newly generated manifest; +- failure identifies the managed name/module and does not update hash/SRI; +- the browser diagnostic remains useful for external, stale, or modified bundles; +- core remains vendor-neutral and continues to forward `params` opaquely. + +Replace the example-file warning with concise wording such as: + +```toml +# `ts prebid bundle` resolves every managed name through the checked-in User ID +# registry and fails if the generated manifest omits its required module. +``` + +- [ ] **Step 2: Verify documentation no longer claims the pairing is unvalidated** + +Run: + +```bash +rg -n "Nothing validates|not validated|pairing is not validated" \ + docs/guide/integrations/prebid.md \ + docs/guide/configuration.md \ + trusted-server.example.toml +``` + +Expected: no matches. + +- [ ] **Step 3: Format and check documentation** + +Run: + +```bash +cd docs +npm run format:write +npm run format +``` + +Expected: Prettier writes any required formatting changes, then reports all +documentation files formatted. + +- [ ] **Step 4: Commit locally** + +```bash +git add docs/guide/integrations/prebid.md docs/guide/configuration.md trusted-server.example.toml +git commit -m "Document managed User ID bundle validation" +``` + +Do not push. + +## Task 5: Final verification and local review + +**Files:** + +- Review: all files changed since `origin/issue-355-liveramp-integration` + +- [ ] **Step 1: Run the focused gate** + +```bash +./scripts/test-cli.sh +cargo clippy-cli +cargo fmt --all -- --check +cd docs && npm run format +``` + +Expected: every command exits 0; no test, lint, or formatting failures. + +- [ ] **Step 2: Run the broader PR regression gate** + +The branch also contains core and JS LiveRamp work, so run the repository-required relevant suites: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy-codegen +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cd crates/trusted-server-js/lib +npx vitest run +node build-all.mjs +npm run format +``` + +Expected: all commands exit 0. If an environment-dependent suite cannot run, record the exact command and error rather than claiming it passed. + +- [ ] **Step 3: Review the complete local diff** + +```bash +git status --short --branch +git diff --check origin/issue-355-liveramp-integration...HEAD +git diff --stat origin/issue-355-liveramp-integration...HEAD +git log --oneline origin/issue-355-liveramp-integration..HEAD +``` + +Confirm: + +- production CLI code contains no vendor name; +- registry and manifest are the only mapping/inclusion sources; +- malformed input cannot be silently skipped; +- stale manifests cannot be reused; +- metadata is patched only after validation; +- unrelated `main` changes are present only through the local merge commit; +- no secrets, Placement IDs, or envelope values were added. + +- [ ] **Step 4: Request code review** + +Invoke `@superpowers:requesting-code-review` against the final local diff and address any verified findings one at a time. + +- [ ] **Step 5: Stop before remote mutation** + +Report the local commits, verification evidence, and any remaining live-validation work. Do not push, update PR #1054, reply to GitHub comments, or change the draft state without explicit user authorization. diff --git a/docs/superpowers/plans/2026-08-31-managed-user-id-consent-activation.md b/docs/superpowers/plans/2026-08-31-managed-user-id-consent-activation.md new file mode 100644 index 000000000..1ce7cc2a8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-31-managed-user-id-consent-activation.md @@ -0,0 +1,316 @@ +# Managed User ID Consent Activation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ensure server-managed Prebid User IDs activate the existing TCF enforcement modules when an IAB CMP is present, without overwriting publisher-owned consent configuration. + +**Architecture:** Add one focused browser-side helper in the existing Prebid shim. It reads effective Prebid consent configuration, recognizes any existing own `gdpr` property as publisher-owned, and otherwise installs only `gdpr.cmpApi = "iab"` when managed User IDs and a callable `window.__tcfapi` are present. The `setConfig`/`mergeConfig` wrappers preserve publisher precedence; when a later call first claims GDPR ownership, they retire the automatically created IAB collector before forwarding the publisher value so stale CMP events cannot overwrite it. + +**Tech Stack:** TypeScript, Prebid.js 10.26.0, Vitest, JSDOM, generated external Prebid bundle, Markdown. + +--- + +## File structure + +- Modify `crates/trusted-server-js/lib/src/integrations/prebid/index.ts`: detect and install the minimum managed-ID TCF configuration before managed IDs are seeded. +- Modify `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts`: unit-test activation conditions, merge semantics, malformed values, and publisher precedence. +- Modify `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs`: prove the real generated bundle blocks IdentityLink without publisher-side Prebid consent setup. +- Modify `docs/guide/integrations/prebid.md`: document automatic activation and ownership boundaries. +- Modify `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md`: mark the consent hardening and bundle guard as implemented and record sanitized live-validation results accurately. + +### Task 1: Add failing shim unit tests + +**Files:** + +- Modify: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts:65-105` +- Modify: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts:244-252` +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts:835-1130` + +- [ ] **Step 1: Extend the test window and reset state** + +Add an optional `__tcfapi` function to `PrebidTestWindow`. Delete it in the shared `beforeEach`, and reset `mockGetConfig` so tests do not leak effective configuration. + +- [ ] **Step 2: Write the activation-condition tests** + +Add tests that install the shim and assert the original `mockSetConfig` receives: + +```ts +{ + consentManagement: { + gdpr: { cmpApi: 'iab' }, + }, +} +``` + +only when `managedUserIds` is non-empty, `window.__tcfapi` is callable, and effective `consentManagement` has no own `gdpr` property. Assert this call precedes the managed `userSync.userIds` call and `processQueue()`. + +- [ ] **Step 3: Write preservation and degraded-behavior tests** + +Cover: + +- no managed IDs; +- missing and non-callable `__tcfapi`; +- sibling `gpp` configuration preserved; +- effective own `gdpr` object, `null`, and `false` preserved without an automatic GDPR call; +- root `null`, `false`, strings, arrays, and throwing effective consent state log a diagnostic and are not replaced; +- queued and late publisher `setConfig`/`mergeConfig` consent fields pass through unchanged; +- automatic configuration is applied only once. + +- [ ] **Step 4: Run the focused unit tests and verify RED** + +Run: + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH \ + npx vitest run test/integrations/prebid/index.test.ts +``` + +Expected: the new activation assertion fails because no automatic `consentManagement.gdpr` call exists. + +- [ ] **Step 5: Remove the masking publisher consent setup from the artifact test** + +Change the primary denied-consent harness cases in +`test/prebid-consent-enforcement.test.mjs` so they do not call publisher-side +`pbjs.setConfig({ consentManagement: ... })`. Keep only the +`userSync.auctionDelay` setup required to resolve IDs during one auction. Add an +optional publisher consent configuration for later preservation coverage. + +- [ ] **Step 6: Run the generated-artifact test and verify RED** + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH \ + npx vitest run test/prebid-consent-enforcement.test.mjs +``` + +Expected: the denied-consent case fails because IdentityLink makes its envelope +request or writes storage when Prebid consent management is not activated. + +- [ ] **Step 7: Commit the failing tests** + +```bash +git add crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts \ + crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs +git commit -m "Test managed User ID consent activation" +``` + +### Task 2: Implement the minimum non-clobbering TCF setup + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/index.ts:130-210` +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/index.ts:1187-1241` +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + +- [ ] **Step 1: Add the focused helper** + +Add a private helper receiving the original `setConfig`, `getConfig`, and managed entries. It must: + +1. return unless managed entries exist and `typeof window.__tcfapi === 'function'`; +2. read `getConfig('consentManagement')` inside `try/catch`; +3. preserve any record with an own `gdpr` property; +4. merge record-valued sibling settings with `gdpr: { cmpApi: 'iab' }`; +5. treat every defined non-record value, including `null`, or a thrown accessor as publisher-owned/unsafe, log once, and return; +6. call the original Prebid `setConfig` exactly once, without adding timeout or `defaultGdprScope`. + +- [ ] **Step 2: Invoke it before managed ID seeding** + +Call the helper after capturing the original config APIs and before the first managed `userSync.userIds` update. Do not add a new core/TOML field and do not alter pages without managed IDs. + +- [ ] **Step 3: Run the focused unit tests and verify GREEN** + +Run the Task 1 command. Expected: all tests in `index.test.ts` pass. + +- [ ] **Step 4: Run formatting and type-aware JS tests** + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH npm run format +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH npx vitest run test/integrations/prebid/index.test.ts +``` + +- [ ] **Step 5: Commit the implementation** + +```bash +git add crates/trusted-server-js/lib/src/integrations/prebid/index.ts \ + crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +git commit -m "Activate TCF for managed User IDs" +``` + +### Task 3: Complete generated-artifact enforcement coverage + +**Files:** + +- Modify: `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs:115-205` + +- [ ] **Step 1: Add artifact-level preservation coverage** + +Use the harness option introduced in Task 1 and prove an existing custom GDPR +object remains effective after the shim loads. + +- [ ] **Step 2: Run the generated-artifact suite and verify GREEN** + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH \ + npx vitest run test/prebid-consent-enforcement.test.mjs +``` + +Expected: denied Purpose 1 and denied vendor 97 make no envelope request and write no LiveRamp storage; granted consent still makes one request and writes storage. + +- [ ] **Step 3: Commit the completed artifact regression test** + +```bash +git add crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs +git commit -m "Prove managed-only TCF enforcement" +``` + +### Task 4: Align documentation and PR handoff text + +**Files:** + +- Modify: `docs/guide/integrations/prebid.md:130-145` +- Modify: `docs/guide/integrations/prebid.md:540-565` +- Modify: `docs/guide/integrations/prebid.md:620-635` +- Modify: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md:9` +- Modify: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md:151-205` +- Modify: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md:620-720` + +- [ ] **Step 1: Document exact browser consent ownership** + +State that callable `__tcfapi` plus managed IDs activates only `gdpr.cmpApi = "iab"`, existing publisher GDPR configuration wins, and pages without TCF remain unchanged. + +- [ ] **Step 2: Update implementation and validation status** + +Mark the registry-driven bundle guard and managed consent hardening implemented. Record only sanitized live evidence: anonymous/unresolved browser returned 204/no EID; resolvable test identity returned 200, stored an envelope, and exposed one `liveramp.com` EID. Do not record Placement IDs, cookies, or envelope values. Do not claim the unperformed PBS/EC follow-on checks are complete. + +Keep the full live-validation acceptance criterion explicitly pending. List the +remaining external checks: denied-consent behavior on an approved live origin, +unapproved-origin degradation, controlled PBS `user.ext.eids` forwarding, and +later EC/KV ingestion. These require publisher/LiveRamp test conditions and are +not replaced by automated artifact tests. + +- [ ] **Step 3: Prepare corrected PR description text** + +Prepare a concise handoff in the final response replacing vendor-specific core wording, removing the unrelated credential-blocked status, recording completed browser validation, and describing ATS server-side work as deferred pending team confirmation. Do not mutate GitHub. + +- [ ] **Step 4: Format documentation** + +```bash +cd docs +npm run format:write +npm run format +``` + +- [ ] **Step 5: Commit documentation** + +```bash +git add docs/guide/integrations/prebid.md \ + docs/superpowers/specs/2026-08-21-liveramp-integration-design.md +git commit -m "Align LiveRamp consent and validation status" +``` + +### Task 5: Full verification and final review + +**Files:** + +- Review: all changes from the pre-plan HEAD through the final HEAD + +- [ ] **Step 1: Run the full JavaScript suite with the pinned Node version** + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH npx vitest run +``` + +- [ ] **Step 2: Run repository formatting checks** + +```bash +cargo fmt --all -- --check +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH npm run format +cd ../../.. && cd docs +npm run format +``` + +- [ ] **Step 3: Run the repository test matrix** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +``` + +- [ ] **Step 4: Run the repository lint matrix** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy-cli +cargo clippy-codegen +``` + +- [ ] **Step 5: Inspect repository and PR diff hygiene** + +```bash +git diff --check origin/main...HEAD +git status --short +git diff --stat origin/main...HEAD +``` + +Confirm `fastly.toml` remains the user's uncommitted local file and no Placement ID, cookie, or envelope value entered the committed diff. + +- [ ] **Step 6: Request final code review** + +Review the complete diff for correctness, privacy regressions, scope, stale documentation, and test gaps. Fix any blocking finding test-first and rerun the relevant verification. + +- [ ] **Step 7: Report readiness without pushing** + +Summarize commits, verification evidence, remaining external steps, and corrected PR-description text. Do not push or mark the PR ready. + +### Task 6: Retire automatic consent ownership safely + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` +- Modify: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` +- Modify: `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs` +- Modify: `docs/guide/integrations/prebid.md` +- Modify: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + +- [ ] **Step 1: Reproduce the stale-listener failure in the real bundle** + +Start with shim-owned IAB consent, apply late publisher static denial, emit a +later granting CMP event, and verify the test fails because the original CMP +listener remains registered. + +- [ ] **Step 2: Add focused ownership-transfer tests** + +Require cleanup before publisher `setConfig`, cleanup exactly once before +publisher `mergeConfig`, restoration of the normal enabled default for a merged +object-valued GDPR config, and safe degradation for throwing effective consent +accessors. + +- [ ] **Step 3: Implement one-time collector retirement** + +Track successful automatic activation. When a later publisher call claims GDPR +ownership, send `gdpr.enabled = false` through the original Prebid `setConfig` +before forwarding the publisher call. Preserve sibling consent state, avoid +leaking the temporary disabled flag through `mergeConfig`, and never reactivate +the automatic collector. Guard the automatically registered callback so a +delayed first CMP response cannot bypass transfer before a listener ID exists; +prepare merge normalization before cleanup, and skip replacement cleanup when +unknown sibling state or the publisher merge cannot be inspected safely. + +- [ ] **Step 4: Verify focused and full JavaScript suites** + +Run the focused shim and generated-artifact suites, formatting, and the full +Vitest suite with pinned Node 24.12.0. Then repeat diff hygiene and final review. diff --git a/docs/superpowers/plans/2026-09-10-pbs-stored-request-intent.md b/docs/superpowers/plans/2026-09-10-pbs-stored-request-intent.md new file mode 100644 index 000000000..27245615b --- /dev/null +++ b/docs/superpowers/plans/2026-09-10-pbs-stored-request-intent.md @@ -0,0 +1,264 @@ +# PBS stored-request intent implementation plan + +Status: Implemented and verified. All applicable Task 6 gates passed; independent review found no correctness issues. Deployment has not been performed. + +Issue: [IABTechLab/trusted-server#1086](https://github.com/IABTechLab/trusted-server/issues/1086) + +Goal: Let TSJS invoke non-PBS all-eligible providers without inventing PBS stored demand. A slot without usable inline PBS params or permitted stored demand must not enter the outbound PBS request or invalidate valid sibling impressions. + +## Baseline and scope + +This plan follows the configuration-driven provider implementation merged in [#1016](https://github.com/IABTechLab/trusted-server/pull/1016). Source review used `origin/main` at `d704d0ab0c5916d429b80a5707c0f2d74b99cab1`. + +The parent fast-forwarded `fix/pbs-stored-requests` to `d704d0ab` before implementation and preserved this plan as an untracked file. Initial `git status --short` showed only this plan. No unrelated changes were present. + +Scope correction approved during implementation: the baseline compiler already rejects PBS `all_eligible` in `auction/plan.rs`. Do not enable it or mutate compiled plans for tests. Runtime PBS tests use supported explicit routing and server-owned trusted routes. The existing `compiler_rejects_all_eligible_for_prebid_server_only` regression covers that boundary; APS/standard all-eligible behavior remains supported. + +Keep the change limited to intent admission, provider routing, PBS request construction, TSJS envelope generation and refresh state, and their tests and documentation. No dependency, adapter, provider-configuration, or endpoint changes are expected. + +Explicit stored IDs, browser-selected provider IDs, revised stored-demand fanout, and removal of legacy inference are out of scope. + +## Contract + +The field lives inside `trustedServer.params`, beside `bidderParams` and `zone`: + +```json +{ + "bidder": "trustedServer", + "params": { + "bidderParams": {}, + "storedRequest": false + } +} +``` + +- `false` disables stored-request fallback for every PBS provider for this slot. It does not disable valid inline demand or eligible APS/standard providers. +- `true` explicitly permits stored-request fallback. Usable inline params still take precedence within each PBS provider. If overrides leave no usable inline params, the provider may use the slot code as its stored impression ID. +- Omission retains legacy behavior for existing direct `/auction` callers and server-generated opportunities. Preserve both empty-envelope stored inference and the existing fallback when routed empty bidder objects remain unusable after overrides. +- Explicit `null`, strings, numbers, arrays, and objects are invalid values. Do not deserialize through plain `Option` or use `as_bool().unwrap_or(...)`, which would conflate invalid values with omission. +- Invalid intent rejects the complete envelope atomically, including its inline params and zone. Increment the existing malformed-envelope diagnostic. Do not reinterpret rejection as missing demand. Valid direct sibling bidder entries and non-PBS all-eligible participation retain their existing behavior; this does not introduce whole-request HTTP rejection. +- Browser input still cannot name provider routes. Intent grants no new provider-selection authority. Explicit and legacy stored demand retain existing PBS fanout for this fix. + +Internally, introduce a narrow `StoredRequestIntent` enum with `Disabled`, `Explicit`, and `Legacy` states. Retain the distinction through provider-local request construction. Legacy inference depends on the original admitted shape, not merely the map left after route filtering. If needed, carry that admission fact as a payload of `Legacy`, rather than adding independently mutable flags that must agree with the enum. Existing routed candidate params remain necessary for legacy post-override fallback. + +A malformed envelope must normalize to no stored permission, not a default `Legacy` state. Keep parsing and validation in `auction/routing.rs` rather than repeatedly inspecting raw JSON in providers. + +## Demand flow + +```mermaid +flowchart TD + A[Validate trustedServer envelope] --> B{Valid envelope?} + B -->|No| C[Reject envelope demand and record diagnostic] + B -->|Yes| D[Normalize intent and bidder params] + C --> E[Route valid sibling demand independently] + D --> E + E --> F{Provider profile} + F -->|Non-PBS| G[Preserve explicit and all-eligible routing] + F -->|PBS| H[Admit inline candidates or permitted stored demand] + H --> I[Apply provider-local bidder overrides] + I --> J{Usable inline params remain?} + J -->|Yes| K[Emit inline PBS impression] + J -->|No| L{Stored fallback permitted?} + L -->|Yes| M[Emit stored impression using slot code] + L -->|No| N[Omit impression] + K --> O[Check final impression count before signing and transport] + M --> O + N --> O +``` + +## Target files + +Paths below are relative to the repository root. + +- `crates/trusted-server-core/src/auction/routing.rs`: intent normalization, legacy admission facts, provider-local intent, PBS admission rules, and routing tests. +- `crates/trusted-server-core/src/auction/openrtb.rs`: post-override demand decisions, impression omission, and final empty-request handling. +- `crates/trusted-server-core/src/auction/openrtb/tests.rs`: serialized request and override regressions. +- `crates/trusted-server-core/src/auction/orchestrator.rs`: transport-level mixed-slot and no-request regressions using existing test support. +- `crates/trusted-server-core/src/auction/formats.rs`: wire-to-auction regression coverage and accurate request documentation. Avoid adding a second envelope parser. +- `crates/trusted-server-core/src/creative_opportunities.rs`: preserve and test server-generated stored demand and zone behavior; change production generation only if required by the new internal model. +- `crates/trusted-server-js/lib/src/integrations/prebid/index.ts`: synthetic envelope defaults, publisher intent preservation, immutable snapshots, and refresh reconstruction. +- `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts`: initial, repeated, and refresh auction payload tests. +- `crates/trusted-server-js/lib/test/core/auction.test.ts`: prove shared serialization retains the boolean without changing `core/auction.ts` unless a test exposes a need. +- `crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs`: built-adapter payload expectations. +- `docs/guide/api-reference.md`, `docs/guide/integrations/prebid.md`, and `docs/guide/auction-orchestration.md`: intent semantics, examples, and deployment compatibility. +- `CHANGELOG.md`: concise unreleased fix entry. + +## Task 1: Establish the baseline and failing evidence + +- [x] Complete the approved branch update and inspect `git status`. +- [x] Run the current routing, PBS builder, and Prebid JS test groups before changing production code. Record environmental failures separately from product failures. +- [x] Add focused regressions in small steps, running each before its production fix. Do not bulk-update snapshots to conceal changed demand. +- [x] First prove that TSJS should serialize `storedRequest: false` for a generated all-eligible envelope but currently omits it. +- [x] Prove that an explicit `true` empty envelope should retain PBS demand and a `false` envelope with valid inline params should retain those params. The current unknown-field validator rejects both, so these distinguish missing support from a working fix. +- [x] Preserve existing omitted-field tests as compatibility evidence. A `false` empty-envelope skip test alone is insufficient: the old validator already rejects the unknown field and can make that assertion pass for the wrong reason. + +Focused commands: + +```bash +cargo test-fastly auction::routing::tests +cargo test-fastly auction::openrtb::tests +( + cd crates/trusted-server-js/lib + npx vitest run test/integrations/prebid/index.test.ts test/core/auction.test.ts +) +``` + +Acceptance: saved command output identifies at least one relevant failing Rust assertion and one failing JS serialization assertion before their production changes. + +## Task 2: Normalize intent and enforce PBS routing + +- [x] Add strict optional-field parsing to the existing envelope allowlist and atomic validator. Validate intent before returning from any missing, null, or empty `bidderParams` branch. +- [x] Introduce the internal intent type and carry the required legacy admission facts into `ProviderSlotInput`. +- [x] For PBS, route a slot only when it has assigned inline candidates or permitted stored demand. `AllEligible` alone cannot override this rule, including for malformed envelopes. +- [x] Keep non-PBS routing unchanged. Preserve existing server-owned route handling without exposing it through the browser envelope; a trusted route alone must not cause a demandless PBS wire impression. +- [x] Retain empty object candidates for configured envelope bidders until provider overrides run. Do not treat an empty object as usable inline demand at the final wire boundary. +- [x] Do not turn unconfigured bidders removed by plan filtering into new stored demand when intent is disabled. +- [x] Test two explicit PBS providers plus APS, retaining compiler rejection of all-eligible PBS configurations, empty envelopes, empty bidder objects, unconfigured bidders, and mixed inline ownership. +- [x] Test every malformed intent type, a malformed envelope containing valid-looking inline params, and preservation of independent valid direct demand. Assert diagnostics and absence of stored fallback. +- [x] Retain omitted-field direct `/auction` conversion coverage and the server-generated creative-opportunity stored-request/zone regression. + +Run the routing and format test groups after each focused change: + +```bash +cargo test-fastly auction::routing::tests +cargo test-fastly auction::formats::tests +cargo test-fastly creative_opportunity_canonical_slot_feeds_shared_stored_router_with_zone +``` + +Acceptance: `false` suppresses PBS providers without candidate demand while APS remains eligible. Explicit and legacy demand retain their documented routes. + +## Task 3: Enforce intent after overrides and prove outbound requests + +The current `apply_prebid` fallback uses `has_trusted_stored_request() || !slot.bidder_params().is_empty()`. Updating the router alone leaves this second source of stored inference intact. + +- [x] First add and run a builder regression for `false` with a configured bidder whose params remain `{}` after overrides. At this stage it must expose the remaining fallback bug. +- [x] Apply overrides before deciding whether inline params are usable. Preserve the positive case where an override fills an empty object. +- [x] Replace implicit stored inference with the provider-local intent policy. Emit inline demand first; otherwise emit stored demand only when allowed; otherwise omit the impression. +- [x] Preserve slot/impression pairing while filtering. Do not remove impressions and then zip the shortened list with the original slots, which could attach another slot's params or stored ID. +- [x] Check for `NoImpressions` after profile augmentation, before final signing and transport. The existing pre-augmentation check is insufficient once PBS can drop impressions. +- [x] Keep omitted-intent empty-candidate fallback covered. Add `true` with inline params, `true` with unusable post-override params, and `false` with override-populated params. +- [x] Using the existing deterministic executor/orchestrator test support, capture an actual serialized PBS transport request containing a valid inline slot beside a synthetic no-PBS slot. Assert only the valid impression is sent, no stored reference appears for the omitted slot, and the valid bid survives. +- [x] Repeat the mixed routing case across two PBS instances and APS. Assert no-demand PBS providers make zero requests and APS still runs. +- [x] Test an explicit PBS request whose last candidate disappears after overrides. All-eligible PBS is rejected by the compiler and is not a reachable runtime case. Assert zero transport calls, not a serialized empty `imp` array or a debug assertion failure. + +```bash +cargo test-fastly auction::openrtb::tests +cargo test-fastly auction::orchestrator::tests +``` + +Acceptance: evidence observes both the final wire payload and the absence of transport for empty requests. No live PBS service is required to prove the offending impression is absent. Do not claim this prevents HTTP 400 responses caused by unrelated invalid bidder params. + +## Task 4: Emit and preserve browser intent + +- [x] Add `storedRequest: false` to every newly generated TSJS envelope without publisher-supplied stored intent, including envelopes currently carrying inline candidates. Client-side filtering cannot predict the final server-local demand after routing and overrides. +- [x] Preserve a publisher-authored existing envelope's `true`, `false`, or omitted state during ordinary `requestBids` reuse. Do not rewrite legacy publisher intent merely because its bidder map is empty. +- [x] Extend the request-scoped snapshot with stored intent and preserve field presence. Do not coerce invalid authored values into omission or `false`; leave server validation authoritative. +- [x] Recover intent during synthetic refresh using the same live-ad-unit authority and snapshot fallback as bidder params. Distinguish a publisher-authored omitted legacy envelope from no recovered envelope, which needs the synthetic `false` default. +- [x] Keep intent attached to the slot when code aliases such as container IDs are used. Reuse existing refresh lookup rules rather than adding another matching policy. +- [x] Cover initial empty envelopes, repeated calls on mutated ad units, synthetic refresh without recovered demand, publisher `true` and `false`, legacy omission, live data replacing a stale snapshot, snapshot-only recovery, and client-side bidder preservation. +- [x] Assert actual JSON request bodies through `buildRequests`/`buildAdRequest`, not just intermediate objects. Update the built-artifact expectation as well. + +```bash +( + cd crates/trusted-server-js/lib + npx vitest run test/integrations/prebid/index.test.ts test/core/auction.test.ts + npx vitest run test/prebid-artifact-integration.test.mjs + node build-all.mjs +) +``` + +Acceptance: TSJS-generated no-stored-demand envelopes remain explicitly disabled on initial and refresh requests, while authored explicit and legacy stored demand are preserved. + +## Task 5: Document and sequence deployment + +- [x] Document field location, all three valid wire states, invalid `null`, inline-first behavior, post-override fallback, and unchanged server-owned routing authority. +- [x] Show separate examples for synthetic no-PBS demand and intentional stored demand using `example.com` data. +- [x] Update comments that equate every empty bidder map with a stored request. +- [x] Separate server support from TSJS emission in the delivery sequence. Deploy and verify server support on all serving instances before distributing the new JS bundle. If publishing automatically couples the artifacts, resolve the release mechanism before rollout rather than assuming staging is possible. +- [x] Document that the pre-fix #1016 router rejects unknown envelope fields. Early JS deployment can discard valid inline envelope demand, not just retain the original stored-lookup bug. +- [x] Document rollback ordering: after new JS has reached browsers or caches, do not roll back to a server that rejects `storedRequest`. Keep compatible server admission until old clients can safely be served again. +- [x] Keep omission supported in this change. Record removal criteria for a separate migration: inventory direct callers and server-generated paths, migrate them to explicit intent, account for cached clients, and approve a versioned contract change. Do not add an arbitrary expiry or a new telemetry system here. + +## Task 6: Full verification and handoff + +Shared core changes affect every adapter. Run the repository's applicable gates before PR handoff: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +( + cd crates/trusted-server-js/lib + npx vitest run + npm run lint + npm run format + node build-all.mjs +) +( + cd docs + npm run format + npm run build +) +git diff --check +git status --short +``` + +- [x] Inspect the final diff against this plan. Preserve unrelated edits and avoid broad formatting churn. +- [x] Report changed files, failing-before and passing-after commands, full gate results, outbound payload/zero-I/O evidence, and any unavailable runtime checks. +- [x] Confirm no browser provider selector, explicit stored ID feature, legacy-removal change, or unrelated adapter refactor entered the patch. +- [x] Keep remaining risks explicit: legacy omitted callers can still infer stored demand; intentional stored requests still fan out and may use nonexistent slot-code IDs; the release depends on server-first deployment. + +### Focused implementation evidence + +Logs are saved under `/tmp/pbs-intent/`. + +- Baselines passed: routing, OpenRTB builder, and the Prebid/shared JS test groups. +- `red-routing-assertion.log` records the old validator rejecting explicit stored intent, with only APS routed rather than APS plus both PBS instances. +- `red-js.log` records serialized generated intent as `[undefined, undefined]` instead of `[false, false]`. +- `red-post-override.log` records the remaining fallback bug after the router fix: disabled empty candidates still produced a stored impression instead of `NoImpressions`. +- Routing, builder, format conversion, creative-opportunity stored/zone compatibility, compiler boundary, orchestrator, and JS serialization/refresh groups passed after the changes. The transport test captures one or two valid PBS wire requests beside APS and observes zero PBS calls when all PBS candidates are unusable. +- Built-artifact integration and `node build-all.mjs` passed. Source changes did not require a shared serializer or creative-opportunity generator change. +- Early local failures were test setup issues, not environment blockers: a JS helper was scoped to a sibling suite; the first Rust rerun briefly hid a still-used method behind `cfg(test)`; a planned PBS AllEligible fixture hit the pre-existing compiler rejection. These were corrected without changing dependencies or configuration. The meaningful red assertions above were then captured independently. + +Final focused results: 16 routing, 29 OpenRTB, 33 formats, 77 orchestrator, one creative-opportunity compatibility, and one PBS configuration-boundary test passed. The combined JS source/shared/built-artifact group passed 191 tests. + +### Full verification results + +All Task 6 commands above were executed. Logs and exact command metadata are under `/tmp/pbs-task6-gates/`. + +- `cargo test-fastly`: 2,886 passed, 10 explicitly ignored, including doctests; executed through Viceroy. +- `cargo test-axum`: 41 passed. `cargo test-cloudflare`: 44 passed. `cargo test-spin`: 86 passed. `./scripts/test-cli.sh`: 87 passed. +- `npx vitest run`: 923 tests passed across 45 files, with no type errors. JS lint, formatting, and `node build-all.mjs` passed. +- Rust formatting, documentation formatting/build, and `git diff --check` passed. +- All six clippy aliases initially failed on the new parser's redundant closure. The parent replaced it with `serde_json::Map::is_empty` and corrected one new test URI to `publisher.example.com`. All six clippy aliases, Rust formatting, the 16 routing tests, the 29 OpenRTB tests, and diff checks passed afterward. The full test suites ran before these two mechanical edits and were not repeated in full. +- Independent read-only correctness review inspected the diff and reran focused Rust and JS tests. It found no correctness issues. Parent inspection confirmed the result and the two subsequent mechanical edits. +- No live PBS or deployed-adapter check was run. Transport tests establish that the offending impressions are absent, not that all possible PBS HTTP 400 responses are prevented. +- Documentation dependency installation used the existing lockfile and reported 17 vulnerabilities. No dependency changes were made. Documentation build warnings about `vcl` highlighting and bundle size were non-blocking. + +Actual changed files: + +- `crates/trusted-server-core/src/auction/routing.rs` +- `crates/trusted-server-core/src/auction/openrtb.rs` +- `crates/trusted-server-core/src/auction/openrtb/tests.rs` +- `crates/trusted-server-core/src/auction/orchestrator.rs` +- `crates/trusted-server-core/src/auction/formats.rs` +- `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` +- `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` +- `crates/trusted-server-js/lib/test/core/auction.test.ts` +- `crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs` +- `docs/guide/api-reference.md` +- `docs/guide/integrations/prebid.md` +- `docs/guide/auction-orchestration.md` +- `CHANGELOG.md` +- `docs/superpowers/plans/2026-09-10-pbs-stored-request-intent.md` + +Remaining risks are the preserved legacy inference and stored-demand fanout, unavailable slot-code stored IDs, and server-first release ordering. The Rust artifact embeds JS, so deployment requires a server-support-only build retaining old JS before the full build. Verification did not change dependencies or perform deployment. diff --git a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md new file mode 100644 index 000000000..98af3e176 --- /dev/null +++ b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md @@ -0,0 +1,842 @@ +# Server-Side Ad Template CLI Design + +**Date:** 2026-06-26 +**Status:** Draft design +**Scope:** Static and browser-backed CLI diagnostics for server-side ad templates + +## 1. Goal + +Add Trusted Server CLI support for server-side ad-template onboarding and +verification without resurrecting the stale standalone `ts-config` design. + +The CLI must answer two operator questions: + +1. Given an effective `trusted-server.toml`, which configured ad-template slots + match this path? +2. Given one or more live publisher URLs, are the configured slots for the + final navigated paths actually present on the page according to DOM, GPT, + and provider evidence, and do any runtime gates explain why Trusted Server + would not inject or auction for that page? + +The command surface is split by whether the command is local-config-only or +browser-backed: + +```bash +ts config ad-templates lint +ts config ad-templates match +ts config ad-templates check +ts config ad-templates explain + +ts audit ad-templates verify ... +``` + +Static commands live under `ts config` because they only load local effective +app config. Browser-backed verification lives under `ts audit` because it loads +public publisher pages in Chrome/Chromium and observes live page behavior. + +## 2. Context + +This design replaces the stale PR #724 direction. + +PR #724 designed a standalone `ts-config` binary around a +`creative-opportunities.toml` file. That is no longer the project shape: + +- Trusted Server configuration now flows through the unified `ts` CLI from PR + #799. +- Server-side ad-template slots live under `[creative_opportunities]` / + `[[creative_opportunities.slot]]` in `trusted-server.toml`. +- Effective config can include EdgeZero app-config environment overlays unless + `--no-env` is passed. +- Operator-owned `trusted-server.toml` is ignored; the repository tracks + `trusted-server.example.toml`. + +PR #799 is the CLI base. It owns the `ts` binary, EdgeZero lifecycle delegates, +and typed app-config validation/push/diff behavior. + +PR #800 is the audit dependency. It adds the generic browser-backed +`ts audit ` collector using local Chrome/Chromium. At the time this spec +was written, PR #800 was stale relative to the latest #799 head, so this work +depends on the #800 audit collector after it is rebased onto the latest #799 +typed blob-config model. + +## 3. Non-Goals + +- Do not add a standalone `ts-config` binary. +- Do not reintroduce `creative-opportunities.toml`. +- Do not implement browser-backed generation in Phase 1. +- Do not mutate `trusted-server.toml` from `verify`. +- Do not probe PBS, GAM, or APS management APIs. +- Do not require EdgeZero platform adapters for local static diagnostics. +- Do not make `ts audit ad-templates verify` push, provision, deploy, or update + platform resources. +- Do not rely on real GPT or APS network calls in tests. + +Browser-backed generation is a later phase: + +```bash +ts audit ad-templates generate ... +``` + +That phase needs separate rules for slot ID derivation, page-pattern inference, +multi-URL merging, TOML ordering, and whether the command emits a patch, a draft +file, or full config blocks. + +## 4. Command Surface + +### 4.1 Shared Config Flags + +All `ts config ad-templates ...` commands and +`ts audit ad-templates verify` accept the same local app-config flags: + +```bash +--app-config +--manifest +--no-env +``` + +Defaults match PR #799: + +| Option | Default | +| -------------- | ------------------------------------------------ | +| `--app-config` | `.toml`, resolved from `edgezero.toml` | +| `--manifest` | `edgezero.toml` | +| `--no-env` | `false`; app-config env overlay is applied | + +If an explicit `--app-config` path is supplied and missing, the command reports +that path as the error. It must not silently fall back to an environment or +manifest-derived path. + +### 4.2 Static Config Diagnostics + +```bash +ts config ad-templates lint [--app-config ] [--manifest ] [--no-env] +``` + +Reports whether `[creative_opportunities]` is configured, how many slots exist, +GAM network ID, auction timeout, auction enablement, configured auction +providers, and whether current EdgeZero routing will fall back to the legacy +path when configured slots are present. + +```bash +ts config ad-templates match [--details] ... +``` + +Normalizes a path or full URL to a path and reports the slots matched by the +runtime `creative_opportunities::match_slots` logic. `--details` includes slot +div ID, GAM unit path, page patterns, formats, and configured providers. + +```bash +ts config ad-templates check \ + (--expected-slot ... | --expect-no-slots) \ + [--allow-extra-slots] ... +``` + +CI-friendly assertion wrapper around the same matching logic. + +```bash +ts config ad-templates explain \ + [--method GET] \ + [--non-navigation] \ + [--prefetch] \ + [--bot] \ + [--consent-denied] \ + [--edgezero-enabled] ... +``` + +Explains the major runtime gates that decide whether the server-side ad stack +would run for a page request. This is a local model, not a live request replay. + +### 4.3 Browser-Backed Verification + +```bash +ts audit ad-templates verify ... \ + [--app-config ] \ + [--manifest ] \ + [--no-env] \ + [--strict] \ + [--json] \ + [--scroll] +``` + +Behavior: + +- Accept one or more `http` or `https` URLs. +- Reject all other schemes before launching a browser. +- Load the effective Trusted Server app config. +- For each URL, navigate first, collect the final URL, normalize the final URL + to a path, and call `creative_opportunities::match_slots`. +- Preserve the requested URL/path separately from the final URL/path. +- Emit a redirect warning when the final path differs from the requested path. +- Expect only the slots matched for the final URL path to be present on that + live page. +- Report live DOM/GPT/APS ad-slot evidence that does not correspond to a + matched configured slot as structured extra evidence. +- Launch Chrome/Chromium through the audit collector from the rebased #800 work. +- Inject a read-only ad-template collector before publisher scripts run. +- Compare configured matched slots against DOM, GPT, and APS evidence. +- Report runtime ad-stack gate evidence separately from placement evidence. +- Print human output by default. +- Emit stable machine-readable output with `--json`. +- Exit `0` by default for missing or partial live evidence; this is an + auditor-assist mode. +- Exit non-zero under `--strict` when a matched configured slot is missing or + only partially confirmed. + +`--scroll` performs a deterministic scroll pass after initial load and settle. +It is opt-in because it is slower and can trigger additional page behavior. +Slots first observed during scroll count as confirmed when the GPT evidence is +otherwise sufficient. + +## 5. Confirmation Model + +The verifier compares configured expected slots to live page evidence. + +It must keep three concepts separate: + +1. **Static slot matching:** which configured slots match a URL path according + to `creative_opportunities::match_slots`. +2. **Runtime ad-stack eligibility:** whether Trusted Server would run its + server-side ad stack for the audited navigation. This mirrors + `should_run_server_side_ad_stack` for the initial publisher request and the + `/__ts/page-bids` kill-switch/consent behavior for SPA route updates. +3. **Live placement evidence:** what the browser actually observes on the + rendered page through DOM, GPT, and APS evidence. + +`verify` is primarily a live placement verifier. `--strict` fails when matched +configured slots for an eligible page are missing or partial. Runtime gates are +reported so operators can distinguish "the slot is not on the page" from "the +current request/config would intentionally suppress Trusted Server ad-template +injection or page-bids slot output". + +### 5.1 Expected Slots + +For each input URL: + +1. Navigate the browser to the requested URL. +2. Record `requested_url`, `requested_path`, `final_url`, and `final_path`. +3. Match configured slots through the core runtime matcher using `final_path`. +4. Build an expected-slot record for each matched slot: + - slot ID; + - resolved div ID; + - resolved GAM unit path; + - configured formats; + - configured providers; + - matching page patterns. + +Only these expected slots are verified for that page. For example, slots whose +only pattern is `/` are expected for the homepage path, not for `/news/story`. + +When a navigation redirects, `verify` uses the final path for expected slots and +reports the requested path in output. This matches runtime behavior: Trusted +Server evaluates the actual publisher request path it handles, not the URL the +operator typed before redirects. + +### 5.2 Runtime Gate Evidence + +For each page result, `verify` reports a local runtime-gate model: + +| Gate | Source | +| ------------------------ | -------------------------------------------------------------------------------------------------------- | +| `method_get` | Browser navigation request; expected to pass for normal `verify`. | +| `navigation` | Browser navigation request; expected to pass for normal `verify`. | +| `not_prefetch` | Browser request headers; expected to pass unless the collector is extended with prefetch simulation. | +| `not_bot` | Browser User-Agent checked against the runtime bot fragments. | +| `matched_slots` | Final-path slot matching. | +| `auction_enabled` | Effective `[auction].enabled` / orchestrator enablement from app config. | +| `consent_allows_auction` | `unknown` unless the collector can prove a consent-allowed or consent-denied state for the live request. | + +`runtime_ad_stack_expected` is a three-state value: `yes`, `no`, or `unknown`. +Known blocking gates produce page warnings and set +`runtime_ad_stack_expected = "no"`. Unknown gates set +`runtime_ad_stack_expected = "unknown"` but do not by themselves fail +`--strict`. + +If `runtime_ad_stack_expected = "no"` because of a known config/request gate +such as `[auction].enabled = false`, strict mode does not fail missing GPT/APS +evidence for that page. The page result is reported as skipped for runtime +verification while still showing the static expected slots and any live +placement evidence that was observed. + +If `runtime_ad_stack_expected = "yes"` or `"unknown"`, strict mode applies the +normal missing/partial placement rules from §5.6. + +For SPA routes, `/__ts/page-bids` returns no slots when the ad-stack kill switch +or consent gate blocks the stack. Browser verification should report observed +page-bids responses when available, but it must not require real partner bids in +tests. + +Live ad-slot evidence that does not map to a matched expected slot is reported +as structured extra evidence. Extra evidence can identify publisher-owned slots +that have not yet moved into server-side ad templates, slots whose +`page_patterns` are too narrow, or slots that should stay outside Trusted +Server. It does not make `--strict` fail in Phase 1. + +### 5.3 DOM Slot Resolution + +The verifier must mirror the runtime GPT bootstrap's slot-root resolution: + +1. Try `document.getElementById(slot.div_id)`. +2. If absent, find the first element with an ID that starts with `slot.div_id`. +3. Ignore elements whose ID ends with `-container`. + +This is required because `div_id` may intentionally be a stable prefix for +framework-generated IDs, for example `ad-header-0-`. + +### 5.4 GPT Evidence + +A slot is confirmed by GPT evidence when the live page exposes a GPT slot whose: + +- ad unit path equals the configured resolved GAM unit path; +- slot element ID equals the resolved DOM element ID or an existing + `${resolved_dom_id}-container` element used by Trusted Server when defining + its own slot; +- configured sizes are compatible with the observed GPT sizes. + +The collector should observe both direct `googletag.defineSlot` calls and +post-load `googletag.pubads().getSlots()` state. + +Size compatibility is defined for Phase 1 as follows: + +- Normalize configured sizes from `CreativeOpportunityFormat` values where + `media_type = "banner"` into `(width, height)` pairs. +- Normalize observed GPT sizes from `defineSlot` input and `getSizes()` output: + - `[300, 250]` becomes one `(300, 250)` pair. + - `[[300, 250], [728, 90]]` becomes two pairs. + - non-numeric values such as `"fluid"` are ignored for numeric matching and + reported as warnings. +- A GPT slot's sizes are compatible when the configured banner size set and the + observed numeric GPT size set have at least one pair in common. +- Extra observed GPT sizes do not block confirmation, but they are reported as + warnings so auditors can decide whether to add formats to config. +- Configured banner sizes that are not observed do not block confirmation when + at least one configured size was observed, but they are reported as warnings. +- If ad unit path and div match but no numeric size overlap exists, the slot is + `partial`, not `confirmed`. +- Configured `video` and `native` formats are not used for Phase 1 GPT size + confirmation. A matched slot with only non-banner formats is `unconfirmable` + with an unsupported-format warning and does not fail `--strict`. +- A sizeless live GPT slot is `partial` when the config declares banner sizes, + because that is observable drift and must fail `--strict`. + +### 5.5 APS Evidence + +Phase 1 does not wrap or collect `apstag.fetchBids`: APS is server-side provider +configuration and client-side calls are neither required nor authoritative for +the runtime ad-template decision. + +### 5.6 Statuses + +| Status | Meaning | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `confirmed` | GPT evidence matches the configured GAM unit path, div resolution, and compatible sizes. | +| `partial` | The page has some evidence for the configured slot, but not enough to confirm it. This includes DOM-only evidence, GPT path/div matches with incompatible sizes, GPT path/div matches for unsupported non-banner-only configured formats, and other non-confirming GPT evidence. | +| `missing` | No DOM or GPT evidence confirms the configured slot. | +| `unconfirmable` | The checker cannot evaluate the configured format with Phase 1 evidence, such as a non-banner-only slot. This is reported but does not fail strict mode. | + +In `--strict` mode: + +- `missing` fails. +- `partial` fails. +- `unconfirmable` does not fail. + +Provider issues are not statuses. They are warnings attached to the slot result. +For example, a slot can be `confirmed` and still carry a warning that configured +APS evidence was missing or ambiguous. Provider warnings do not fail `--strict` +unless a future `--strict-providers` flag is added. + +## 6. Architecture + +The architecture should keep command parsing thin and move ad-template behavior +into pure, testable modules. + +```text +crates/trusted-server-cli/src/ + app_config.rs + ad_templates/ + mod.rs + expected.rs + compare.rs + output.rs + config_ad_templates.rs + audit/ + page.rs + browser.rs + ad_templates.rs +``` + +### 6.1 `app_config.rs` + +Shared loader for effective Trusted Server app config. + +Responsibilities: + +- read `edgezero.toml` through EdgeZero manifest helpers; +- resolve the default `.toml` path; +- apply EdgeZero app-config env overlay unless `--no-env`; +- return `TrustedServerAppConfig` / `Settings`; +- report errors in the same terms as #799 config commands. + +This avoids duplicating config path and env-overlay behavior between +`ts config ad-templates ...` and `ts audit ad-templates verify`. + +The current branch already has a private loader in `config_ad_templates.rs`. +Before adding browser-backed verification, move that behavior into this shared +module and route the existing static commands through it so both command +families load the same effective config. + +### 6.2 `ad_templates::expected` + +Pure local expected-slot model. + +Responsibilities: + +- normalize path-or-URL input; +- call `creative_opportunities::match_slots`; +- convert matched slots into stable expected-slot structs; +- preserve deterministic ordering by slot order from config. + +This module must not compile glob patterns independently or duplicate matching +semantics. + +If richer pattern diagnostics are needed, add a small helper to +`trusted-server-core::creative_opportunities` and use it from both runtime and +CLI. + +### 6.3 `ad_templates::compare` + +Pure comparison between expected slots and collected browser evidence. + +Responsibilities: + +- implement DOM prefix matching rules; +- compare GPT path, div, and size evidence; +- compare APS evidence; +- collect unmatched live DOM/GPT/APS ad-slot evidence as structured + `extra_evidence`; +- assign `confirmed`, `partial`, `missing`, and provider warning details; +- decide strict failure status. + +This module should be testable without launching Chrome. + +### 6.4 `ad_templates::output` + +Human and JSON output model. + +Responsibilities: + +- serialize stable JSON output; +- keep arrays ordered by input URL, then configured slot order, then provider + name; +- render concise human summaries; +- avoid leaking page HTML, cookies, local storage, or arbitrary page data. + +### 6.5 `config_ad_templates.rs` + +Thin Clap adapter for `ts config ad-templates ...`. + +Responsibilities: + +- parse command arguments; +- call `app_config` and `ad_templates::expected`; +- delegate formatting to `ad_templates::output`; +- keep no browser-specific logic. + +### 6.6 `audit::browser` + +Shared browser utility extracted from or aligned with the rebased #800 audit +collector. + +Responsibilities: + +- locate Chrome/Chromium; +- launch an isolated profile; +- reject non-HTTP(S) URLs before navigation; +- set bounded navigation and settle timeouts; +- run optional init scripts; +- perform optional deterministic scroll; +- collect final URL, title, rendered scripts, resource entries, and optional + ad-template evidence. + +The generic `ts audit ` command from #800 should continue to work without +ad-template verification enabled. + +### 6.7 `audit::ad_templates` + +Browser-backed verifier orchestration. + +Responsibilities: + +- parse `ts audit ad-templates verify`; +- load effective config through `app_config`; +- compute expected slots for each URL; +- run the browser collector with ad-template evidence enabled; +- call `ad_templates::compare`; +- print human or JSON output; +- apply default auditor-assist exit behavior and `--strict` behavior. + +## 7. Browser Collector + +The ad-template collector is injected before page scripts run. It is read-only: +it records evidence and calls original page functions with unchanged arguments. + +The rebased #800 collector must grow a pre-navigation init-script hook before it +can satisfy this spec. The stale #800 collector only navigates, waits, and reads +post-load page state; that is insufficient for GPT/APS call evidence. + +Instrumentation requirements: + +- install the collector through the browser's "evaluate on new document" / + init-script mechanism before navigation; +- serialize only configured div prefixes and provider IDs needed for matching; +- observe pages that create `window.googletag = { cmd: [] }` after injection; +- wrap `googletag.cmd.push` callbacks without changing callback order; +- record direct `googletag.defineSlot` calls and calls executed from the GPT + command queue; +- read final `googletag.pubads().getSlots()` state after settle and after + scroll; +- observe pages that assign `window.apstag` after injection and wrap + `apstag.fetchBids` when present; +- tolerate pages that never load GPT or APS and report warnings instead of + throwing collector errors. + +Evidence to collect: + +- DOM elements with IDs relevant to configured slot div prefixes; +- calls to `googletag.defineSlot`; +- final `googletag.pubads().getSlots()` state after settle and after scroll; +- calls to `apstag.fetchBids`; +- timestamps or phases indicating whether evidence was observed during + `initial_load` or `scroll`. + +The collector must not: + +- block, rewrite, or suppress publisher scripts; +- override `navigator.webdriver`; +- capture cookies, local storage, session storage, request bodies, or arbitrary + page data; +- require real GPT/APS network calls in test fixtures. + +## 8. JSON Output Contract + +`--json` emits deterministic JSON. Shape: + +```json +{ + "ok": true, + "strict": false, + "pages": [ + { + "url": "https://www.example.com/news/story", + "final_url": "https://www.example.com/news/story", + "requested_path": "/news/story", + "path": "/news/story", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "atf", + "status": "confirmed", + "phase": "initial_load", + "configured": { + "div_id": "ad-atf-", + "gam_unit_path": "/123/news/atf", + "formats": [ + { "width": 300, "height": 250, "media_type": "banner" } + ], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-atf-0", + "gpt": { + "gam_unit_path": "/123/news/atf", + "div_id": "ad-atf-0", + "sizes": [[300, 250]] + } + }, + "warnings": [] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +Warning entries are objects with stable `code` and human-readable `message` +fields. Human output may print only the message. JSON consumers must not need to +parse warning strings. + +Extra live evidence is structured: + +```json +{ + "kind": "gpt", + "phase": "initial_load", + "dom_id": "ad-right-rail-0", + "gam_unit_path": "/123/publisher/right-rail", + "sizes": [[300, 250]], + "reason": "no_configured_slot_matched" +} +``` + +Allowed `kind` values for Phase 1 are `dom` and `gpt`. + +Strict-mode failures with page results use the same shape and set `ok` to +`false`. Example partial slot: + +```json +{ + "ok": false, + "strict": true, + "pages": [ + { + "url": "https://www.example.com/", + "final_url": "https://www.example.com/", + "requested_path": "/", + "path": "/", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "homepage-header", + "status": "partial", + "phase": "initial_load", + "configured": { + "div_id": "ad-header-0-", + "gam_unit_path": "/123/homepage/header", + "formats": [{ "width": 728, "height": 90, "media_type": "banner" }], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-header-0-_R_abc123", + "gpt": null + }, + "warnings": [ + { + "code": "dom_without_gpt", + "message": "DOM element matched, but no GPT slot evidence was observed" + } + ] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +For errors that occur before any page result can be produced, the command exits +non-zero and prints the normal CLI error. JSON error output can be added later +if the base CLI standardizes it. + +For multi-URL runs, browser/navigation failures after argument validation are +page-level failures when possible. The command continues to the remaining URLs, +sets top-level `ok` to `false`, and includes a page result: + +```json +{ + "url": "https://www.example.com/broken", + "final_url": null, + "requested_path": "/broken", + "path": null, + "error": { + "code": "navigation_failed", + "message": "failed to read main document navigation response" + }, + "slots": [], + "extra_evidence": [], + "warnings": [] +} +``` + +Invalid schemes are still rejected before browser launch for the whole command, +because they are argument errors rather than page collection results. + +## 9. Error Handling + +Static commands fail when: + +- config cannot be loaded; +- `[creative_opportunities]` is malformed; +- CLI assertions in `check` fail. + +Browser verification fails when: + +- config cannot be loaded; +- any URL is not HTTP(S); +- Chrome/Chromium cannot be found or launched; +- all navigations fail before any page result can be collected; +- at least one page-level error occurs in a multi-URL run; +- command output cannot be written; +- `--strict` is set, runtime verification is not skipped by a known gate, and + at least one matched slot is missing or partial. `unconfirmable` is excluded. + +Browser collection can still produce a page result with warnings when: + +- page settle times out; +- a navigation redirects before final URL matching; +- scroll evidence is incomplete; +- GPT is not loaded; +- extra live DOM/GPT ad-slot evidence has no matched configured slot; +- no slots match the URL. + +## 10. Testing + +Static tests: + +- parse every `ts config ad-templates` command; +- load temp `edgezero.toml` and temp `trusted-server.toml`; +- verify `--app-config`, `--manifest`, and `--no-env` behavior; +- verify `/`, `/news/*`, and full URL normalization behavior; +- verify `check` success and failure output. +- verify the existing static command loader uses the shared `app_config` module. + +Pure comparison tests: + +- exact DOM ID match; +- prefix DOM ID match for framework-generated suffixes; +- ignore `-container` elements; +- GPT confirms by GAM unit path, div ID, and compatible sizes; +- DOM-only creates `partial`; +- no DOM/GPT creates `missing`; +- APS match creates no provider warning; +- APS missing/ambiguous creates provider warnings; +- `--strict` fails only missing and partial slots. + +Browser fixture tests: + +- local HTML fixture with direct `googletag.defineSlot`; +- fixture using `googletag.cmd.push`; +- fixture assigning `window.googletag` after collector injection; +- fixture with delayed/lazy slot observed only with `--scroll`; +- fixture with APS `fetchBids`; +- fixture assigning `window.apstag` after collector injection; +- redirect fixture that matches expected slots on final path; +- multi-URL fixture where one URL fails and one URL returns page results; +- fixture where `[auction].enabled = false` reports runtime skipped instead of + strict missing-slot failure; +- invalid non-HTTP(S) URL rejection before browser launch; +- JSON contract tests for warning codes, `extra_evidence`, page errors, + deterministic ordering, `partial`, `missing`, and strict failures; +- fixture with no real GPT/APS network dependency. + +Verification commands: + +```bash +cargo test --workspace +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test --package trusted-server-cli --target +``` + +## 11. Branch And PR Plan + +The implementation should not be built on stale #724. + +Recommended dependency order: + +1. Land or rebase PR #799 as the CLI base. +2. Rebase PR #800 onto the latest #799 head so `ts audit` uses the current typed + blob app-config model. +3. Harden and refactor the existing static `ts config ad-templates ...` + diagnostics on top of the current server-side ad-template branch and #799: + extract the private config loader into `app_config`, move pure expected-slot + logic into `ad_templates::expected`, and keep existing behavior covered by + tests. +4. Extend the rebased #800 collector with pre-navigation init scripts, + ad-template evidence hooks, optional scroll, page-level errors, and bounded + structured output. +5. Build `ts audit ad-templates verify` on top of that collector and the + server-side ad-template branch. +6. Keep `generate` for a separate Phase 2 spec and PR. + +If delivery needs to be split, static diagnostics can land before browser-backed +verification. Browser-backed verification should not duplicate the #800 browser +collector. + +## 12. CLI Namespace Decision + +`ts audit ad-templates verify` is the final command shape for browser-backed +ad-template verification. + +When this work is combined with the rebased #800 audit command, `ts audit` +should become a subcommand namespace: + +```bash +ts audit page +ts audit generate +ts audit ad-templates verify ... +``` + +The existing #800 `ts audit ` behavior should be preserved as a +compatibility alias for `ts audit generate ` during the transition, +including its artifact output flags. This avoids a successful but silent +behavior change for existing onboarding scripts. + +Parsing contract: + +- `ts audit page ` is the canonical generic page-audit command. +- `ts audit generate ` is the canonical artifact-generation command. +- `ts audit ad-templates verify ...` is the canonical ad-template verifier. +- `ts audit ` is a hidden compatibility alias for + `ts audit generate ` and is accepted only when `` parses as `http` + or `https`. +- `ts audit ad-templates` must never be treated as a legacy URL positional. +- `ts audit page` without a URL must fail with the normal Clap missing-argument + error. + +Implementation shape: + +```rust +#[derive(Debug, clap::Args)] +struct AuditArgs { + #[command(subcommand)] + command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + legacy_url: Option, +} + +#[derive(Debug, clap::Subcommand)] +enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), +} +``` + +If Clap cannot enforce the optional-subcommand plus hidden positional contract +cleanly, implement a small custom dispatcher for the `audit` argv tail and test +it directly. Required parser tests: + +- `ts audit https://www.example.com/` dispatches to artifact generation; +- `ts audit page https://www.example.com/` dispatches to page audit; +- `ts audit ad-templates verify https://www.example.com/` dispatches to + ad-template verification; +- `ts audit ad-templates` does not parse as a URL; +- `ts audit ftp://www.example.com/` fails before browser launch. + +JSON error output is intentionally left to the broader CLI output contract. This +spec only standardizes successful verification result JSON and strict-mode +verification failure JSON where page results exist. diff --git a/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md new file mode 100644 index 000000000..200a8abbe --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md @@ -0,0 +1,15 @@ +# Contiguous Generated Slot Tables Design + +## Problem + +`splice_creative_slots` parses rendered slots in a temporary `toml_edit::DocumentMut` and moves its `ArrayOfTables` into the target document. Parsed tables retain document-local numeric positions. Those positions collide with positions in the target document, so serialization can interleave generated slot and provider tables with unrelated top-level tables even though the resulting TOML remains semantically valid. + +## Design + +Before insertion, assign the generated slot tables and all nested provider tables the target `[creative_opportunities]` table's document position. `toml_edit` performs a stable position sort, so equal positions retain traversal order: the creative table, each slot, and that slot's provider tables remain contiguous. For a newly created creative section, allocate an anchor after the greatest existing parsed-table position. + +The update continues to preserve unrelated values, comments, line endings, and semantic table ownership. It does not reformat existing operator-authored content or modify slot inference. + +## Testing + +Add a regression fixture with a late `[creative_opportunities]` section and unrelated tables whose positions overlap those from the temporary generated document. Assert that the parent, generated slots, and provider subtables serialize contiguously before the next unrelated table. Retain the existing semantic-preservation and CRLF tests, then run the CLI test suite, formatting, and native CLI clippy. diff --git a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md new file mode 100644 index 000000000..5ff4d0707 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md @@ -0,0 +1,254 @@ +# PR 823 Review Resolution Design + +## Goal + +Resolve the actionable findings in review `4958563121` on PR 823 without +unrelated refactoring, verify the complete branch, publish the fixes, and reply +to every inline review thread with concrete resolution evidence. + +## Scope + +The implementation covers all 28 inline threads and all actionable items in the +review summary. The summary's explicitly out-of-scope pre-existing +partially-invalid `page_patterns` behavior is not expanded into this PR unless a +fix is required by another in-scope change. The PR description's stale legacy +alias sentence is corrected after the branch changes are published. + +Each reviewer suggestion is verified against the current code. A suggestion is +implemented when it is correct for this repository. Where repository evidence +contradicts a suggestion, the implementation retains the correct behavior and +the review response explains the evidence. + +## Design Principles + +- Preserve operator-authored configuration, comments, ordering, and unrelated + sections byte-for-byte wherever possible. +- Never print secrets or whole effective configuration documents as diagnostic + output. +- Never turn uncertain crawl evidence into a runnable fabricated ad-unit path. +- Treat browser navigation as a session, not a sequence of isolated launches. +- Keep `generate`, `verify`, static CLI commands, and runtime matching on shared + domain rules instead of parallel reimplementations. +- Bound all page-controlled data and browser operations. +- Use test-first changes for behavior corrections and minimal annotations for + code-quality-only corrections. + +## Component Design + +### 1. Configuration integrity and command output + +`slot_toml` will replace the line-oriented slot-boundary heuristic with a +TOML-aware edit strategy. The resulting document must preserve every top-level +item outside the managed creative-opportunity fields and preserve comments +adjacent to or between operator sections. Non-contiguous slot declarations, +multiline values, arrays whose continuation lines begin with `[`, trailing +comments, CRLF input, and inline-slot conversion receive regression coverage. +The updater will reject a candidate if preservation cannot be proven. + +Generation will re-read the source config immediately before the atomic write +and refuse to overwrite a concurrently edited file. `--dry-run` will emit only +the managed creative-opportunities change, never the complete config. Notes and +rollback warnings go to stderr so machine-readable stdout remains clean. Tests +will prove that dry-run leaves the source file byte-identical and does not expose +unrelated secret-bearing keys. + +Merge behavior remains add-only for operator-authored data: existing templated +unit paths are retained, newly observed formats are unioned, and multiple +discovered placements absorbed by one broad configured div prefix produce an +operator note. + +### 2. Crawl evidence and inference + +Inference will preserve evidence instead of silently collapsing it: + +- Non-ASCII shared-prefix computation uses UTF-8 byte boundaries. +- Same-page normalization collisions retain distinct raw placements and emit a + diagnostic rather than silently dropping formats. Numeric-only stable tokens + are not classified as hexadecimal hash noise. +- Multi-slot SRA request fallbacks are ignored when `dids` names more than one + slot. +- A page is considered empty only when no audited profile found slots there. +- Fragment detection requires stronger evidence: a useful shared prefix, or at + least three disjoint fragments. Ambiguous two-slot groups are retained with a + note. +- Locale landing paths are emitted literally when they are shorter than the + inferred section depth, and literal path segments are escaped before being + interpolated into globs. +- Refused template decisions are omitted from generated slots and surfaced with + their reasons. The documentation and tests will consistently describe these + cases as refusal, not literal fallback. +- The redundant witness rule is removed or made independently meaningful. The + actual crawler will support the section depth that inference can produce; + locale-prefixed behavior will not exist only in hand-built evidence tests. +- Dropped-section diagnostics are capped, percent-encoded paths are normalized + before filtering, and page-like extensions are classified consistently. + +The root page and section pages for a device profile are collected in one +browser session. Page analysis that parses full HTML is moved off the +current-thread CDP event pump. Each page/tab is closed on every success and +error path. + +### 3. Shared browser behavior + +The browser collectors will share executable discovery and launch/session +configuration. Browser options exposed to operators will have one meaning in +`page`, `verify`, and `generate`: Chrome override, settling, headful/headless +mode, device profile/viewport, proxy, consent assumption, cookies, and TLS +policy. + +`verify` will reuse one browser/runtime/profile across its URLs so clearance and +session state survive. The generic/legacy generator will default to the same +consent assumption as ad-template generation and expose the opt-out rather than +depending on `derive(Default)`. + +Cookie parameters are explicitly host-only with `Path=/`. A same-host +`http`-to-`https` upgrade is accepted with a redirect note; host changes, +downgrades, and unexpected port changes remain cross-origin refusals. Failure to +read or parse the final browser URL fails closed instead of substituting the +requested URL. + +Every post-navigation evaluation is time-bounded. The collector enlarges the +resource timing buffer before navigation, waits for an interactive or complete +document before accruing quiet time, honors sub-poll quiet windows, validates +`quiet <= max`, and reports saturation. Navigation load-event timeout is a +warning after a successful `goto`; it does not discard readable page evidence. +Evidence payload bytes and captured string lengths are capped before expensive +decode/allocation. + +Init-script and page-evaluation failures become explicit warnings or errors +rather than empty evidence. Promise-returning sitemap evaluation awaits its +result. Main-frame-only collection is disclosed when frames are skipped. + +The injected collector will be behavior-preserving: size pairs enforce the +`u32` range, the `googletag` setter is total, the unused non-variadic `cmd.push` +wrapper is removed, wrapping markers are closure-local/non-enumerable, and +page-derived warning text is terminal-safe. + +### 4. Runtime and static-command parity + +Expected-slot projection uses the runtime's renderability rule. Slots the +runtime omits for a path do not count as matched verification slots; diagnostics +state that the runtime omits the slot on that path rather than claiming the +whole config is rejected. + +Configured media type remains a typed `MediaType` through comparison and is +rendered to a string only at the output boundary. Slots that the phase-one +checker cannot confirm (video/native-only) are represented as unconfirmable and +do not fail `--strict`; genuinely partial or missing confirmable slots still +fail, including a live out-of-page slot with no sizes matched against +banner-configured formats, which is partial. Slot phase is absent when no +evidence exists. +The server-side APS compatibility field no longer creates unconditional +client-side `fetchBids` warnings. + +Collector warnings are included in page results. Human output includes the +runtime expectation, gate summary, matched count, extra evidence, and warnings +already present in JSON. Output escaping covers Unicode bidi controls and all +config-derived strings. + +`explain` reports exactly the shared runtime gate result. Provider configuration +is a separate advisory. The unsupported `--edgezero-enabled` model and stale +legacy-fallback claim are removed because no runtime condition backs them. +Gate diagnostics consume the shared gate result instead of rebuilding lists by +hand. The hot runtime gate avoids heap allocation, the seven-boolean wrapper is +removed, and the consent tri-state is documented and exhaustively tested. + +`compile_page_pattern` becomes crate-private and a public validation-only API is +used by the CLI. `lint` explicitly reports every configured page pattern the +runtime would drop, while the broader pre-existing runtime acceptance policy +remains out of scope. Specific compile failures are retained in logs. HTTP +methods use `http::Method` parsing so CLI semantics match the runtime. + +Full URLs and bare path inputs pass through the same URL normalization rules: +percent-encoding, dot-segment resolution, query/fragment removal, and leading +slash behavior must be identical. Scheme detection is anchored to the path +portion before `?`, so an absolute URL inside a query value does not cause a +bare path to be parsed as a full URL. + +### 5. CLI contracts, documentation, and CI + +Clap owns argument validation: URL parsing happens at the value parser, the +audit namespace uses help-on-missing-subcommand, `check` uses an argument group +and conflicts, and settle bounds are rejected during parsing. Parser tests cover +the visible command shapes and legacy restrictions. + +CI-oriented assertion failures exit 1; tool/configuration/navigation failures +exit 2. Assertion text is written directly and cannot disappear behind a log +filter. The guide documents all four `ts config ad-templates` commands, all +flags, shared config-loading flags, browser flags, consent/profile behavior, +dry-run output, and exit codes. + +Browser fixture CI either installs/resolves Chrome and requires the tests to +execute, or explicitly opts into a mode that fails when Chrome is unavailable; +it may not report success after silently skipping every browser assertion. + +All real-looking customer identifiers and names introduced by this PR are +replaced with fictional values in tests, comments, and documentation. Stale +module-level lint suppressions, inaccurate docs, assertion messages, enum +ordering, dead query matching, and orphaned comments are corrected without +unrelated cleanup. + +## Inline Review Traceability + +| Thread | Resolution area | +| -------------------------- | ------------------------------------------------------------------ | +| `3802056460`, `3802056470` | TOML-aware splice and comment/value preservation | +| `3802056474` | Secret-safe dry-run and stderr diagnostics | +| `3802056481` | Omit and explain refused slots | +| `3802056488` | UTF-8-safe div prefix calculation | +| `3802056494` | Same-page normalized-div collisions | +| `3802056497` | Locale landing-page patterns | +| `3802056502` | Multi-profile empty-page accounting | +| `3802056508` | Close every browser tab | +| `3802056513` | Enforce JavaScript-to-Rust `u32` bounds | +| `3802056521`, `3802056529` | Total GPT hook and removal of behavior-changing `cmd.push` wrapper | +| `3802056539` | Shared faithful browser launch configuration | +| `3802056549`, `3802056555` | Correct settling and load-timeout handling | +| `3802056559` | Preserve injected collector warnings | +| `3802056564`, `3802056571` | Runtime renderability parity and accurate diagnostics | +| `3802056580`, `3802056584` | Unconfirmable status and removal of false APS warning | +| `3802056586` | Identical URL and bare-path normalization | +| `3802056593` | Fictional committed examples | +| `3802056599` | Browser fixture CI must execute or fail loudly | +| `3802056605` | Add-only merge of formats with broad-prefix diagnostics | +| `3802056614` | Consent parity for generic and legacy generation | +| `3802056623` | Refusal behavior, tests, and documentation agree | +| `3802056628` | Safe same-host HTTP-to-HTTPS redirect handling | +| `3802056638` | Remove ungrounded EdgeZero fallback model | + +## Error Handling and Compatibility + +All new Rust fallible paths use the repository's existing `CliResult` / +`error-stack` conventions. Browser failures identify the operation and URL but +do not include cookies, configuration values, or page payloads. Best-effort +cleanup must not replace an earlier collection error. + +JSON compatibility is preserved where possible. New distinctions are additive +or correct semantically invalid fields: unconfirmable status is explicit, and +phase may be omitted when there was no evidence. Documentation is updated with +the exact wire behavior. + +## Verification Strategy + +Each behavioral issue follows red-green-refactor: + +1. Add the smallest unit, parser, orchestration, or fixture test reproducing the + review finding. +2. Run the narrow test and confirm the expected failure. +3. Implement the minimal correction. +4. Re-run the narrow test and the affected crate suite. + +Final verification runs the repository-required commands relevant to the +changed surface: CLI tests through `scripts/test-cli.sh`, target-matched Rust +tests, JS tests when the collector script changes, `cargo fmt --all -- --check`, +all target-matched clippy aliases, documentation formatting, and browser fixture +tests with an available Chrome. Any environment-dependent test that cannot run +is reported explicitly and is not described as passing. + +## Review Replies and Publication + +Changes are grouped into reviewable commits by component, then pushed to the PR +branch after final verification. Each inline reply is posted in its existing +thread and states the concrete change, relevant test, or evidence-backed reason +for retaining behavior. Replies avoid generic acknowledgements. Threads are not +replied to as fixed until the corresponding commit is visible on GitHub. diff --git a/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md new file mode 100644 index 000000000..e9025174d --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md @@ -0,0 +1,15 @@ +# Pre-navigation Cookie Installation Design + +## Problem + +The audit collectors open `about:blank` so initialization scripts can be installed before publisher code runs. Cookies are explicitly scoped by domain and `/`, but `chromiumoxide::Page::set_cookie` rejects cookies without a URL while the page is still `about:blank`. Consequently, any audit using `--cookie` fails before navigation; audits without cookies are unaffected. + +## Design + +Build the same host-only, root-scoped `CookieParam` values, then install them through `Browser::set_cookies` before creating the page. Browser-level installation sends the explicit domain/path cookie directly to Chrome without deriving scope from the current page URL. Both verification and generation collectors use one shared helper so their behavior cannot drift. + +Cookie-installation errors remain fatal and identify the affected cookie without logging its value. Page initialization, first-request authentication, browser-session reuse, and cookie scope remain unchanged. + +## Testing + +Add a Chrome-backed regression test that starts with `about:blank`, installs a cookie through the shared browser helper, navigates to a local HTTP fixture, and verifies the cookie is visible on the first loaded document. Run the focused CLI tests, formatting, and lint checks required for the touched crate. diff --git a/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md new file mode 100644 index 000000000..8f75d0dcf --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md @@ -0,0 +1,59 @@ +# Ad-template generation progress design + +## Problem + +`ts audit ad-templates generate` audits up to the configured page budget for +each selected device profile (17 pages by default). Navigation and page settling +are intentionally bounded but can still take tens of seconds per page. The +browser collector buffers page results until the browser session closes, so the +command currently emits no output during most of that work and appears stuck. + +## Design + +Emit line-oriented progress on stderr while collection is running. Progress +must identify the device profile, current page, known total, and safe page +location. It must also identify non-page phases where a noticeable pause can +occur: launching the browser, planning the crawl after the root page, and +finalizing the browser session. + +Progress is an explicit collector callback rather than direct terminal output +inside the browser implementation. This keeps output policy in the command +layer, makes the behavior testable with in-memory writers, and lets non-browser +collectors preserve the same contract. Each line is flushed immediately. + +The first profile's root navigation has no final total because follow-up pages +are planned from the rendered root. It is reported as `1/?`; once planning +finishes, subsequent pages use a stable `current/total` count. Later profiles +receive the complete target list and report the root as `1/total`. Totals include +the root, and every attempted page advances the current count even if collection +fails. + +Progress never prints a full URL. It renders only the origin-free path, omitting +userinfo, query, and fragment data, then applies the CLI's existing terminal-text +sanitizer. An empty path is rendered as `/`. + +Stdout remains reserved for the generated diff or success summary. This keeps +`--dry-run` and shell redirection stable. Progress is intentionally plain text, +not an animated spinner, so it remains useful in logs and does not add a terminal +UI dependency. + +## Error handling + +Failure to write or flush progress is returned as a normal CLI output error. A +callback failure during a browser session stops further collection but does not +skip finalization, browser close, or process wait. An earlier collection or +planning error takes precedence over a later progress error; either takes +precedence over teardown errors. Close and wait are still attempted +independently. No cookie values, URL credentials, query values, fragments, or +browser credentials are included in progress. + +## Tests + +Unit tests will verify that progress is emitted before collection completes, +contains the specified profile-aware page counts, keeps stdout unchanged, +redacts URL credentials/query/fragment data, sanitizes paths, and reports +finalization. Writer tests will cover write failure, flush failure, and explicit +flush invocation. Collector tests will verify teardown still runs after progress +failure and that collection/planning errors, progress errors, and teardown errors +retain the stated precedence. The existing CLI and Chrome-backed suites will +verify the collector behavior and browser lifecycle remain intact. diff --git a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md new file mode 100644 index 000000000..bdc9c5b9c --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md @@ -0,0 +1,95 @@ +# Refuse Volatile Div-ID Collisions + +## Problem + +GPT discovery normalizes per-render div IDs such as +`ad-in_content--in_content-0` to the stable prefix `ad-in_content`. +When several live elements on the same page normalize to that prefix, the +runtime cannot represent them safely: one prefix resolves at most one element, +while each exact raw ID changes on a later render. The current collision path +preserves the raw IDs, causing `--replace` to write unusable literal slots. + +## Design + +Treat a source-local normalized collision as ambiguous and refuse the entire +group. The first observation remains tentatively accepted. When a second raw div +ID that describes a _different element_ normalizes to the same prefix, remove +the first slot, record the group as ambiguous, and suppress every later member. +Emit one diagnostic when the group first becomes ambiguous, naming the +normalized prefix and explaining that neither a single prefix nor volatile exact +IDs are safe. Tell the operator to expose distinct stable div IDs or prefixes in +publisher markup before configuring the placements. + +Two raw IDs sharing a stem are not by themselves two elements. One element +re-rendered under a fresh framework token produces exactly that shape, and +absorbing it is what normalization is for: a React publisher reports +`ad-header-0-_R_3f_` from the server render and `ad-header-0-_r_0_` from the +client one, and refusing that pair would generate no slots at all. The two cases +are separated by comparing what the ephemeral markers did _not_ cover — the +marker spans are excised and the remaining parts compared, so identical +residues mean one element observed twice, while `-in_content-0` against +`-in_content-1` means two siblings and is refused. + +The verdict is site-wide, not page-local. Article pages carry several in-content +units and refuse the shared prefix while a landing page carries one, so a +page-local refusal would let crawl sampling decide whether the ambiguous prefix +reaches the config. `DiscoveredSlots` therefore carries the refused stems, +`EvidenceTable` unions them across pages, and the slot iterator the writer reads +suppresses them regardless of which page contributed them. + +Registry and request-derived evidence retain separate collision maps, matching +the current source precedence: even an ambiguous registry stem continues to +suppress request fallback for that stem. Network-ID discovery is unaffected. + +`DiscoveredSlots` records whether any otherwise usable GPT slot evidence was +seen independently of how many safe slots remain. `EvidenceTable::fold_page` +uses that signal when classifying empty pages, so a collision-only page is not +mistaken for a bot challenge. Cross-page slot inference, merging, and +`--replace` otherwise remain unchanged because ambiguous slots never enter +those stages. + +Some ad stacks build IDs as `__`, where the +render token — at least ten leading digits followed by more alphanumerics, +that is, a millisecond timestamp plus entropy — sits _before_ the part that +distinguishes one placement from the next. Such an ID can be written neither +literally nor as a prefix: the only stable prefix stops at the token and reaches +every placement in the family at once. Discovery refuses a single otherwise +usable registry or request observation of that shape, preserves the page/network +evidence, and emits one diagnostic naming the family prefix. The shape decides +rather than a vendor name, so any stack with this layout is covered without a +code change, and every placement after the token is covered rather than an +enumerated few. A token in trailing position is _not_ this case — everything +before it still identifies the element — and is left to normalization and the +collision check. + +## Safety and Output + +The generator prefers omission over a configuration that cannot match future +renders. For an observed desktop crawl of a site with this mix, replacement +output should therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` +slots, while the in-content collision group and the volatile-token family are +explained in notes. + +## Tests + +- A two-element same-page normalization collision yields no slots and one + diagnostic containing the prefix, both unsafe alternatives, and operator + action. +- Two renders of one element (identical residues either side of the marker, + including a React server/client pair) collapse to one slot with no diagnostic. +- Repeats of the first and second IDs plus a third distinct ID after a collision + remain suppressed and do not create additional diagnostics. +- Request-derived collisions follow the same policy. +- An ambiguous registry stem still suppresses request fallback, and network-ID + discovery survives when every collided slot is omitted. +- A stem refused on one page stays refused after a later page contributes a + single member of the group. +- A collision-only page is recorded as having evidence rather than as an empty + challenge page. +- Single registry- and request-derived render-token observations are omitted + while retaining evidence and any parseable network ID, for every placement + suffix after the token. +- IDs with no render token, with a bare digit run, or with a trailing token stay + eligible. +- Existing normalization, request fallback, fragment detection, and full CLI + tests remain green. diff --git a/docs/superpowers/specs/2026-08-21-liveramp-integration-design.md b/docs/superpowers/specs/2026-08-21-liveramp-integration-design.md new file mode 100644 index 000000000..c4611ff08 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-liveramp-integration-design.md @@ -0,0 +1,856 @@ +# LiveRamp Integration Design + +**Issue:** [#355 — Investigate and document LiveRamp integration](https://github.com/IABTechLab/trusted-server/issues/355) + +**Parent epic:** [#354 — LiveRamp integration](https://github.com/IABTechLab/trusted-server/issues/354) + +**Initiative:** [#55 — Monetization integrations](https://github.com/IABTechLab/trusted-server/issues/55) + +**Status:** Draft PR implemented; external live validation partially complete + +**Date:** 2026-08-21 + +**Revised:** 2026-08-31 + +## 1. Executive summary + +LiveRamp integration is feasible in two distinct forms, but they must not be +treated as one protocol: + +1. **RampID identity-envelope forwarding through Prebid.js is feasible now.** + Trusted Server already bundles Prebid's `identityLinkIdSystem`, reads + `liveramp.com` EIDs through `pbjs.getUserIdsAsEids()`, sends them to + `/auction`, merges them with EC/KV identities, applies consent gating, and + forwards them to Prebid Server as OpenRTB `user.ext.eids`. +2. **LiveRamp ATS Direct audience segments are not part of that EID flow.** ATS + Direct returns a separate segment envelope and has separate subscription, + storage, TTL, deal-approval, and activation requirements. The cited + LiveRamp documentation describes activating these values as GAM `atsd` + targeting, not as a `liveramp.com` EID. + +The first implementation makes the existing RampID path operationally complete +through vendor-neutral `managed_user_ids` configuration under the Prebid +integration. The generated bundle command must resolve every managed config +name through the checked-in User ID registry and reject a manifest that omits +the corresponding module. Native server-to-server ATS resolution and ATS +Direct segment activation remain separate follow-up decisions. + +## 2. Issue hierarchy and collected requirements + +The GitHub issue hierarchy is: + +```text +#55 Initiative: Monetization integrations +└── #354 Epic: LiveRamp integration + └── #355 Task: Investigate and document LiveRamp integration +``` + +The work also references `IABTechLab/uid2-optout#385`, which tracked access to +LiveRamp test credentials. It is a related cross-repository dependency, not part +of the trusted-server issue hierarchy. + +The three trusted-server issues have empty or placeholder bodies, so their +comments and linked documentation define the operative requirements. + +### 2.1 Issue #354 + +The only comment asks the team to confirm whether LiveRamp segments are passed +to auction requests through the Prebid.js integration. This specification must +therefore distinguish identity envelopes from segment data and answer both +questions explicitly. + +### 2.2 Issue #355 + +The comments establish the following sequence and requirements: + +1. Review LiveRamp's Real-Time Identity Service (RTIS) tag documentation. +2. Wait for LiveRamp to clarify the integration. +3. Test the documentation LiveRamp supplied. +4. Review the ATS Envelope API page LiveRamp recommended. +5. Write a specification and determine feasibility. + +The issue's direct deliverable is an evidence-backed specification. If the +recommended path is feasible, implementation follows the approved design. + +### 2.3 Credential dependency + +The work references the cross-repository tracking issue +[IABTechLab/uid2-optout#385](https://github.com/IABTechLab/uid2-optout/issues/385), +named “Get test credentials from LR team.” The implementation owner has since +confirmed access to a test Placement ID and a MITM-assisted browser validation +environment. Those values remain outside the repository. + +Automated tests must not depend on LiveRamp configuration. A live Placement ID +and a LiveRamp-approved test origin remain necessary for the outstanding live +validation matrix, but their availability is no longer an implementation +blocker. + +## 3. Terminology and product boundaries + +### 3.1 RampID identity envelope + +Prebid's LiveRamp module is named `identityLinkIdSystem`, its configuration name +is `identityLink`, and its EID source is `liveramp.com`. It resolves an encrypted +RampID envelope into Prebid's identity APIs. The envelope identifies a user to +authorized demand partners; Trusted Server treats the value as opaque. + +### 3.2 RTIS + +LiveRamp's Real-Time Identity Service tag is a pixel or JavaScript tag that uses +LiveRamp cookie recognition and redirects a RampID to an endpoint registered +with LiveRamp. It requires LiveRamp to configure a tag ID and callback endpoint. +Trusted Server has no RTIS callback route today. + +RTIS is not selected for the first implementation because the managed Prebid +module already provides the browser-to-bidstream path, while a new callback +would require correlation, endpoint authentication, storage, abuse protection, +and a LiveRamp-specific server contract. + +### 3.3 ATS Envelope API + +The ATS Envelope API resolves hashed email, hashed phone, or configured custom +IDs into one or more encrypted envelopes. A server-to-server call requires a +Placement ID, a privacy-approved Origin, consent parameters where applicable, +and the browser's client IP in `X-Forwarded-For`. + +The ordinary ATS response contains an identity envelope with `type: 19` and +`source: "envelopeLiveramp"`. A no-consent response is HTTP 204. Configuration, +authorization, service, and geographic/consent failures use distinct 4xx +statuses. + +### 3.4 ATS Direct segments + +ATS Direct is a separate product layered onto an approved ATS placement and +subscription. Its V2 response can include `type: 26`, `source: "atsDirect"`, +whose value represents matching deal/segment IDs. LiveRamp documents storing +this in `_lr_atsDirect`, maintaining a region-dependent TTL, refreshing it, and +applying selected deal IDs to GAM under the `atsd` targeting key. + +An ATS Direct segment envelope is not a RampID and must not be placed in +`user.ext.eids` under `liveramp.com`. + +## 4. Current Trusted Server capabilities + +The following capabilities already exist on `main`: + +- `crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json` + includes `identityLinkIdSystem` in the default preset, maps the Prebid config + name `identityLink`, and maps EID source `liveramp.com`. +- `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` reads + `pbjs.getUserIdsAsEids()`, validates EID structure, and includes valid EIDs in + the current `/auction` request. +- The same TSJS module persists structured OpenRTB-style EIDs in the first-party + `ts-eids` cookie after auction completion. +- `crates/trusted-server-core/src/auction/endpoints.rs` parses current-request + EIDs, loads server-resolved EIDs from the EC/KV graph, merges and deduplicates + them, and applies centralized consent gating. +- `crates/trusted-server-core/src/integrations/prebid.rs` serializes the merged + set to Prebid Server as OpenRTB `user.ext.eids`. +- `crates/trusted-server-core/src/ec/prebid_eids.rs` ingests `ts-eids` on a + later request and maps configured sources such as `liveramp.com` into the + EC/KV identity graph. +- The external bundle manifest and runtime diagnostics already identify which + Prebid User ID modules were compiled into the bundle. + +### 4.1 Bundle consistency + +The implementation validates managed entries against the checked-in User ID +module registry during `ts prebid bundle`. The CLI resolves each managed config +name through the registry, rejects unknown or ambiguous names, invalidates any +stale manifest before generation, and confirms that the fresh manifest contains +every required module before updating deployable hash metadata. Runtime +diagnostics retain the same defense for externally supplied or stale artifacts. + +### 4.2 Browser consent activation + +Bundling Prebid's consent collector and activity-control modules makes browser +enforcement available, but does not activate it. Prebid activates the TCF path +only after `consentManagement.gdpr` is configured. The managed User ID path +therefore initializes the standard IAB collector when all of the following are +true: + +- at least one managed User ID entry is configured; +- the publisher has not already configured `consentManagement.gdpr`; and +- the page exposes the IAB `__tcfapi`. + +The shim performs this check before seeding managed User IDs and before +`processQueue()`. It preserves every publisher-owned consent setting and does +not force GDPR scope or add a CMP configuration on pages without the TCF API. +This keeps the browser behavior vendor-neutral and avoids imposing GDPR latency +or defaults on non-TCF publishers. + +The effective value returned by `pbjs.getConfig("consentManagement")` is the +source of truth at installation time. If it is an object with its own `gdpr` +property, that property is publisher-owned and the shim leaves it untouched +regardless of its value, including `null`, `false`, or a partial object. If the +effective value is absent, or is an object without its own `gdpr` property, the +shim adds only: + +```js +{ + consentManagement: { + ...existingConsentManagement, + gdpr: { cmpApi: "iab" }, + }, +} +``` + +Prebid's timeout and `defaultGdprScope` defaults remain authoritative. Existing +sibling settings such as `gpp` are copied into the update. A non-object or +throwing effective value is not safe to merge: the shim logs a diagnostic and +does not replace it. + +The automatic update uses the original Prebid `setConfig` function and records +that the shim owns the resulting IAB collector. Publisher configuration already +applied before the shim therefore wins immediately. If a queued or late +`setConfig` or `mergeConfig` call later supplies an own `gdpr` value, including +`null` or `false`, ownership transfers to the publisher. Before forwarding that +call, the shim sends `gdpr.enabled = false` through the original `setConfig` API. +This invokes Prebid's supported consent reset path and removes the CMP event +listener when its ID is already known. Together with the callback guard below, +it prevents a later IAB event from overwriting publisher-owned static or custom +consent. + +Prebid cannot remove an IAB listener before the CMP returns its listener ID. To +cover that interval, the shim guards only the callback registered by its own +automatic activation. After ownership transfers, a delayed first response is +not forwarded into Prebid's consent handler; when it carries a listener ID, the +guard asks the TCF API to remove that stale subscription. The page's current +callable `__tcfapi` is used for removal, with the function captured at activation +as a fallback for pages whose API disappears. The page's global `__tcfapi` +function is restored immediately after activation, so publisher and CMP calls +outside that subscription are unchanged. + +The cleanup update preserves effective sibling consent settings. A following +publisher `setConfig` call retains its normal replacement semantics. Because +Prebid's `mergeConfig` deep-merges with the temporary disabled value, the shim +adds `enabled = true` only when the publisher supplied an object-valued `gdpr` +whose `enabled` value is missing or `undefined`; this restores Prebid's normal +enabled default without changing an explicit boolean publisher choice. The +publisher merge is prepared before the cleanup update. If it cannot be safely +inspected, cleanup is skipped so the temporary disabled value cannot leak into +the publisher's effective configuration. The transfer occurs at most once, +sibling-only consent updates do not claim GDPR ownership, and the shim never +re-applies its automatic minimum afterward. Throwing configuration accessors +are caught and logged rather than breaking shim installation. If effective +consent state cannot be read or copied during transfer, the shim does not issue +a replacement cleanup update that could erase unknown sibling state; the +guarded callback still rejects stale automatic responses, and the publisher call +is forwarded unchanged. + +## 5. Approaches considered + +### 5.1 Selected: vendor-neutral managed User IDs with bundle validation + +Add `managed_user_ids` to `PrebidIntegrationConfig`, inject the opaque entries +through `window.__tsjs_prebid`, and let the TSJS Prebid shim install and protect +each operator-owned Prebid User ID configuration before queued work is +processed. At bundle time, the CLI reads the same registry as the JavaScript +generator, resolves each managed config name, and confirms that the freshly +generated manifest contains every required module before updating hash/SRI +metadata. + +Benefits: + +- Uses the existing module, bundle generator, EID transport, consent gate, and + EC/KV ingestion path. +- Keeps browser identity configuration beside the Prebid bundle that consumes + it. +- Adds no new upstream route or PII-bearing server API. +- Fails an unusable managed-name/module pairing during the bundle command. +- Can be fully tested without external credentials, with a separate live + verification gate. + +Trade-offs: this only resolves identities visible to browser modules, and the +CLI must deserialize the registry's vendor-neutral module/config-name schema. +It does not add server-side HEM resolution or ATS Direct segments. + +### 5.2 Alternative boundary: standalone LiveRamp integration + +A new `integrations/liveramp` module could own browser and server APIs. This is +not needed for the browser path implemented by the current PR, while the ATS +API input contract and ATS Direct product scope remain unresolved. It would +also duplicate Prebid lifecycle and bundle validation responsibilities if added +before a server-side consumer is confirmed. + +Revisit this boundary if a future approved design adds server-to-server ATS +resolution or a non-Prebid LiveRamp consumer. + +### 5.3 Deferred: RTIS callback endpoint + +An RTIS endpoint would introduce a new unauthenticated redirect/callback +surface and a correlation problem without improving the already-supported +Prebid identity path. LiveRamp also requires per-endpoint configuration. It is +not implemented by the current PR and requires explicit team confirmation +before being treated as a follow-up requirement. + +### 5.4 Deferred: native server-to-server ATS resolution + +Native resolution is technically possible with Trusted Server's platform HTTP +abstractions, consent context, geo context, and client IP access. It is not +implementation-ready because: + +- Trusted Server has no approved source for hashed email, hashed phone, or a + LiveRamp custom ID. +- Sending a hashed identifier to LiveRamp is a privacy and publisher-contract + decision, not merely a transport detail. +- ATS API enablement and placement configuration for server-to-server use are + not confirmed by the browser Placement ID alone. +- Rate limits, timeout policy, caching, envelope refresh, and identifier + deletion semantics are not confirmed. +- [#630 — HEM Resolution (LiveRamp)](https://github.com/IABTechLab/trusted-server/issues/630) + was closed as not planned and must not be silently revived. + +## 6. Proposed configuration + +> **Revision, 2026-08-25.** An earlier draft of this section specified a typed +> `[integrations.prebid.liveramp]` subsection, which named a single identity +> vendor inside `trusted-server-core`. It is superseded by the vendor-neutral +> `managed_user_ids` surface below. RampID is now a configuration choice, not a +> type in core. + +Managed Prebid User ID modules are optional and nested under the existing Prebid +integration. Each entry is an opaque passthrough: core validates only what +Prebid needs to address the module, and never interprets `params`. + +```toml +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-.js" +external_bundle_sha256 = "" +external_bundle_sri = "sha384-" + +# RampID, expressed purely as operator configuration. +[[integrations.prebid.managed_user_ids]] +name = "identityLink" +params = { pid = "999", notUse3P = false } + +[integrations.prebid.managed_user_ids.storage] +type = "cookie" +name = "idl_env" +expires = 15 +refresh_in_seconds = 1800 +``` + +The Rust representation names no vendor: + +```rust +pub struct PrebidIntegrationConfig { + // Existing fields omitted. + pub managed_user_ids: Vec, +} + +pub struct PrebidManagedUserIdConfig { + pub name: String, + pub params: serde_json::Map, + pub storage: Option, +} + +pub struct PrebidManagedUserIdStorage { + pub storage_type: PrebidUserIdStorageType, + pub name: String, + pub expires: Option, + pub refresh_in_seconds: Option, +} + +pub enum PrebidUserIdStorageType { + Cookie, + Html5, +} +``` + +Validation: + +| Field | Rule | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Required. Non-empty, untrimmed-free ASCII token of letters, digits, `_`, `-`, or `.`. Unique across entries: Prebid keys `userSync.userIds` by name, so a repeat gives one submodule two conflicting configurations | +| `params` | Optional. Any TOML table; forwarded to Prebid without inspection | +| `storage.type` | Optional. `cookie` (default) or `html5` | +| `storage.name` | Required when `storage` exists. Same token rule as `name` | +| `storage.expires` | Optional. At least 1 when present; omitted leaves Prebid's default. No upper bound — a ceiling is the module's | +| `storage.refresh_in_seconds` | Optional. At least 1 when present; omitted leaves Prebid's default | + +Values that used to be typed defaults in core — `notUse3P = false`, +`idl_env`, 15 days, 1800 seconds — are now operator-supplied, because each is a +property of the module the operator selected rather than of Trusted Server. + +The operator selects both managed entries and bundle modules, but `ts prebid +bundle` validates that selection. It reads +`crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json`, the +same registry used by the JavaScript generator, and joins each managed `name` +against the registry's `configNames`. No vendor-specific mapping is compiled +into the CLI, and registry additions extend both the generator and validation. + +### 6.1 Bundle consistency validation + +The CLI performs focused validation in this order: + +1. Parse the managed User ID names alongside the existing bundle inputs. An + absent key becomes an empty list; a non-array value, non-table entry, or + missing, empty, or non-string `name` fails instead of being skipped. +2. Locate the JavaScript library and load its checked-in User ID registry. +3. Resolve every managed name to exactly one `moduleName`. Unknown or + ambiguously mapped names fail before generator invocation. +4. Remove only the exact `/manifest.json` file when it already exists. A + generator that returns success without writing a new manifest must not reuse + stale metadata from an earlier build. +5. Generate the external Prebid bundle normally. +6. Deserialize `userIdModules` from the newly written manifest. +7. Confirm that every resolved module appears in the manifest. +8. Update `external_bundle_sha256` and `external_bundle_sri` only after the + consistency check succeeds. + +Unknown-name errors identify the managed name and registry path. Ambiguous-name +errors additionally list the candidate modules. Missing-manifest-module errors +identify the managed name, its required module, and the corrective +`integrations.prebid.bundle.user_id_modules` field. A failed consistency check +leaves the existing config metadata unchanged. The browser-side warning remains +as defense in depth for externally built, stale, or modified artifacts. + +An empty `managed_user_ids` preserves current behavior and emits no managed User +ID configuration. + +## 7. Browser configuration and ordering + +The Rust Prebid head injector extends `window.__tsjs_prebid` with a camel-cased +`managedUserIds` array containing the validated entries. A Placement ID is an +operator identifier rather than a secret, but diagnostics must not copy +envelope values. + +The TSJS Prebid shim translates the injected object into: + +```javascript +{ + userSync: { + userIds: [ + { + name: 'identityLink', + params: { + pid: '999', + notUse3P: false, + }, + storage: { + type: 'cookie', + name: 'idl_env', + expires: 15, + refreshInSeconds: 1800, + }, + }, + ] + } +} +``` + +Publisher commands may already be waiting in `window.pbjs.que`, including a +`requestBids` command. Appending the managed configuration would be too late: +Prebid processes existing commands in insertion order, so a publisher auction +could run before the new entry. + +When one or more managed entries are configured, the shim instead installs +narrowly scoped, idempotent wrappers around the public `pbjs.setConfig` and +`pbjs.mergeConfig` APIs before calling `pbjs.processQueue()`: + +1. Capture and bind the real `pbjs.setConfig` and, when present, + `pbjs.mergeConfig` implementations. +2. Replace both public APIs with wrappers that normalize every call containing + a `userSync` object. Calls without `userSync` pass through unchanged. +3. Build a set containing every configured managed name. For a call with an + explicit `userSync.userIds`, preserve entries whose names are outside that + set, remove every publisher-supplied entry whose name is managed, and append + one fresh copy of each operator-managed entry in configuration order. + Preserve sibling `userSync` and top-level fields. +4. Calls whose `userSync` object omits `userIds` pass through unchanged. The + pinned generated Prebid artifact retains its effective `userIds` defaults + across partial `setConfig` and `mergeConfig` updates, so injecting a copied + list in the shim would duplicate Prebid behavior and make the wrapper depend + on a mocked configuration model that does not match the shipped artifact. + A real-artifact characterization test protects this pinned behavior. +5. During initial installation, read the already-effective + `pbjs.getConfig('userSync.userIds')` value, + normalize its supported array/config shape, preserve entries whose names are + not managed, append fresh copies of every managed entry, and apply that + merged list synchronously through the captured function. This covers + publisher configuration that ran after the external Prebid bundle loaded + but before the deferred TSJS shim. An absent or malformed effective list + degrades to an empty publisher list. Complete this step before processing + any existing queue entries. +6. Call `pbjs.processQueue()`. Queued publisher `setConfig` and `mergeConfig` + calls flow through the wrappers, so a later queued `requestBids` observes + the managed entry. +7. Keep the wrappers installed after queue processing so later publisher calls + through either public configuration API cannot silently replace or delete + operator-owned managed entries. Repeated TSJS installation must not stack + wrappers. + +This is configuration ownership for supported Prebid API usage, not a security +boundary against adversarial same-origin JavaScript that retained an earlier +function reference or mutates internal configuration objects directly. + +This policy gives the operator ownership of every configured managed entry. +Publishers retain ownership of all other Prebid and User ID configuration. +Omitting `managed_user_ids` installs no wrapper and preserves current publisher +behavior exactly. + +After queue processing, existing runtime diagnostics repeat the registry-backed +module check against the browser bundle stamp. This is a fallback for bundles +that were built externally, became stale, or were modified after `ts prebid +bundle`; a bundle created by the CLI has already passed the build-time check. + +## 8. Data flow + +```mermaid +sequenceDiagram + participant O as Operator config + participant TS as Trusted Server + participant B as Browser + participant LR as LiveRamp + participant PBS as Prebid Server + participant KV as EC identity graph + + O->>TS: Configure integrations.prebid.managed_user_ids + TS-->>B: Inject managed User ID config and Prebid bundle + B->>B: Guard setConfig/mergeConfig and merge managed entries + B->>LR: Prebid identityLink module resolves/refreshes envelope + LR-->>B: Opaque RampID envelope + B->>B: pbjs.getUserIdsAsEids() + B->>TS: POST /auction with source=liveramp.com EID + TS->>TS: Validate, merge, deduplicate, consent-gate EIDs + TS->>PBS: OpenRTB user.ext.eids + B->>B: Persist structured EIDs in ts-eids after auction + B->>TS: Later request with ts-eids + ts-ec + TS->>KV: Upsert configured liveramp.com partner UID +``` + +Identity resolution is asynchronous. The design does not promise a LiveRamp +EID in the first auction on a new browser. Current-request forwarding applies +as soon as `getUserIdsAsEids()` exposes the envelope; `ts-eids` and EC/KV +ingestion provide reuse on later requests. + +## 9. Consent, privacy, and security + +- Trusted Server continues to apply its centralized consent gate before EIDs + reach providers. No LiveRamp-specific bypass is introduced. +- Prebid's User ID and consent-management modules remain responsible for + deciding whether the browser may call LiveRamp. LiveRamp must be configured + correctly in the publisher's CMP/GVL posture. +- When managed User IDs are active and the publisher exposes `__tcfapi`, the + Trusted Server shim activates Prebid's standard IAB GDPR collector if the + publisher has not already configured one. Existing publisher + `consentManagement.gdpr` settings always win. Pages without `__tcfapi` are + unchanged, and Trusted Server does not synthesize GDPR applicability. +- Correction applied during implementation: `consentManagementTcf` only + _retrieves_ the TC string. Enforcement lives in Prebid's `tcfControl` + activity-control module, which the generated external bundle did not carry. + Without it a denied Purpose 1 still permitted the vendor call and the + `idl_env` write; only EID _forwarding_ was gated, server-side. The bundle now + imports `tcfControl`, covered by + `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs`. + Equivalent GPP/US-state activity controls (`gppControl_usnat`, + `gppControl_usstates`) remain unbundled; US opt-outs are still enforced only + at the server's forwarding gate. +- Pinned Prebid's default `tcfControl` rules do not treat every denied purpose + identically. Purpose 1 plus the module's GVL vendor consent controls + IdentityLink device access, resolution, and storage. Purpose 2 controls bid + fetching. Purpose 3 has no standalone default `tcfControl` rule. Purpose 4 + controls user-provided-data activity. With the default + `eidsRequireP4Consent: false`, EID transmission is permitted when any Purpose + 2–10 has the required purpose/legal-interest and vendor basis; publishers may + opt into requiring Purpose 4 specifically. Therefore a Purpose 3 or Purpose + 4 denial alone does not establish that the LiveRamp vendor request or + `idl_env` write is blocked. Automated artifact tests must vary Purpose 1, + Purposes 3/4, and vendor 97 independently, and the operator guide must + describe these exact defaults rather than claiming that every denied purpose + blocks resolution. +- LiveRamp envelope values are opaque identifiers. They must never appear in + logs, public diagnostics, error bodies, or telemetry dimensions. +- The implementation does not collect plaintext or hashed email and does not + add an API for publishers to submit either value. +- The managed configuration preserves unrelated publisher User ID entries but + owns every configured managed name. A managed `identityLink` entry therefore + prevents ambiguous duplicate LiveRamp configurations without special-casing + LiveRamp in core. +- Existing EID size limits, source/UID validation, cookie caps, merge rules, + and consent withdrawal behavior remain authoritative. +- Live credentials and Placement IDs must not be committed to fixtures or + repository configuration. + +## 10. Error and degraded behavior + +| Condition | Behavior | +| ------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `managed_user_ids` absent | Preserve current behavior; configure no operator-managed User ID entries. | +| Managed name is absent or ambiguous in the registry | Fail `ts prebid bundle` before updating config metadata. | +| Required module is absent from the generated manifest | Fail `ts prebid bundle` and name both the config name and required module. | +| An externally supplied runtime bundle omits a required module | Emit existing browser diagnostics; continue auctions without that module's EID. | +| LiveRamp network or recognition failure | Prebid module yields no EID; continue auction normally. | +| TCF Purpose 1 or LiveRamp vendor consent denied | Default `tcfControl` blocks IdentityLink resolution/storage; continue auction normally. | +| TCF Purpose 3 or 4 denied alone | Default rules do not prove resolution/storage is blocked; publisher policy may add stricter rules. | +| US-state opt-out | Server forwarding gate drops the LiveRamp EID; browser activity controls remain a documented gap. | +| Malformed LiveRamp EID | Existing client/server EID sanitizers drop it. | +| Oversized `ts-eids` payload | Existing bounded cookie behavior truncates whole UID/source entries; no partial UID is written. | +| EC/KV unavailable | Current-request EID can still reach `/auction`; persistence degrades without blocking the auction. | + +Trusted Server does not parse LiveRamp envelope contents and therefore cannot +distinguish authenticated ATS envelopes from cookie-recognized RTIS envelopes. +That distinction remains inside LiveRamp's module and encrypted envelope. + +## 11. Testing strategy + +Implementation follows test-driven development. + +### 11.1 Rust configuration tests + +Add tests in `crates/trusted-server-core/src/integrations/prebid.rs` and the +settings tests to prove: + +- managed entries deserialize with opaque nested `params`; +- documented storage defaults are applied; +- blank or whitespace-padded managed and storage names fail; +- invalid expiry and zero refresh values fail; +- unknown storage types fail; +- duplicate managed names fail; +- omission remains backward-compatible; +- serialized head configuration uses the expected camel-cased keys; +- script-breaking input cannot escape the injected script element. + +### 11.2 TypeScript unit tests + +Add tests in +`crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` proving: + +- no managed config produces no operator-owned entry; +- managed config creates the exact documented Prebid object; +- unrelated publisher `userIds` entries are preserved; +- a publisher-provided entry with a managed name is replaced, not duplicated; +- with at least two managed names, `setConfig` and `mergeConfig` preserve + unrelated publisher entries, replace publisher duplicates of both managed + names exactly once, and append fresh managed copies in configuration order; +- managed configuration is active before an already-queued publisher + `requestBids` call; +- queued publisher `setConfig` followed by `requestBids` preserves other User + ID entries while the auction observes the managed `identityLink` entry; +- User ID entries already effective before TSJS installation are preserved + while the managed `identityLink` entry is added; +- malformed pre-install `userSync.userIds` state degrades to the managed entry + without throwing; +- queued and late publisher `identityLink` updates through `mergeConfig` are + normalized back to the operator-managed values; +- a publisher `identityLink` update through `setConfig` after `processQueue()` + is normalized back to the operator-managed values; +- repeated installation does not stack either configuration wrapper; +- configuration calls without an explicit `userIds` list pass through + unchanged; +- missing `identityLinkIdSystem` appears in existing diagnostics; +- `getUserIdsAsEids()` output for `liveramp.com` enters the current auction; +- malformed and empty envelope values are dropped; +- envelope values are not written to logs or diagnostics; +- the existing `ts-eids` persistence path preserves the opaque value without + decoding it. + +### 11.3 Bundle tests + +Extend external bundle tests to prove: + +- the default preset contains `identityLinkIdSystem`; +- explicitly selecting it stamps the module into the manifest; +- the manifest stamps the exact selected User ID module list; +- a generated-real-bundle case that denies only Purpose 1 while granting + Purposes 3/4 and vendor 97 produces no LiveRamp request and no `idl_env`; +- a separate case that denies only vendor 97 while granting Purposes 1/3/4 + produces no LiveRamp request and no `idl_env`; +- separate cases that deny only Purpose 3 or only Purpose 4 while granting + Purpose 1 and vendor 97 still produce one LiveRamp request and write + `idl_env` under pinned Prebid's default rules. +- a generated-real-bundle case proves a partial `userSync` update retains the + publisher entry and exactly one managed `identityLink` entry. + +### 11.4 CLI bundle consistency tests + +Add focused tests in `crates/trusted-server-cli/src/prebid_bundle.rs` proving: + +- no managed entries preserve existing bundle behavior; +- a known name passes when its resolved module is in the manifest; +- a known name fails when its resolved module is absent; +- multiple managed names are checked; +- `pubCommonId` resolves to `sharedIdSystem`; +- multiple aliases may resolve to the same required module; +- an unknown name fails with an actionable registry error; +- an ambiguously mapped synthetic name fails deterministically; +- malformed `managed_user_ids` containers, entries, and names fail instead of + being skipped; +- unknown, ambiguous, and malformed-name failures occur before generator + invocation and leave hash/SRI metadata unchanged; +- a missing or malformed `userIdModules` manifest field fails; +- a pre-existing manifest is invalidated before generation, so a fake generator + that succeeds without writing a replacement cannot reuse stale metadata; +- omission of `bundle.user_id_modules` works with the generated default preset; +- failed consistency validation does not update hash/SRI config metadata; +- the checked-in registry maps `identityLink` to `identityLinkIdSystem`. + +### 11.5 Rust auction/EC regression tests + +Existing generic EID tests cover most transport behavior. Add or retain a +LiveRamp-named fixture proving that a `liveramp.com` EID: + +- is forwarded as `user.ext.eids` to the Prebid provider; +- merges without duplication against the EC/KV version; +- is removed when consent denies identity forwarding; +- is ingested into the configured `liveramp.com` EC partner namespace on a + later request. + +### 11.6 Managed browser-consent activation + +Extend the generated-artifact test before changing production code. The matrix +must prove: + +- managed User IDs plus a callable `__tcfapi`, with no publisher-side Prebid + consent configuration, activates `gdpr.cmpApi = "iab"` and blocks the + IdentityLink request and storage when Purpose 1 or vendor 97 is denied; +- no managed User IDs results in no automatic consent configuration; +- a missing or non-callable `__tcfapi` results in no automatic consent + configuration; +- an already-effective publisher `gdpr` value is preserved, including an + object and an explicit non-object value; +- existing sibling consent settings are preserved when the minimum is added; +- queued and late publisher GDPR configuration retains precedence; and +- the shim does not re-apply the automatic minimum after publisher changes. + +### 11.7 Live configuration validation + +Run outside CI against a LiveRamp-approved non-production origin: + +1. Obtain a test Placement ID and confirm the origin is approved. +2. Generate a Prebid bundle containing `identityLinkIdSystem`. +3. Configure a managed `identityLink` entry with the test Placement ID. +4. Load the publisher page with positive consent. +5. Confirm `idl_env` is created or refreshed according to the selected storage. +6. Confirm `pbjs.getUserIdsAsEids()` returns a `liveramp.com` entry without + recording its value. +7. Inspect a controlled Prebid Server request and confirm the same source is + present in `user.ext.eids`. +8. Confirm a later request can ingest the EID into the configured EC partner. +9. Repeat with opt-out/no-consent and confirm no LiveRamp EID is forwarded. +10. Repeat with an unapproved origin and document the expected degraded result. + +Record only booleans, source names, counts, and status codes. Do not capture or +publish live envelopes. + +Sanitized browser validation completed on the approved publisher origin: + +- an unresolved browser identity returned HTTP 204 and exposed no LiveRamp EID; +- a resolvable test identity returned HTTP 200, stored an envelope, and exposed + one `liveramp.com` EID; and +- automated generated-bundle coverage proves denied Purpose 1 or vendor 97 + consent suppresses the IdentityLink request and browser storage without + publisher-side Prebid consent configuration. + +The full live-validation acceptance criterion remains pending. A controlled +environment must still confirm live denied-consent behavior, unapproved-origin +degradation, the resulting `user.ext.eids` on the Prebid Server request, and +later EC/KV ingestion. These checks require publisher and LiveRamp test +conditions and are not replaced by the automated artifact suite. + +## 12. Documentation changes + +Implementation updates: + +- `trusted-server.example.toml` with a commented managed User ID example; +- `docs/guide/integrations/prebid.md` with configuration, lifecycle, bundle, + consent, troubleshooting, and verification guidance; +- `docs/guide/configuration.md` with the vendor-neutral managed field reference; +- optionally a short `docs/guide/integrations/liveramp.md` page if the Prebid + guide would become difficult to navigate. The first implementation should + avoid duplicating the authoritative Prebid flow across two pages. + +The documentation must state that: + +- RampID envelopes, not audience segments, are forwarded as EIDs; +- a Placement ID and LiveRamp-approved origin are operational prerequisites; +- the first auction may not contain a newly resolved identity; +- a module included in a bundle is inert until configured; +- ATS Direct segments require separate enablement and implementation. + +## 13. Rollout and observability + +1. Land configuration and tests with `managed_user_ids` empty by default. +2. Generate and publish a test bundle that includes `identityLinkIdSystem`. +3. Validate on a non-production approved origin with debug logging restricted + to source names/counts. +4. Enable for a canary publisher property. +5. Monitor missing-module diagnostics, LiveRamp EID presence counts, auction + error rates, and cookie/header size truncation counts. Never dimension + metrics by envelope value. +6. Validate opt-out behavior before broader rollout. +7. Document the tested Placement/origin configuration in operator-owned, + non-repository deployment records. + +No database or KV migration is required. Removing the managed entries provides +an immediate configuration rollback. + +## 14. Acceptance criteria + +Issue #355's implementation portion is complete when: + +- operators can configure LiveRamp RampID through vendor-neutral Trusted Server + config; +- invalid configuration fails before serving traffic; +- managed configuration preserves non-LiveRamp publisher User ID modules and + owns one deterministic `identityLink` entry; +- `ts prebid bundle` rejects unknown or ambiguous managed names and a generated + manifest that omits `identityLinkIdSystem` for `identityLink`; +- runtime bundle diagnostics retain the same missing-module defense for + externally supplied or stale artifacts; +- valid `liveramp.com` EIDs follow the existing browser → `/auction` → Prebid + Server path without exposing envelope contents; +- existing consent, validation, merge, cookie, and EC/KV behavior is preserved; +- automated Rust and TypeScript tests pass; +- a generated-bundle test proves that a denied TCF signal blocks managed + IdentityLink network access and storage without a publisher-side Prebid + consent configuration; +- operator documentation explains setup, timing, privacy, failure behavior, + and live verification; +- live configuration validation is completed and recorded without Placement ID + or envelope values; and +- the parent epic receives the explicit answer: RampID identity envelopes can + be passed through the Prebid auction path; ATS Direct segments are not passed + by this implementation. + +## 15. Out of scope and follow-up work + +### 15.1 Server-to-server ATS resolution + +Create or reopen a dedicated issue only after product approval. Its design must +define the hashed-identifier source, origin approval, consent mapping, +`X-Forwarded-For` handling, timeout/cache/refresh policy, geographic failure +behavior, data deletion, and credential storage. It must also reconcile the +decision that closed #630 as not planned. + +### 15.2 ATS Direct audience segments + +Create a separate issue if publishers require LiveRamp segment activation. It +must define: + +- ATS Direct subscription and approved-deal prerequisites; +- whether the integration calls the API or consumes existing browser storage; +- `_lr_atsDirect` and TTL ownership; +- refresh behavior and regional TTL rules; +- whether activation targets GAM (`atsd`), Prebid first-party data, a Prebid + real-time-data module, or more than one destination; +- consent and deletion behavior; and +- the exact evidence needed to confirm segment delivery. + +### 15.3 RTIS callback + +Do not add an RTIS callback unless a concrete non-Prebid use case demonstrates +that the browser module is insufficient and LiveRamp approves the endpoint +contract. + +## 16. Authoritative references + +- [LiveRamp: Implementing the Real-Time Identity Service Tag](https://docs.liveramp.com/identity/en/implementing-liveramp-s-real-time-identity-service-tag.html) +- [LiveRamp: Call the ATS Envelope API](https://developers.liveramp.com/authenticatedtraffic-api/docs/4-call-the-ats-envelope-api) +- [LiveRamp: Retrieving Envelope Endpoints](https://developers.liveramp.com/authenticatedtraffic-api/v1.0/docs/about-the-ats-api) +- [LiveRamp: ATS Direct](https://developers.liveramp.com/authenticatedtraffic-api/docs/implement-ats-direct-via-api) +- [Prebid: LiveRamp RampID User ID module](https://docs.prebid.org/dev-docs/modules/userid-submodules/ramp.html) +- [Prebid: User ID module](https://docs.prebid.org/dev-docs/modules/userId.html) diff --git a/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md new file mode 100644 index 000000000..8962f4862 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md @@ -0,0 +1,98 @@ +# PR 823 Round-5 Review Resolution + +## Goal + +Resolve review `4989897698` on PR 823 without weakening the generator's safety +rules, silently changing existing CLI defaults, or expanding the change beyond +the audit CLI and its documentation. + +## Browser and CLI Compatibility + +The hidden `ts audit ` compatibility form keeps accepting the same browser +flags as `ts audit generate `, but those flags must remain hidden and must +require the legacy URL positional. A dedicated `LegacyBrowserOpts` mirrors the +seven generation browser fields and converts into `GenerateBrowserOpts` when the +legacy command is dispatched. Consequently, flags placed before a real audit +subcommand are rejected instead of parsed and ignored. + +Generation retains its established 750 ms quiet period and 12-second maximum +settle wait. Generation defaults have one source of truth shared by clap, +`GenerateBrowserOpts::default`, and `BrowserAuditCollector::default`; applying +parsed options must not silently shorten the collector's maximum. The generic +page/verification collector keeps its existing independent 10-second default. + +Redirect notes show the origin and path for both requested and final URLs. This +makes scheme and host changes visible without exposing URL userinfo, queries, or +fragments. + +## Root-Less Template Safety + +Template inference records which slot stems borrowed the config-level +`section_root` because those slots were never witnessed on a path without the +configured section segment. Such a template is safe only while its page patterns +are derived from the paths where the slot was observed. + +Operator-supplied `--page-pattern` values replace those derived patterns for +every slot. If inference contains any borrowed-root slot and explicit patterns +were supplied, generation fails before rendering or writing a candidate config. +The error identifies the affected slots, explains that explicit patterns cannot +prove the borrowed-root invariant, and directs the operator to remove +`--page-pattern`. Failing the command is preferable to silently omitting real +inventory or attempting an unsound glob intersection. + +When no config-level section policy can be inferred because every otherwise +templatable slot lacks a root witness, each affected slot's refusal reason names +that crawl gap rather than claiming that its paths failed to generalize. + +## Merge Policy + +An explicitly configured `section_segment` is operator intent even when +`section_root` is currently unset. If preserved `{section}` slots exist and an +inferred policy would change that configured segment, merge fails and requires +`--replace` for the migration. If the configured segment matches, or is unset, +the inferred `section_root` may be adopted so the previously incomplete config +becomes loadable. + +## Diagnostics and Early Validation + +Warnings produced while folding a collected page include the device-profile +label as well as the path. Identical warnings from desktop and mobile therefore +remain distinguishable. The consent-stub warning remains a single unscoped +run-level note, and site-wide discovery warnings remain deduplicated. + +The existing config is parsed as TOML before Chrome starts. A whole-document +syntax error is returned immediately; a valid document with settings unknown to +the CLI still permits extraction of `[creative_opportunities]`; and a present +but unreadable creative section remains an error. + +The volatile div-id token recognizer requires at least ten leading digits plus +an alphanumeric suffix. This continues to recognize timestamp-like generated +tokens while preventing an eight-digit calendar date followed by a stable +letter from causing a single-observation family refusal. + +## Consistency Corrections + +Tests pin the Rust evidence cap to the embedded JavaScript collector constant. +The terminal-escaping test claims only controls it can actually inject; URL's +own percent-encoding is covered by an exact final-URL assertion rather than +presented as evidence for terminal escaping. Existing code escaping the final +URL remains as defense in depth. + +The affected guide, prior volatile-collision spec and plan, documentation +comments, `expect` message, and method spacing are corrected to describe the +implemented behavior exactly. The root-less templating behavior and this review +resolution are documented by this design and its paired implementation plan. + +## Testing and Delivery + +Every behavioral correction starts with a focused regression test that fails on +the current branch. Tests cover hidden legacy flags, the 12-second generation +default, complete redirect notes, borrowed-root rejection with explicit +patterns, configured-segment preservation, profile-specific warnings, +whole-document TOML failure, the evidence-cap invariant, and the calendar-date +token control. + +After focused tests pass, verification runs the host-target CLI suite and +audit/generate tests, CLI clippy with warnings denied, Rust formatting, docs +formatting, and `git diff --check`. No GitHub replies or push are part of this +change unless separately requested. diff --git a/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md b/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md new file mode 100644 index 000000000..9290e6e5c --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md @@ -0,0 +1,118 @@ +# Ad-template div-ID reconciliation design + +## Goal + +Prevent `ts audit ad-templates generate` from losing numeric sibling creative +opportunities during merge or persisting a singleton div ID whose middle token +is demonstrably per-render. + +This follows a live validation crawl. The crawl observed +`ad-sidebar-1`, `ad-sidebar-10`, and other siblings, but the merge treated the +configured literal `ad-sidebar-1` as a prefix and absorbed the longer IDs. It +also proposed one `vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1`-shaped slot +because the volatile-token classifier recognizes ten leading digits but this +token has eight. + +## Scope + +The change is limited to div-ID identity and volatility classification during +generation: + +- Preserve every distinct normalized, usable div identity retained by the + evidence table when an existing configured div ID was itself observed + exactly. +- Preserve intentional configured prefix behavior when that prefix was not + observed as a literal element ID. +- Refuse singleton IDs with a conservative eight-digit-plus-long-suffix token + shape in a non-trailing segment. +- Keep the existing warning and refusal behavior for ambiguous and fragmented + placements. + +This does not implement the broader cross-page-type preservation requested by +GitHub issue #1059, change crawl planning, or change runtime slot resolution. + +## Exact versus prefix reconciliation + +The generator already carries the div identities from `EvidenceTable::slots()` +into the TOML merge. This is intentionally not collector-level raw DOM input: +the identities have passed per-page normalization and usability checks, while +slots later rejected by template inference or cross-page fragmentation remain +present. Page-local volatile and ambiguous identities already refused by GPT +discovery do not re-enter reconciliation. + +The merge will classify a configured or newly appended slot as an observed +literal when its resolved div identity appears exactly in that normalized +evidence set. + +Matching proceeds in this order: + +1. Prefer an exact stable-key match. +2. Otherwise consider configured-prefix matches whose prefix was not observed + as a literal normalized div identity during this crawl. +3. Choose the longest remaining prefix, retaining configuration order for + equal-length ties. +4. Append the discovered slot when neither exact nor eligible prefix matching + succeeds. + +Consequently, `ad-sidebar-1` matches itself but cannot claim +`ad-sidebar-10`. A hand-authored broad prefix such as `ad-`, absent as a literal +DOM ID, retains its existing merge behavior. Newly appended discovered slots +are also protected because the decision is based on the normalized evidence +set, not only the original configuration indexes. + +The same reconciliation rules will drive observed/unobserved diagnostics so a +slot cannot be merged one way and classified for staleness another way. + +## Volatile token classification + +The existing vendor-neutral classifier refuses a div ID when a non-trailing +segment contains a per-render token before the placement suffix. It currently +recognizes a segment with at least ten leading digits followed by alphanumerics. + +Retain that rule and add a narrower alternative for shorter counters: + +- at least eight leading ASCII digits; and +- at least eight trailing ASCII alphanumeric characters in the same segment. + +The token must still occur before another div-ID segment. This catches the +`12345678AbCdEfGhIjKl` shape without claiming: + +- bare numeric placement IDs; +- seven-digit counters with long suffixes; +- eight-digit values with fewer than eight trailing characters, including + calendar-like `20260820a`; or +- trailing tokens whose preceding prefix can still identify the element. + +The warning remains vendor-neutral and names the stable family prefix. The slot +continues to count as evidence of an ad stack but is not rendered into config. + +## Diagnostics and failure behavior + +No new command failure is introduced. Unsafe singleton volatile slots are +skipped with the existing volatile-family note. Literal numeric siblings are +written separately and no longer produce the broad-prefix collision note. +Truly intentional broad prefixes can still produce that note when they claim +multiple observed divs. + +Normal merge continues to preserve configured slots. `--replace` retains its +existing replacement semantics. + +## Testing + +Use test-driven development with focused regressions: + +- A merge containing configured `ad-sidebar-1` and normalized observations for + `ad-sidebar-1`, `ad-sidebar-10`, and `ad-sidebar-11` must produce three slots. +- A configured `ad-` prefix that was not observed literally must continue to + merge multiple matching discovered divs and emit its collision note. +- Newly appended observed literals must not absorb later numeric siblings. +- A framework-bearing DOM ID normalized to a stable stem must classify the + matching configured stem as literal; identities refused during per-page GPT + discovery must not be reintroduced solely for merge classification. +- Registry and request evidence containing a singleton shorter high-entropy token + must be refused with the volatile-family warning. +- Boundary tests cover seven leading digits, eight digits with a seven-character + suffix, eight digits with an eight-character suffix, bare digits, and the + existing calendar-shaped example. +- Run the complete CLI suite, including the real-Chrome scrolling fixture, plus + formatting and the repository's target-specific verification gates. diff --git a/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md b/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md new file mode 100644 index 000000000..8709f6330 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md @@ -0,0 +1,98 @@ +# Ad-template generation scroll and staleness diagnostics design + +## Problem + +`ts audit ad-templates generate` currently collects each page only after its +initial settle. Unlike `ts audit page` and `ts audit ad-templates verify`, it +cannot request the deterministic scroll pass that triggers lazy ad inventory. +On a lazy-loading publisher site this produced fewer observable frames than a +scrolled page audit of the same page. + +Generation also merges by default, deliberately preserving configured slots +that the current crawl did not rediscover. That safety behavior is correct, but +it is silent: stale slots look as though the latest crawl confirmed them. + +## Scope + +Add opt-in scrolling to `ts audit ad-templates generate` and report configured +slots that a merge preserved without observing during the current crawl. + +This change does not prune slots automatically, enable scrolling by default, +alter crawl planning or budgets, change volatile-div refusal, or implement +GitHub issue #1059. `--replace` remains the only intentional pruning mode. + +## Command behavior + +`ts audit ad-templates generate` accepts a boolean `--scroll` option. Its +default is false, preserving current crawl cost and side effects. When enabled, +every page on every selected device profile performs the same deterministic +stepped scroll used by the existing page audit: scroll to 33%, 66%, and 100% of +the document, pause between steps, return to the top, then wait for the page to +settle again before reading HTML, GPT registry entries, and network evidence. + +The browser collector carries the option as session configuration so root, +planned section, desktop, and mobile page loads all behave consistently. Scroll +evaluation failures are best-effort page warnings; they do not discard evidence +that was already available after the initial settle. + +The implementation will share the deterministic scroll primitive with the +existing browser audit rather than maintain a second sequence of scroll steps. +Verifier-only evidence-phase bookkeeping remains in the verifier call path. + +## Merge diagnostics + +During a normal merge, generation tracks which pre-existing configured slots +matched at least one discovered slot. After processing all discovered slots, it +reports every unmatched pre-existing slot in configuration order. Those slots +remain unchanged in the output. + +The diagnostic is explicit about the limits of negative crawl evidence. Its +human-readable form for a non-scrolling run is equivalent to: + +```text +note: preserved 2 configured slot(s) not observed during this crawl: ad-header-0, ad-fixed_bottom-0. Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover. +``` + +When the current run already used `--scroll`, the follow-up omits that redundant +suggestion and recommends broader page/profile coverage before intentional +pruning. + +No staleness diagnostic is emitted when all configured slots were rediscovered, +when there were no existing slots, or under `--replace`, because that mode does +not preserve unmatched slots. Matching uses the same reconciliation logic as +the merge itself, avoiding a second definition of slot identity. + +Diagnostics go to stderr through the existing generation-note path. Stdout +remains limited to the dry-run diff or successful write summary, so redirection +and machine comparison remain stable. + +## Safety and compatibility + +The default command behavior, merge result, and generated TOML remain unchanged +unless `--scroll` discovers additional evidence. The warning never mutates or +deletes operator configuration. It names only configured slot IDs and does not +include cookies, URL credentials, query strings, or fragments. + +Scrolling can trigger additional ad requests and publisher behavior, which is +why it remains explicit. Existing page-delay, settle-window, browser-proxy, +certificate, cookie, and device-profile behavior applies unchanged. + +## Tests + +CLI parsing tests cover `--scroll` and its false default. Browser-collector tests +use a deterministic local page that defines a GPT slot only after scrolling and +prove that generation captures it with the option enabled but not without it. +Existing browser lifecycle and settle tests continue to cover teardown and +timeouts. + +Merge unit tests cover multiple unmatched configured slots, stable diagnostic +ordering, partial rediscovery, full rediscovery, an empty existing config, and +`--replace`. Command-level tests verify that the warning reaches stderr while +stdout and the preserved generated configuration retain their existing +contracts. + +Verification will run the host CLI test suite and relevant Chrome-backed CLI +tests, followed by the repository-required formatting and CLI lint gates. A +manual dry run against a live publisher site may be used when a fresh +bot-protection cookie and proxy are available, but network-dependent behavior +is not a required CI test. diff --git a/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md b/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md new file mode 100644 index 000000000..0cd12f02c --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md @@ -0,0 +1,682 @@ +# Request phase timing: Server-Timing subtimings and access telemetry + +**Date:** 2026-08-24 +**Status:** Approved design, revised for review rounds 1 and 2, pending implementation +plan. +**Scope:** `trusted-server-core`, Fastly and Axum adapters, `tinybird/` schema, +performance dashboard (separate repo). + +--- + +## 1. Problem + +On 2026-08-21 a production deployment (publisher redacted, `prospect-a.example`) showed +an episodic stall: for a window of roughly 40 minutes, every request that reached the +application path carried a uniform extra ~600 ms of Fastly `time-elapsed`, and then +recovered to 20-50 ms with no deploy or config change we could observe. `/health` +(2-4 ms, short-circuits before app construction) and `/_ts/debug/ja4` (6-9 ms, settings +load only) stayed fast throughout, so the stall lived between app construction and +response send. + +Attributing that window required a live probing session: route-by-route bisection, +cookie-deletion experiments, and an eight-agent code trace. The trace found no +unconditional await on the path that could cost 570 ms, and exactly two +config-conditional candidates (the pre-route request filter's synchronous verification +POST, and EC identity KV writes before send), plus one dependency shared by every +application route (two geo hostcalls per request). We could not tell which one stalled, +because nothing in the response says where server time went. + +The Compute CPU budget is ~50 ms per request, so a large `time-elapsed` strongly +suggests wall-clock time outside active guest CPU: dependency awaits are the leading +explanation, with platform scheduling and hostcall queueing as the residual ones. The +comparison figure here is the fronting delivery layer's `time-elapsed` Server-Timing +entry, observed at its deliver phase. Either way, these are exactly the numbers a +response can carry about itself. + +## 2. Goals + +1. Every normal application response attributes its own server time by phase in a + standard header. Browsers expose the values to same-origin JavaScript via + `PerformanceResourceTiming.serverTiming`, so RUM tooling that reads that API can + surface the breakdown. Whether a given vendor or the publisher's own monitoring + extension actually collects it is verified separately in rollout; the publisher + extension needs a small change to render it. +2. The same numbers flow to Tinybird so we hold p50/p95/p99 per phase, per route class, + per PoP, per deployed version, and a future stall window self-diagnoses in one query. +3. No additional awaited I/O before first byte. The pre-send cost is a handful of + monotonic clock reads, one small allocation at entry, and rendering one header; + telemetry emission happens strictly after the last body byte. + +Scope note: phases cover the application lifecycle after T0. The `/health` and +`/_ts/debug/ja4` short-circuits, config-store open failures, and request-conversion +failures bypass the lifecycle and emit nothing. Requests served entirely by the +fronting cache never reach the guest and produce neither header entries nor rows. + +## 3. Non-goals + +- No trailer-based Server-Timing for body-phase spans (browsers do not expose trailer + values to JavaScript). +- No per-filter naming in any emitted surface. The request-filter span is `ts-filter` + regardless of which filter runs; vendor identity stays out of headers and telemetry. +- No Cloudflare or Spin emission wiring in v1. Core collection is adapter-neutral; those + adapters can wire emission later without core changes. +- No Tinybird endpoint pipe and no rollup materialized views in v1. Grafana queries the + datasource through the ClickHouse connector, matching the auction dashboards; rollups + only if panel latency demands them. +- No sampling of the header. The header is all-traffic when enabled; only Tinybird rows + sample. +- No cross-request circuit breaker for telemetry emission. Compute runs one isolate per + request; there is no shared mutable state to hold breaker state. The controls are the + bounded per-request cost and the `access_sample_rate` lever (section 10). + +## 4. Design overview + +``` +adapter entry (T0) + | RequestTimings::new() -> shared handle + v +app construction ................ ts-appbuild (adapter) +pre-route request filters ....... ts-filter (adapter wrapper) +geo lookup (single, deduped) .... ts-geo (adapter; result carried forward) +template cache lookup ........... ts-template-cache (core: publisher.rs) +origin fetch to resp headers .... ts-origin (core: publisher.rs) +EC identity KV, pre-send ........ ts-kv (core: KV abstraction) +auction wait, buffered mode ..... auction_wait_ms (row only; pre-header in this mode) + | +send_edgezero_response, immediately before into_parts(): + mark_headers_ready() snapshot (unconditional) + build AccessTelemetrySnapshot (only when tinybird.access_enabled) + append Server-Timing header (flag-gated, only on conclusively private responses) + | +headers committed; body streams + auction hold at seam .......... auction_wait_ms (row only; in-stream in this mode) + stream duration, bytes ........ stream_ms, resp_bytes (row only) + | +post-send (adapter main): + request_elapsed snapshot, then existing pull-sync, then: + sample gate -> one NDJSON row -> Tinybird Events API + bounded response await, 2xx validated +``` + +Collection is always-on and flag-free, including the `mark_headers_ready()` snapshot. +Two independent flags gate emission: the header (`observability.server_timing_enabled`) +and the telemetry row (`tinybird.access_enabled`). + +## 5. `RequestTimings` (core) + +New module `crates/trusted-server-core/src/request_timing.rs`. + +- `Phase`: a closed enum: `AppBuild`, `Filter`, `Geo`, `EcKv`, `Origin`, + `TemplateCacheLookup`, `AuctionWait`, `Stream`. Header rendering covers the first six + plus the stored total; the last two are row-only. +- Inner state: one fixed-size array of `Option` slots indexed by phase, + `t0: Instant`, `headers_ready_total: Option`, + `auction_wait_placement: Option` (`PreHeader` or `InStream`), + and `resp_bytes: Option`. Phases that repeat within a request (geo, KV) + accumulate by saturating addition into the same slot. +- `mark_headers_ready()`: stores `t0.elapsed()` once at the response-commit boundary, + unconditionally, before either emission flag is consulted. The header renders this + stored value as `ts-total`; the telemetry row reads the same stored value as + `time_elapsed_ms`. The two surfaces cannot disagree, and the row stays correct when + the header flag is off. Full request duration is captured separately as + `request_elapsed_ms`, snapshotted immediately after the body-stream drive returns and + before any other post-send work, so pull-sync and telemetry emission are never + included in it. +- Sharing: `RequestTimings` is a cheap-clone handle, `Arc>`. It crosses + three boundaries: adapter entry to core handlers, the streaming body closure (records + body-phase spans after the response object has been handed off), and the adapter's + post-send emission read. Access is exclusively `try_lock()`: a contended lock + drops the one sample rather than waiting, so recording can never delay a + request, and a poisoned lock is recovered (the guarded data are plain counters + with no invariant a panic can break) so one panic cannot silence the rest of + the request's timing. +- Recording API: `timings.record(Phase::Geo, dur)` and a scope guard + `timings.span(Phase::Origin)` that records on drop. Guards use saturating duration + math; a non-monotonic reading records zero rather than panicking. The auction-wait + recorder takes the placement explicitly so the two modes cannot be conflated. +- Rendering: `server_timing_value(&self) -> Option` produces + `ts-total;dur=41.2, ts-appbuild;dur=18.4, ts-filter;dur=9.1` with durations in + milliseconds at one decimal. Phases never recorded are omitted. Returns `None` when + `mark_headers_ready()` has not run. + +`Instant` is already used freely in the guest (`publisher.rs`, `auction/telemetry.rs`), +so no new clock abstraction is needed. + +## 6. Span taxonomy and recording sites + +| Entry | Measures | Site | +| ------------------- | ------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| `ts-total` | T0 to `mark_headers_ready()` at the response-commit boundary | stored snapshot | +| `ts-appbuild` | config-store open, Settings parse, orchestrator + registry + router build | Fastly `main.rs` around `open_trusted_server_config_store()` + `build_app_with_state()` | +| `ts-filter` | pre-route request filters, end to end (backend ensure, secret read, POST) | around `run_pre_route_filters` (`app.rs:751`) | +| `ts-geo` | geo hostcall (single after dedupe; accumulates if any path still repeats) | `build_ec_request_state` (`app.rs:410`) and timed finalizer fallback lookups | +| `ts-kv` | EC identity KV operations before response send (see enumeration below) | the shared KV abstraction | +| `ts-origin` | publisher backend send to response headers available (read-through cache hit or miss) | `publisher.rs` around the origin `send` | +| `ts-template-cache` | template cache `lookup_or_reserve`, including the hit-path full-body read | `publisher.rs` around the lookup | + +Naming follows the completed template-cache terminology migration (`x-ts-template-cache` +is the emitted header on `main`; `c2` naming is retired). + +`ts-kv` is instrumented by a timing decorator implementing `PlatformKvStore` that +wraps the store handed to request-scoped consumers, because no single existing +abstraction covers the taxonomy: EC graph operations go through `KvIdentityGraph` +while consent persistence uses `PlatformKvStore` directly, and graphs are constructed +independently in request setup, identify, admin lookup, batch sync, and finalization. +Every request-path graph construction receives the timed store; pull-sync explicitly +constructs its graph from an untimed store. Consent-store reads pass through the same +decorator and are timed like any other store call. Included pre-send operations: EC +generation `create_or_revive`, identify-path graph reads and evaluation, finalize-path +`ingest_eid_cookies`/`upsert_partner_ids` and withdrawal tombstones, consent-store +reads on consent routes, and batch-sync graph access when it runs before send. +Explicitly excluded: pull-sync work, which runs strictly after `send_to_client` and is +invisible to both surfaces. The decorator measures store-call latency only: no value +passing through it is read, parsed, or recorded, and the emitted surfaces carry no +consent or identity payloads. This feature therefore needs no consent gate; it is the +site measuring its own infrastructure, not processing user data. The timings handle +reaches `ec_finalize_response` inside the graph it already receives; that function +keeps the repository maximum of seven arguments and does not gain an eighth. + +Row-only fields: + +| Field | Measures | Site | +| -------------------- | --------------------------------------------------------- | ------------------------------------------------ | +| `auction_wait_ms` | wait on the dispatched auction (placement varies by mode) | seam hold (streaming) or buffered finalizer wait | +| `body_mode` | `streamed` or `buffered` response assembly | set where the response body is built | +| `stream_ms` | headers committed to last body byte | adapter around the body-stream drive | +| `resp_bytes` | bytes written to the client body | same | +| `request_elapsed_ms` | T0 to immediately after the body-stream drive returns | post-send snapshot, before pull-sync | + +Auction-wait placement is not universal. On the ordinary streaming path the wait +happens at the `` seam inside the body stream and nests inside `stream_ms`. On +buffered paths (the Fastly shared-template authorized miss, which buffers the full +transform and auction before returning a response, and every Axum response) the wait +completes before headers commit. The row therefore carries `body_mode` plus +`auction_wait_placement` (`pre_header` or `in_stream`), and derivations are +conditional: + +- `in_stream`: `stream_other_ms = greatest(coalesce(stream_ms, 0) - coalesce(auction_wait_ms, 0), 0)`. +- `pre_header`: `auction_wait_ms` joins the pre-header phase set, and `stream_other_ms = coalesce(stream_ms, 0)`. + +`unattributed_ms = greatest(coalesce(time_elapsed_ms, 0) - (coalesce(appbuild_ms, 0) + +coalesce(filter_ms, 0) + coalesce(geo_ms, 0) + coalesce(kv_ms, 0) + +coalesce(origin_ms, 0) + coalesce(template_cache_ms, 0) + pre-header auction wait), 0)`. +Every phase column is nullable, so every query-time formula wraps each term in +`coalesce(column, 0)` and every subtraction in `greatest(..., 0)`; query tests cover +sparse phase combinations. + +## 7. Freeze point and header emission + +The freeze-and-emit point is `send_edgezero_response` (Fastly `main.rs`), immediately +before `response.into_parts()`. This is the single choke point every send path shares, +and it runs after everything that can still mutate the response: the router middleware, +entry-point finalize (`apply_finalize_headers`, asset-policy reapplication), EC +finalization and its KV work, and terminal filter/privacy effects. +`apply_finalize_headers` itself does not emit; the `HEADER_X_TS_FINALIZED` sentinel +marks middleware finalization, not header commitment, and must not be treated as the +timing boundary. + +At the freeze point, in order: `mark_headers_ready()` (unconditional), the +`AccessTelemetrySnapshot` build (gated on `tinybird.access_enabled`, +section 10), then, gated on +`observability.server_timing_enabled`, append one `Server-Timing` header from +`server_timing_value()`. Append semantics, never insert: an origin-supplied +Server-Timing survives, and the fronting delivery layer's own entries (`time-elapsed`, +`hit-state`) are additive per the header's list semantics. + +Header emission is conservative: it happens only when the response is conclusively +non-storable by any shared cache, meaning `Cache-Control` contains `private` or +`no-store` (the existing `cache_control_headers_are_private_or_no_store` predicate). +Anything else, including bare `max-age`, `s-maxage` without `private`, +heuristically-cacheable responses with no cache header at all, and anything a fronting +cache override might store, emits no header, because a stored object would replay one +request's timings for its full lifetime. The long-lived immutable `tsjs` asset route +is the concrete excluded case. The snapshot and the telemetry row are unaffected by +this skip, so excluded routes still report through Tinybird. + +The Axum adapter applies the same emission rule at its terminal point before response +serialization, with adapter-specific phase semantics (section 8a). + +## 8. Geo lookup dedupe (rider) + +Today every dispatched request pays two geo hostcalls for one answer: request-phase in +`build_ec_request_state` (`app.rs:410`) and response-phase in +`FinalizeResponseMiddleware` (`middleware.rs:83`, alternate site `main.rs:285`). + +Plain request extensions cannot carry the result out: the middleware moves the request +context into `next.run(ctx)` and holds only the response afterward. The resolved geo +travels on a dedicated `GeoLookupState` response extension, attached on every exit +path that attempted a lookup, including the asset fallback, which runs +`build_ec_request_state` and then returns without `EcFinalizeState` (which is why +`EcFinalizeState` is not an acceptable carrier). States: `NotAttempted`, +`Attempted(None)` (lookup ran and failed, do not retry), and `Resolved(GeoInfo)`. The +finalize path consumes the carried value and performs a live lookup only in the +`NotAttempted` state; those legitimate fallback lookups (admin, batch, error paths) +are themselves timed into `ts-geo` so degraded geo cannot hide inside +`unattributed_ms`. The 401 rule (`resolve_geo_for_response` skips lookup for +unauthorized responses) is preserved. + +## 8a. Adapter phase semantics + +The Fastly adapter is the reference implementation of the taxonomy. Axum differs +structurally and its emissions are defined accordingly rather than pretending parity: + +- `ts-appbuild` is absent: Axum builds application state once at startup. +- `body_mode` is always `buffered`: the Axum HTTP client buffers upstream bodies, so + `stream_ms` measures buffered-body write-out and `auction_wait_placement` is always + `pre_header`. +- The freeze point is an outer service wrapper around the `RouterService` inside + `AxumDevServer`, not router middleware: router-generated 404/405 responses bypass + router middleware, and middleware returns before Axum serializes the body. The + wrapper sees every response including router-generated ones; `/health` is excluded + by path match inside the wrapper. +- Axum emits the header only; no Tinybird rows in v1 (unchanged). + +Cloudflare and Spin: collection compiles, no emission wiring in v1 (unchanged). + +## 9. Access telemetry row + +Extends the reserved `tinybird/datasources/access_logs_raw.datasource`. + +Kept columns: `event_ts`, `method`, `status`, `time_elapsed_ms` (defined as the +`mark_headers_ready()` snapshot; nullable because a contended lock drop can lose the +snapshot), `sample_rate`, 30-day TTL. (`event_date` was later dropped for the +`toDate(event_ts)` sorting-key expression; see section 9's schema note.) + +Removed: raw `path`. Route identifiers like `/_ts/admin/ec/{id}` would otherwise put +EC identifiers into a 30-day dataset, and publisher paths carry unbounded cardinality +and user-generated content (search terms, usernames, emails in slugs). Replaced by +`route_template`: + +- Named routes: the matched route-table pattern verbatim, parameters left as + placeholders. +- Publisher fallback: a coarse fixed template, `/` plus the first path segment + restricted to a bounded allowlisted charset, plus `/*` when deeper (for example + `/news/*`). The auction-telemetry normalizer is explicitly not sufficient here: it + redacts long tokens but preserves short identifiers and arbitrary slugs. +- Publisher templates come from an operator-configured allowlist of section names + (`observability.route_sections`, default empty), not from the request: a path + whose first segment matches an allowlist entry and that has at least one further + segment emits `/{section}/*` (the lowercased allowlist entry itself); everything + else emits `/other/*`, and the root path emits `/`. This replaces the earlier + shape heuristics (charset, length, digit bounds), which review showed cannot + bound identity: depth-2 first segments are usernames on `/{username}/posts` + shapes, and single-segment paths are documents under `/%postname%/` permalinks. + With the allowlist, the emitted value set is fixed by configuration, so no + request-derived byte ever reaches the row. The character allowlist alone does not bound identity (`[a-z0-9_-]` + is exactly the alphabet of UUIDs, hex ids, and reset tokens), and a truncated + prefix of any of those is still identifying, so the length and digit bounds reject + the segment outright. +- Tests are adversarial, not just the happy path: a literal EC identifier on the admin + route, an email address in a path segment, search-term-shaped segments, overlong + segments, UUIDs, hex ids, reset tokens, and full article slugs must all normalize + to bounded, content-free templates. + +Added columns (all dimension columns non-nullable with an `unknown` sentinel, because +ClickHouse sorting keys cannot contain nullable columns): + +``` +`service_id` LowCardinality(String), -- FASTLY_SERVICE_ID; immutable deployment identity +`publisher_domain` LowCardinality(String), -- matches auction schema +`env` LowCardinality(String), -- adapter-derived: production | staging | unknown +`route_class` LowCardinality(String), -- publisher_html | tsjs | integration_proxy | ec | auction_api | other +`route_template` String, -- bounded, normalized; replaces path +`body_mode` LowCardinality(String), -- streamed | buffered +`auction_wait_placement` LowCardinality(String), -- pre_header | in_stream | none +`appbuild_ms` Nullable(UInt32), +`filter_ms` Nullable(UInt32), +`geo_ms` Nullable(UInt32), +`kv_ms` Nullable(UInt32), +`origin_ms` Nullable(UInt32), +`template_cache_ms` Nullable(UInt32), +`auction_wait_ms` Nullable(UInt32), +`stream_ms` Nullable(UInt32), +`request_elapsed_ms` Nullable(UInt32), +`resp_bytes` Nullable(UInt64), +`template_cache_state` LowCardinality(String), -- from the typed response extension, not the public header +`country` LowCardinality(String), +`ts_version` LowCardinality(String), +`pop` LowCardinality(String) -- FASTLY_POP, 'unknown' when absent +``` + +The matched route pattern does not survive dispatch today, so a typed +`RouteMetadata` response extension carries `route_class` and `route_template`: each +named-route handler wrapper attaches its route-table pattern verbatim (handlers +serving multiple patterns attach the one that matched), and the fallback and tsjs +handlers attach their class plus the coarse template. The freeze point consumes the +extension; nothing reconstructs routes from a handler enum or path regex. + +Typed sources only: `env` is adapter-owned, derived from the same Fastly +`FASTLY_IS_STAGING` input that drives `x-ts-env` (`Settings` has no environment +field and does not gain one). `template_cache_state` comes from a typed response +extension, not the `x-ts-template-cache` header (operator-configured response +headers can override managed headers): the currently private +`TemplateCacheResponseState` in `publisher.rs` becomes a typed response extension, +and every state transition sets the managed header and the extension together so +the two can never drift. `service_id` and `pop` come from the Fastly environment. `cache_state` +from the reserved schema is dropped: the guest cannot observe the fronting cache, and +guest-visible cache behavior is already carried by `template_cache_state` and +`origin_ms`. Rows exist only for guest-handled requests; fronting-cache hits are +invisible by construction and the dashboard documentation says so. + +Sorting key: `(toDate(event_ts), service_id, publisher_domain, env, route_class, +pop, status)`. Every column carries a `json:$.` path (the Events API rejects +NDJSON into a datasource without JSONPaths, discovered live); `event_date` was +dropped in favor of the sorting-key expression because a DEFAULT column cannot +carry a JSONPath the producer never sends. Grafana time filtering uses `$__timeFilter(event_ts)` and every panel query +also carries a `toDate(event_ts)` predicate so the primary index prunes; rollout validates +the panel queries with `EXPLAIN` before the dashboard is committed. This replaces the +reserved key `(event_date, path, status, method)`. Rollout step 4 verifies whether the +reserved datasource was ever deployed to the remote workspace; if it was, this schema +ships as a versioned replacement datasource with a cutover, not an in-place edit. + +## 10. Emission mechanics + +- `AccessTelemetrySnapshot`: built at the freeze point when + `tinybird.access_enabled` is set (revised in review from the original + unconditional build, so a disabled deployment pays nothing here), before + `into_parts()` consumes the response. It captures method, status, route metadata + (from the `RouteMetadata` extension), and typed dimension states (`env`, + `template_cache_state`, geo country). It exists because nothing else survives to + post-send on every path: the request is consumed by dispatch, the response by + `into_parts()`, and `EcFinalizeState` is absent on asset, admin, and error paths. +- The emitter's transport context is adapter-owned and route-independent: the + Events API target (backend spec, secret store name, dataset, token secret, sample + rate) derives from settings once at entry in `main.rs`, and the HTTP client is the + adapter's stateless platform client. Asset, admin, and error responses therefore + emit without `RuntimeServices` or `EcFinalizeState`. +- `send_edgezero_response` returns a delivery outcome instead of `()`, with + per-mode semantics because the two body paths observe different things. Streamed + bodies: a counting writer reports bytes written and distinguishes complete, + partial (truncated), and error outcomes. Buffered bodies: `send_to_client()` + returns no delivery result, so the byte count is captured from the body length + before the send and the outcome is complete-on-return with no partial detection; + `body_mode` in the row keeps the two regimes distinguishable in analysis. +- Ordering after the body-stream drive returns: snapshot `request_elapsed_ms` first, + run the existing pull-sync dispatch unchanged, then telemetry emission last, so + pull-sync is never delayed behind the ingest await and never included in + `request_elapsed_ms`. +- Sampling: uniform per-request decision against `tinybird.access_sample_rate`. No + client stickiness. Sampled-out requests are silent; every other drop (row build + failure, send failure, non-2xx) logs one warning naming the reason. There is no + cross-request warning suppression (per-request isolates hold no shared state); the + overload controls are the 2 s bounded await, the single-warning-per-request cap, and + `access_sample_rate` pushed down by config as the operational abort lever. Ingest + health is monitored from the Tinybird side via ingestion freshness on the + datasource, which catches quarantine and schema rejection that per-request warnings + cannot. +- Transport: one NDJSON row to the Tinybird Events API: same `api_host`, reserved + `access_dataset` and `access_token_secret`, 2 s first-byte and between-bytes + timeouts, `max_body_bytes` guard, no retry. +- Delivery confirmation: unlike the auction sink, which starts `send_async` and drops + the pending response (it runs before delivery completes and cannot afford to wait), + the access emitter runs after the client has the full response and therefore awaits + the bounded ingest response and validates 2xx. A non-2xx or timeout logs a warning + with the status. +- Budget: at `access_sample_rate = 1.0` this adds one backend request per request to + the service, after delivery; during a Tinybird outage each such request holds its + sandbox for up to the bounded timeout. The sample rate is the budget control; 1.0 is + a diagnosis setting, not a steady state, and rollout treats sustained emission + warnings as the signal to dial it down. +- Axum adapter: emits the header only; no Tinybird rows in v1. + +## 11. Dashboard and query model + +No endpoint pipe in v1. Grafana queries `access_logs_raw` directly through the +ClickHouse connector with `$__timeFilter(event_ts)` plus a `toDate(event_ts)` +predicate, matching the auction dashboards. + +Dashboard: a new standalone `grafana/dashboards/edge-performance.json` in the +telemetry repo (`trusted-server-tinybird`), performance only, no panels shared with +the revenue and auction dashboards. Panels: + +- Phase percentiles (p50/p95/p99) by `route_class`, per phase column. +- Stacked phase breakdown over time using the non-overlapping set: `appbuild_ms`, + `filter_ms`, `geo_ms`, `kv_ms`, `origin_ms`, `template_cache_ms`, pre-header + auction wait (where `auction_wait_placement = 'pre_header'`), and derived + `unattributed_ms`. In-stream auction wait and derived `stream_other_ms` chart in a + separate body-phase panel and never stack with pre-header phases. +- PoP split, `ts_version` overlay, template-cache state rates. +- Stall panel: rows with `request_elapsed_ms > 500` (post-body total, so body-only + stalls are caught) grouped by dominant phase, where `unattributed_ms` competes as a + phase so the panel cannot confidently blame a small measured span while most time is + uninstrumented. + +All derivations use the `coalesce`/`greatest` forms from section 6; query tests cover +sparse phase combinations and both `auction_wait_placement` modes. + +Sampling semantics for every aggregate: `sample_rate` must be operationally stable +within any queried window. Quantile panels filter strictly to a single `sample_rate` +value. Volume panels weight each row by `1.0 / sample_rate` (the inverse-probability +estimator is `sum(1.0 / sample_rate)` over emitted rows; `count() / rate` is valid +only when the query is already filtered to one rate). Pooled unweighted quantiles +across a rate change are documented as invalid. + +## 12. Config surface + +```toml +[observability] +# Append TS phase timings to the Server-Timing response header. +server_timing_enabled = false # example default +``` + +New `ObservabilitySettings` struct with the single boolean, default off, standard +environment override (`TRUSTED_SERVER__OBSERVABILITY__SERVER_TIMING_ENABLED`). +Collection has no flag: the flags gate the two emission surfaces independently. + +Tinybird flag structure: `tinybird.enabled` today arms the auction sink by itself, so +"enable Tinybird for access telemetry" would silently enable auction emission too. The +master flag is demoted to transport-only (host, store, credentials), and each emitter +gets its own switch: a new `tinybird.auction_enabled` defaulting to `true` (preserving +current behavior for existing configs) and the reserved `tinybird.access_enabled` +defaulting to `false`. A settings test locks the decoupling in both directions. + +Validation when `access_enabled = true`: `tinybird.enabled`, non-empty `api_host`, +non-empty `secret_store`, `access_dataset`, and `access_token_secret`, a positive +`max_body_bytes`, and `access_sample_rate > 0`. An armed-but-silent configuration +(`access_enabled = true`, `access_sample_rate = 0`) is a configuration error, not a +valid state; disabling is done with the flag, not the rate. + +Rollback and compatibility, because `Settings` is `deny_unknown_fields`: + +- Deployment order is binary first, config second. Rollback order is config first + (remove the `[observability]` table and any new tinybird keys), binary second. A + config containing the new fields must never be pushed while a pre-observability + binary can still run. +- Config serialization omits the table when it equals the default, so round-tripping a + config through tooling does not inject a field an older binary rejects. A + compatibility test asserts the serialized default config parses under the previous + schema. +- The environment-variable overlay cannot create a missing leaf, so the key ships + present-but-false in the base operator TOML (the same pattern the GPT integration + documents in `trusted-server.example.toml`) and is flipped by config push. + +## 13. Error handling + +- Recording is infallible: saturating math, lock-failure drops the sample, no panics. +- Header rendering failure (defensive `HeaderValue::from_str` error) logs and skips + the header. +- Row emission failure logs one warning naming the reason and drops the row. The + response has already been delivered; there is nothing to degrade. + +## 14. Testing + +- Core unit tests: phase accumulation, saturating math, `mark_headers_ready()` + idempotence and both-surface consistency, render format (one decimal, omission of + unrecorded phases), row serialization shape, auction-wait placement recording. +- Adapter tests (Fastly via Viceroy, Axum native): header present and well-formed on a + conclusively-private publisher route with the flag on; absent with the flag off; + absent on the shared-cacheable tsjs route and on a bare `max-age` response with the + flag on; exactly one TS-owned metric set (a single `ts-total`) with every + pre-existing Server-Timing value preserved, across all send paths; `ts-kv` captures + EC finalize work (proving the freeze point sits after it). +- Body-mode tests: ordinary streaming (in-stream wait nested in `stream_ms`), + Fastly shared-template authorized miss (buffered, pre-header wait), and Axum + (always buffered), each asserting placement and non-negative derivations. +- Geo dedupe: finalize consumes `Resolved`; no retry on `Attempted(None)`; live + lookup only on `NotAttempted`; fallback lookups timed into `ts-geo`; asset-fallback + path carries `GeoLookupState` without `EcFinalizeState`; 401 skip preserved. +- Route template: adversarial normalization tests (literal EC identifier on the admin + route, email address in a segment, search-term segments, overlong segments) all + producing bounded content-free templates. +- Settings: the access validation matrix including the armed-but-silent rejection; + auction/access flag decoupling in both directions; the former rejection test becomes + the wiring test; the serialized-default-config compatibility test against the + previous schema. +- Sink tests: `RecordingHttpClient` pattern; assert URI, NDJSON body shape, token + header, 2xx validation and warning on non-2xx, skip when sampled out, ordering after + pull-sync. +- Query tests: derivation formulas against sparse rows and both placements. + +## 15. Rollout and verification + +1. Land collection + freeze point + header emission behind the flag, off everywhere. + Full CI gate. +2. Staging deploy with the flag on. Delivery-layer verification is two-sided: a + pass-through request confirming the appended Server-Timing survives the fronting + VCL, and a MISS-then-HIT replay against a cacheable route confirming no stale + timing header is ever served from cache. Fallback if the VCL clobbers the header: a + one-line VCL change on the delivery service, or mirroring the value to + `x-ts-timing` while that lands. +3. Production flag on. Confirm + `performance.getEntriesByType('navigation')[0].serverTiming` shows `ts-*` entries + in a real browser session, and separately confirm what the publisher's RUM tooling + actually collects; the publisher monitoring extension renders it only after a small + change on their side. +4. Verify whether `access_logs_raw` exists in the remote Tinybird workspace. If yes, + ship the schema as a versioned replacement with cutover; if no, edit in place. + Validate the dashboard panel queries with `EXPLAIN` against the sorting key. Then + land the row schema, sink, and settings changes; sample at 1.0 during stall + diagnosis with ingestion-freshness monitoring on the datasource; then the + dashboard. +5. Success criterion: the next stall window is attributable from one response header + or one dashboard query, with no live probing session. + +## 16. Overhead + +Roughly ten monotonic clock reads, two stored snapshots, and one ~130-byte header per +request; one sampled HTTP POST with a bounded await after the response has fully +streamed. No allocation in the hot path beyond the one `Arc` at entry, the +`AccessTelemetrySnapshot` at the freeze point, and the rendered header string. + +## 17. Decisions and open questions + +- **Public exposure is a decision, not an open question.** The header is all-traffic + when enabled. Rationale: values are durations only; the delivery layer already + exposes `hit-state` and `time-elapsed` publicly on every response; filter vendor + identity is masked; emission is restricted to conclusively-private responses so no + cache can replay stale timings. Revisit (quantization or gating) only if a concrete + abuse surfaces. +- The fronting delivery layer's Server-Timing pass-through is unverified until the + first staging deploy (step 2). This is the only known external dependency. +- Body-phase capture threads the timings handle into the streaming closure in + `publisher.rs`; the exact seam is an implementation-plan detail, with the constraint + that a dropped handle (error paths, early client disconnect) must still yield a + valid row with null body-phase fields and a recorded delivery outcome. +- The stall window itself remains unattributed until this ships. If it recurs first, + the bisection runbook from 2026-08-21 (cookie-free curl UA request, static-asset + path versus HTML path) is the fallback. + +## 18. Auction timeline offsets (follow-up increment) + +Status: spec amendment for a follow-up PR; not part of the initial implementation +(#1074). Builds only on machinery that spec sections 5, 9, and 10 already define. + +### Problem + +The pipeline has two clocks that never meet. The auction dataset +(`auction_events_raw`, PR #813) measures the auction internally: `total_time_ms` +from auction start to terminal, `provider_response_time_ms` per bidder call. Its +clock starts when the auction observation is created, so nothing places those +numbers on the request timeline. The access row is T0-anchored but records only +`auction_wait_ms`: time the handler was blocked at collect, deliberately not the +auction's own timeline. + +That leaves three questions unanswerable today: + +1. At what request-relative time did the auction start (dispatch leave the edge)? +2. At what request-relative time did the auction resolve (final bid or timeout)? +3. At what request-relative time were the results committed toward GAM? + +These are the overlap-proof questions. A client-side wrapper cannot dispatch until +the browser boots (t≈3000ms on measured prospect pages); the server-side auction +dispatches while the origin fetch is in flight. Proving that requires all +milestones on one clock. + +### Design + +Three first-call-wins marks on `RequestTimings`, in the style of +`mark_headers_ready()`, each storing `Option` since T0: + +| Mark | Recorded at | Meaning | +| --------------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | +| `mark_auction_dispatched()` | immediately before `orchestrator.dispatch_auction` returns control to the caller (`publisher.rs` dispatch site) | bid requests have left the edge | +| `mark_auction_resolved()` | immediately after `collect_dispatched_auction` returns, both collect sites | final bid returned or auction timed out; terminal either way | +| `mark_auction_committed()` | immediately after `write_bids_to_state` returns, both call sites | winning bids are in page state, available to the response pipeline | + +Notes on the definitions: + +- "Committed toward GAM" is defined as `write_bids_to_state` returning: the common + point in buffered and streaming modes where targeting becomes part of the + response. TS never calls GAM server-side; the browser's GPT call carries the + targeting, and that half of the timeline belongs to client-side measurement. + The edge proves when targeting was available; the client proves when GAM saw it. +- First-call-wins on all three marks. A request produces at most one publisher-path + auction today; if a second auction ever occurs in one request, the row describes + the first and the auction dataset still carries both in full. +- Same locking and failure model as every other `RequestTimings` write: `try_lock`, + drop on contention, saturating conversion at serialization. + +### Row changes + +Four additive columns on `access_logs_raw`, all populated from the +`TimingSnapshot` at the existing freeze/emission points (no new emission path): + +``` +`auction_dispatched_ms` Nullable(UInt32), `json:$.auction_dispatched_ms` +`auction_resolved_ms` Nullable(UInt32), `json:$.auction_resolved_ms` +`auction_committed_ms` Nullable(UInt32), `json:$.auction_committed_ms` +`auction_id` String, `json:$.auction_id` +``` + +- The three offsets are null when no auction ran (the common case: assets, EC + endpoints, auction-disabled deployments). Null means "no auction", never "zero". +- `auction_id` is the telemetry auction UUID already present on every + `auction_events_raw` row, carried onto the access row as the join key between + the T0 timeline and per-bidder detail. Sentinel `none` when no auction ran, + matching the non-nullable-dimension convention of section 9. It is a random + UUID, not identity-bearing; unbounded cardinality is accepted for the same + reason it is accepted in the auction dataset. +- Schema evolution is additive with JSONPaths on every new column and + `FORWARD_QUERY` carrying the existing columns, per the deployed datasource's + established evolution path. Verified with `tb --cloud deploy --check` before + deploy. + +### Interpretation model + +Combined with existing columns, one access row now reads as a timeline: + +``` +t=0 ......... request entry +t=D ......... auction_dispatched_ms (bids out; origin fetch typically in flight) +t=R ......... auction_resolved_ms (R - D ~ auction duration; join auction_id + for the per-bidder long pole) +t=C ......... auction_committed_ms (targeting in page state) +t=H ......... time_elapsed_ms (headers committed) +``` + +Derivations the dashboard can add without schema help: auction duration on the +request clock (`R - D`), commit latency (`C - R`), and overlap ratio (share of +`R - D` that ran concurrently with `ts-origin`). `auction_wait_ms` keeps its +existing meaning (blocked time only) and is now interpretable next to the +timeline: `R - D` minus `auction_wait_ms` approximates how much of the auction +was absorbed by work the request needed anyway. + +### Scope + +- Fastly emits; Axum, Cloudflare, and Spin collect the marks but do not emit, + matching section 8a adapter semantics. +- No header emission for any of these values: they are post-hoc analysis fields, + and two of the three are typically unknown at the header freeze point in + streaming mode. +- No config surface: the marks are always-on collection like every other phase, + gated at emission by the existing `tinybird.access_enabled`. diff --git a/fastly.toml b/fastly.toml index a3812999b..ea13b4c6a 100644 --- a/fastly.toml +++ b/fastly.toml @@ -40,10 +40,6 @@ build = """ [[local_server.kv_stores.ec_identity_store]] key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01" data = '{"v":1,"created":1700000000,"last_seen":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US"}}' - - [[local_server.kv_stores.consent_store]] - key = "placeholder" - data = "placeholder" [local_server.secret_stores] [[local_server.secret_stores.signing_keys]] key = "ts-2025-10-A" @@ -78,6 +74,8 @@ build = """ [local_server.config_stores.edgezero_runtime_env] format = "inline-toml" [local_server.config_stores.edgezero_runtime_env.contents] + # Viceroy reports this fixed synthetic service id. EdgeZero scopes + # Fastly runtime mappings by service id. EDGEZERO__SERVICES__0000000000000000000000__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME = "ts_secrets" [local_server.config_stores.trusted_server_config] diff --git a/scripts/template-cache-local-test.sh b/scripts/template-cache-local-test.sh index 58d714cf8..53e86837f 100755 --- a/scripts/template-cache-local-test.sh +++ b/scripts/template-cache-local-test.sh @@ -238,6 +238,7 @@ s = replace_once( f'origin_url = "http://127.0.0.1:{origin_port}"', "publisher origin", ) +# The example publisher domains are reserved placeholders that validation rejects. s = replace_once( s, 'domain = "example.com"', @@ -250,6 +251,7 @@ s = replace_once( 'cookie_domain = ".local-harness.example"', "publisher cookie domain", ) + # A real auction points at the slow HTTPS stub so the timings mean something. s = replace_once( s, diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index eef9e2f7d..7b562c96e 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -19,3 +19,20 @@ if ! rustup target list --installed | awk -v target="$HOST_TARGET" '$0 == target fi cargo test --package trusted-server-cli --target "$HOST_TARGET" +export TS_AUDIT_BROWSER_TESTS=1 +AUDIT_BROWSER_TEST_FILTERS=( + "commands::audit::browser::tests::" + "commands::audit::generate::browser_collector::tests::" +) +for AUDIT_BROWSER_TEST_FILTER in "${AUDIT_BROWSER_TEST_FILTERS[@]}"; do + AUDIT_BROWSER_TEST_COUNT="$({ + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --list + } | awk '/: test$/ { count += 1 } END { print count + 0 }')" + if [ "$AUDIT_BROWSER_TEST_COUNT" -eq 0 ]; then + echo "No ignored browser audit fixtures matched $AUDIT_BROWSER_TEST_FILTER" >&2 + exit 1 + fi + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --test-threads=1 +done diff --git a/tinybird/datasources/access_logs_raw.datasource b/tinybird/datasources/access_logs_raw.datasource index 42f214e07..4441de384 100644 --- a/tinybird/datasources/access_logs_raw.datasource +++ b/tinybird/datasources/access_logs_raw.datasource @@ -1,19 +1,43 @@ DESCRIPTION > - Optional sampled Trusted Server access telemetry rows. Disabled by default in Fastly config. + Per-request phase-timing telemetry rows, sampled and emitted post-send by the edge service. SCHEMA > - `event_ts` DateTime64(3), - `method` LowCardinality(String), - `path` String, - `status` UInt16, - `time_elapsed_ms` UInt32, - `cache_state` LowCardinality(Nullable(String)), - `country` LowCardinality(String), - `sample_rate` Float64, - `event_date` Date DEFAULT toDate(event_ts) + `event_ts` DateTime64(3) `json:$.event_ts`, + `method` LowCardinality(String) `json:$.method`, + `status` UInt16 `json:$.status`, + `time_elapsed_ms` Nullable(UInt32) `json:$.time_elapsed_ms`, + `sample_rate` Float64 `json:$.sample_rate`, + `service_id` LowCardinality(String) `json:$.service_id`, + `publisher_domain` LowCardinality(String) `json:$.publisher_domain`, + `env` LowCardinality(String) `json:$.env`, + `route_class` LowCardinality(String) `json:$.route_class`, + `route_template` String `json:$.route_template`, + `body_mode` LowCardinality(String) `json:$.body_mode`, + `auction_wait_placement` LowCardinality(String) `json:$.auction_wait_placement`, + `appbuild_ms` Nullable(UInt32) `json:$.appbuild_ms`, + `filter_ms` Nullable(UInt32) `json:$.filter_ms`, + `geo_ms` Nullable(UInt32) `json:$.geo_ms`, + `kv_ms` Nullable(UInt32) `json:$.kv_ms`, + `origin_ms` Nullable(UInt32) `json:$.origin_ms`, + `template_cache_ms` Nullable(UInt32) `json:$.template_cache_ms`, + `auction_wait_ms` Nullable(UInt32) `json:$.auction_wait_ms`, + `stream_ms` Nullable(UInt32) `json:$.stream_ms`, + `request_elapsed_ms` Nullable(UInt32) `json:$.request_elapsed_ms`, + `resp_bytes` Nullable(UInt64) `json:$.resp_bytes`, + `template_cache_state` LowCardinality(String) `json:$.template_cache_state`, + `country` LowCardinality(String) `json:$.country`, + `ts_version` LowCardinality(String) `json:$.ts_version`, + `pop` LowCardinality(String) `json:$.pop`, + `auction_dispatched_ms` Nullable(UInt32) `json:$.auction_dispatched_ms`, + `auction_resolved_ms` Nullable(UInt32) `json:$.auction_resolved_ms`, + `auction_committed_ms` Nullable(UInt32) `json:$.auction_committed_ms`, + `auction_id` String `json:$.auction_id` ENGINE "MergeTree" -ENGINE_SORTING_KEY "event_date, path, status, method" -TTL "event_date + INTERVAL 30 DAY" +ENGINE_SORTING_KEY "toDate(event_ts), service_id, publisher_domain, env, route_class, pop, status" +TTL "toDate(event_ts) + INTERVAL 30 DAY" + +FORWARD_QUERY > + SELECT event_ts, method, status, time_elapsed_ms, sample_rate, service_id, publisher_domain, env, route_class, route_template, body_mode, auction_wait_placement, appbuild_ms, filter_ms, geo_ms, kv_ms, origin_ms, template_cache_ms, auction_wait_ms, stream_ms, request_elapsed_ms, resp_bytes, template_cache_state, country, ts_version, pop, CAST(NULL AS Nullable(UInt32)) AS auction_dispatched_ms, CAST(NULL AS Nullable(UInt32)) AS auction_resolved_ms, CAST(NULL AS Nullable(UInt32)) AS auction_committed_ms, 'none' AS auction_id TOKEN ts_access_ingest APPEND diff --git a/tinybird/fixtures/access_logs_raw.ndjson b/tinybird/fixtures/access_logs_raw.ndjson new file mode 100644 index 000000000..983b3acd1 --- /dev/null +++ b/tinybird/fixtures/access_logs_raw.ndjson @@ -0,0 +1 @@ +{"event_ts":"2026-06-23 12:00:00.000","method":"GET","status":200,"time_elapsed_ms":145,"sample_rate":0.1,"service_id":"abc123","publisher_domain":"test-publisher.example","env":"production","route_class":"publisher_html","route_template":"/news/*","body_mode":"streamed","auction_wait_placement":"in_stream","appbuild_ms":12,"filter_ms":5,"geo_ms":3,"kv_ms":8,"origin_ms":25,"template_cache_ms":10,"auction_wait_ms":45,"stream_ms":18,"request_elapsed_ms":145,"resp_bytes":8192,"template_cache_state":"hit","country":"US","ts_version":"v1.2.3","pop":"SFO"} diff --git a/trusted-server.example.toml b/trusted-server.example.toml index c484519ad..c21f0bcd9 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -319,6 +319,12 @@ provider = "pbs-main" # inventory_domain = "publisher.example" # inventory_page_origin = "https://www.publisher.example" +[observability] +# Keep this leaf present so the environment override can apply; the overlay +# cannot create a missing configuration leaf. The Server-Timing header stays +# off until enabled. +server_timing_enabled = false + # Server-side ad slot templates + creative-opportunity auction. Kept active. [creative_opportunities] # Set false to disable server-side ad templates while keeping slot definitions @@ -409,8 +415,14 @@ auction_timeout_ms = 500 # [tinybird] # enabled = true # api_host = "api.us-east.tinybird.example" # required when enabled; host only -# auction_dataset = "auction_events" # Events API datasource name +# auction_enabled = true # emit auction telemetry +# auction_dataset = "auction_events_raw" # auction Events API datasource # auction_token_secret = "tinybird_auction_append_token" # Key in trusted_server_secrets +# access_enabled = false # emit sampled access telemetry +# access_dataset = "access_logs_raw" # access Events API datasource +# access_token_secret = "tinybird_access_append_token" # Key in trusted_server_secrets +# access_sample_rate = 0.0 # fraction from 0.0 through 1.0 +# max_body_bytes = 1048576 # maximum NDJSON request body # Debug endpoints (all default false — never enable in production). # [debug] @@ -472,6 +484,33 @@ client_side_bidders = [] # bidders running via native Prebid.js adapter # adapters = ["rubicon"] # user_id_modules = ["sharedIdSystem"] +# Prebid User ID modules that Trusted Server installs and keeps installed, so +# operators can manage identity centrally without publisher JavaScript changes. +# Each entry is forwarded to Prebid.js verbatim: `name` is a +# `userSync.userIds` entry name, `params` and `storage` are whatever that module +# documents. Trusted Server does not interpret them; supported names come from +# the checked-in User ID registry. Each `name` must appear only once. +# +# The module must be present in the built bundle: name it under +# [integrations.prebid.bundle].user_id_modules, or omit that list to take the +# generator's default preset. `ts prebid bundle` resolves every managed name +# through the checked-in User ID registry and fails if the generated manifest +# omits its required module without updating the configured hash or SRI. +# +# Persisting a resolved ID into the EC identity graph additionally needs a +# matching partner under [[ec.partners]] whose source_domain equals the module's +# OpenRTB EID source; without one the ID still reaches the auction but is never +# written to KV. +# +# [[integrations.prebid.managed_user_ids]] +# name = "sharedId" +# +# [integrations.prebid.managed_user_ids.storage] +# type = "cookie" # or "html5" +# name = "_sharedid" +# expires = 15 # days; omit to keep Prebid's default +# refresh_in_seconds = 1800 # omit to keep Prebid's default + # Next.js first-party rewriting for App Router / RSC payloads. # [integrations.nextjs] # enabled = true